Web UI for running evals (promptfoo#103)

Author: typpo

.prettierignore

@@ -1,2 +1,5 @@
 dist
 venv
+.aider*
+src/web/nextui/out
+src/web/nextui/.next

package-lock.json

@@ -25,13 +25,14 @@
     "promptfoo": "dist/src/main.js"
   },
   "scripts": {
+    "tsc": "tsc",
     "local": "ts-node --esm --files src/main.ts",
-    "install:client": "cd src/web/client && npm install",
+    "install:nextui": "cd src/web/nextui && npm install",
     "build:clean": "rm -rf dist",
-    "build:client": "cd src/web/client && npm run build && cp -r dist/ ../../../dist/src/web/client",
+    "build:nextui": "cd src/web/nextui && npm run build && cp -r out/ ../../../dist/src/web/nextui",
     "build:watch": "tsc --watch",
-    "build": "tsc && cp src/*.html dist/src && npm run build:client && chmod +x dist/src/main.js",
-    "prepare": "npm run install:client && npm run build:clean && npm run build",
+    "build": "tsc && cp src/*.html dist/src && npm run build:nextui && chmod +x dist/src/main.js",
+    "prepare": "npm run install:nextui && npm run build:clean && npm run build",
     "test": "jest",
     "test:watch": "jest --watch",
     "format": "prettier -w ."
@@ -41,6 +42,7 @@
     "@types/cache-manager": "^4.0.2",
     "@types/cache-manager-fs-hash": "^0.0.1",
     "@types/cli-progress": "^3.11.0",
+    "@types/compression": "^1.7.2",
     "@types/cors": "^2.8.13",
     "@types/debounce": "^1.2.1",
     "@types/express": "^4.17.17",
@@ -51,6 +53,7 @@
     "@types/nunjucks": "^3.2.2",
     "@types/opener": "^1.4.0",
     "@types/semver": "^7.5.0",
+    "@types/uuid": "^9.0.2",
     "babel-jest": "^29.5.0",
     "jest": "^29.5.0",
     "jest-watch-typeahead": "^2.2.2",
@@ -69,6 +72,7 @@
     "cli-progress": "^3.12.0",
     "cli-table3": "^0.6.3",
     "commander": "^10.0.1",
+    "compression": "^1.7.4",
     "cors": "^2.8.5",
     "csv-parse": "^5.3.8",
     "csv-stringify": "^6.3.2",
@@ -84,6 +88,7 @@
     "semver": "^7.5.3",
     "socket.io": "^4.6.1",
     "tiny-invariant": "^1.3.1",
+    "uuid": "^9.0.0",
     "winston": "^3.8.2"
   }
 }

package.json

@@ -385,7 +385,6 @@ class Evaluator {
     // Set up progress bar...
     let progressbar: SingleBar | undefined;
     if (options.showProgressBar) {
-      const totalNumRuns = runEvalOptions.length;
       const cliProgress = await import('cli-progress');
       progressbar = new cliProgress.SingleBar(
         {
@@ -394,34 +393,40 @@ class Evaluator {
         },
         cliProgress.Presets.shades_classic,
       );
-      progressbar.start(totalNumRuns, 0, {
+      progressbar.start(runEvalOptions.length, 0, {
         provider: '',
         prompt: '',
         vars: '',
       });
     }
+    if (options.progressCallback) {
+      options.progressCallback(0, runEvalOptions.length);
+    }
 
     // Actually run the eval
     const results: EvaluateResult[] = [];
     await async.forEachOfLimit(
       runEvalOptions,
       options.maxConcurrency || DEFAULT_MAX_CONCURRENCY,
-      async (options: RunEvalOptions, index: number | string) => {
-        const row = await this.runEval(options);
+      async (evalStep: RunEvalOptions, index: number | string) => {
+        const row = await this.runEval(evalStep);
 
         results.push(row);
 
         if (progressbar) {
           progressbar.increment({
-            provider: options.provider.id(),
-            prompt: options.prompt.raw.slice(0, 10).replace(/\n/g, ' '),
-            vars: Object.entries(options.test.vars || {})
+            provider: evalStep.provider.id(),
+            prompt: evalStep.prompt.raw.slice(0, 10).replace(/\n/g, ' '),
+            vars: Object.entries(evalStep.test.vars || {})
               .map(([k, v]) => `${k}=${v}`)
               .join(' ')
               .slice(0, 10)
               .replace(/\n/g, ' '),
           });
         }
+        if (options.progressCallback) {
+          options.progressCallback(results.length, runEvalOptions.length);
+        }
 
         // Bookkeeping for table
         if (typeof index !== 'number') {
@@ -441,13 +446,13 @@ class Evaluator {
           resultText = row.response?.output || row.error || '';
         }
 
-        const { rowIndex, colIndex } = options;
+        const { rowIndex, colIndex } = evalStep;
         if (!table.body[rowIndex]) {
           table.body[rowIndex] = {
             outputs: [],
             vars: table.head.vars
               .map((varName) => {
-                const varValue = options.test.vars?.[varName] || '';
+                const varValue = evalStep.test.vars?.[varName] || '';
                 if (typeof varValue === 'string') {
                   return varValue;
                 }
@@ -475,6 +480,9 @@ class Evaluator {
     if (progressbar) {
       progressbar.stop();
     }
+    if (options.progressCallback) {
+      options.progressCallback(runEvalOptions.length, runEvalOptions.length);
+    }
 
     telemetry.record('eval_ran', {});