big update

Author: nima-taheri-mongodb

.gitignore

@@ -13,6 +13,7 @@ state.json
 
 tests/tmp
 coverage
+
 # Generated assets by accuracy runs
 .accuracy
 
@@ -22,3 +23,8 @@ temp
 # Development tool files
 .yalc
 yalc.lock
+
+# Braintrust
+.bt
+# Generated eval pre-bundle for Braintrust
+tests/eval/dist

.vscode/launch.json

@@ -44,6 +44,16 @@
         "scripts/generate/index.ts"
       ],
       "cwd": "${workspaceFolder}"
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug Eval",
+      "skipFiles": ["<node_internals>/**"],
+      "runtimeExecutable": "pnpm",
+      "runtimeArgs": ["run", "eval:debug"],
+      "cwd": "${workspaceFolder}",
+      "envFile": "${workspaceFolder}/.env"
     }
   ]
 }

package.json

@@ -97,7 +97,14 @@
     "test:browser": "pnpm --filter browser test",
     "test:long-running-tests": "vitest --project long-running-tests --coverage",
     "test:local": "SKIP_ATLAS_INTEGRATION_TESTS=true SKIP_ATLAS_LOCAL_TESTS=true pnpm run test",
-    "atlas:cleanup": "vitest --project atlas-cleanup"
+    "atlas:cleanup": "vitest --project atlas-cleanup",
+    "eval:db-start": "sh tests/eval/scripts/evalDb.sh start",
+    "eval:db-stop": "sh tests/eval/scripts/evalDb.sh stop",
+    "eval:run": "bt eval tests/eval/mongodb.eval.ts",
+    "eval:debug": "tsx tests/eval/mongodb.eval.ts",
+    "eval:generate-schemas": "tsx tests/eval/scripts/generateSchemas.ts",
+    "eval:bundle": "tsx tests/eval/scripts/bundleEval.ts",
+    "eval:push": "pnpm run eval:bundle && braintrust push tests/eval/dist/mongodb.eval.cjs --if-exists replace"
   },
   "license": "Apache-2.0",
   "devDependencies": {
@@ -126,7 +133,8 @@
     "@vitest/coverage-v8": "^4.1.4",
     "@vitest/eslint-plugin": "^1.6.16",
     "ai": "^6.0.168",
-    "braintrust": "^3.9.0",
+    "esbuild": "^0.27.1",
+    "braintrust": "^3.17.0",
     "concurrently": "^9.2.1",
     "duplexpair": "^1.0.2",
     "eslint": "^10.2.0",