lib/ai/evaluate-tscircuit-code.ts

@@ -0,0 +1,25 @@
+import { CircuitRunner } from "@tscircuit/eval/eval"
+
+export const evaluateTscircuitCode = async (
+  code: string,
+): Promise<{ success: boolean; error?: string }> => {
+  const circuitRunner = new CircuitRunner()
+  try {
+    await circuitRunner.execute(addBoardToRootCircuit(code))
+    await circuitRunner.renderUntilSettled()
+    return { success: true }
+  } catch (e) {
+    return { success: false, error: e as string }
+  }
+}
+
+export const addBoardToRootCircuit = (code: string): string => {
+  const circuitAddStart = `circuit.add(\n`
+  const circuitAddEnd = `\n)`
+
+  const transformedCode = code
+    .replace(/export const \w+ = \(\) => \(/, circuitAddStart)
+    .replace(/\n\)$/, circuitAddEnd)
+
+  return transformedCode
+}

lib/ai/run-ai-with-error-correction.ts

@@ -1,7 +1,8 @@
-import { safeEvaluateCode } from "lib/code-runner"
 import { askAiWithPreviousAttempts } from "./ask-ai-with-previous-attempts"
 import { saveAttemptLog } from "lib/utils/save-attempt"
 import type OpenAI from "openai"
+import { evaluateTscircuitCode } from "./evaluate-tscircuit-code"
+
 const createAttemptFile = ({
   fileName,
   prompt,
@@ -75,17 +76,15 @@ export const runAiWithErrorCorrection = async ({
   const code = codeMatch ? codeMatch[1].trim() : ""
   const codeBlockMatch = aiResponse.match(/```tsx[\s\S]*?```/)
   const codeBlock = codeBlockMatch ? codeBlockMatch[0] : ""
-  const evaluation = safeEvaluateCode(code, {
-    outputType: "board",
-    preSuppliedImports: {},
-  })
 
-  if (evaluation.success) {
+  const { success, error: evaluationError } = await evaluateTscircuitCode(code)
+
+  if (success) {
     if (onStream) onStream("Local tscircuit circuit created")
     return { code, codeBlock, error: "" }
   }
 
-  const error = evaluation.error || ""
+  const error = evaluationError || ""
   previousAttempts.push({ code, error })
   const attemptFileName = `prompt-${promptNumber}-attempt-${attempt}.md`
   if (logsDir)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tscircuit/prompt-benchmarks",
-  "version": "0.0.38",
+  "version": "0.0.39",
   "main": "dist/index.js",
   "repository": "github:tscircuit/prompt-benchmarks",
   "homepage": "https://github.com/tscircuit/prompt-benchmarks#readme",
@@ -41,13 +41,16 @@
   },
   "dependencies": {
     "@babel/standalone": "^7.25.7",
+    "@tscircuit/eval": "^0.0.93",
     "@tscircuit/featured-snippets": "^0.0.1",
     "@tscircuit/footprinter": "^0.0.102",
+    "comlink": "^4.4.2",
     "debug": "^4.3.7",
     "dotenv": "^16.4.7",
     "evalite": "^0.8.4",
     "extract-codefence": "^0.0.4",
     "openai": "^4.83.0",
+    "jscad-fiber": "^0.0.77",
     "toml": "^3.0.0"
   }
 }

tests/lib/evaluate-tscircuit-code.test.ts

@@ -0,0 +1,57 @@
+import { describe, it, expect } from "bun:test"
+import { evaluateTscircuitCode } from "../../lib/ai/evaluate-tscircuit-code"
+
+const validCircuit = `
+export const StrainGaugeAmplifier = () => (
+  <board width="50mm" height="40mm">
+    <chip footprint="soic8" name="U1" pinLabels={{
+      1: "V+",
+      2: "V-",
+      3: "Ref",
+      4: "In+",
+      5: "In-",
+      6: "Out",
+      7: "Gain",
+      8: "GND",
+    }} pcbX="10mm" pcbY="10mm" />
+
+    <resistor name="R1" resistance="1k" footprint="0402" pcbX="12mm" pcbY="10mm" />
+    <resistor name="R2" resistance="1k" footprint="0402" pcbX="10mm" pcbY="12mm" />
+    <resistor name="R3" resistance="1k" footprint="0402" pcbX="10mm" pcbY="8mm" />
+    <capacitor name="C1" capacitance="100nF" footprint="0603" pcbX="10mm" pcbY="14mm" />
+
+    <trace from=".U1 > .pin4" to=".R1 > .pin1" />
+    <trace from=".U1 > .pin5" to=".R2 > .pin1" />
+    <trace from=".R1 > .pin2" to=".R2 > .pin2" />
+    <trace from=".U1 > .pin6" to=".R3 > .pin1" />
+    <trace from=".R3 > .pin2" to=".U1 > .GND" />
+    <trace from=".R3 > .pin2" to=".C1 > .pin1" />
+    <trace from=".C1 > .pin2" to=".U1 > .pin3" />
+
+    <net name="VCC" />
+    <net name="GND" />
+  </board>
+)
+`
+
+const invalidCircuit = `
+export const InvalidCircuit = () => (
+  <board width="50mm" height="40mm">
+    <invalidComponent />
+  </board>
+)
+`
+
+describe("evaluateTscircuitCode", () => {
+  it("should successfully evaluate a valid circuit", async () => {
+    const result = await evaluateTscircuitCode(validCircuit)
+    expect(result.success).toBe(true)
+    expect(result.error).toBeUndefined()
+  })
+
+  it("should fail to evaluate an invalid circuit", async () => {
+    const result = await evaluateTscircuitCode(invalidCircuit)
+    expect(result.success).toBe(false)
+    expect(result.error).toBeDefined()
+  })
+})