README.md

@@ -19,12 +19,12 @@ You can install this package from npm using Bun:
 bun add @tscircuit/prompt-benchmarks
 ```
 
-## Using the AiCoder
+## Using the TscircuitCoder
 
-Below is the AiCoder interface:
+Below is the TscircuitCoder interface:
 
 ```tsx
-export interface AiCoder {
+export interface TscircuitCoder {
   onStreamedChunk: (chunk: string) => void
   onVfsChanged: () => void
   vfs: { [filepath: string]: string }
@@ -36,15 +36,15 @@ export interface AiCoder {
 }
 ```
 
-*Note: The `createAiCoder` function now accepts an optional `openaiClient` parameter to override the default openai client. This allows you to provide a custom client.
-The AI Coder supports streaming of AI responses and notifying you when the virtual file system (VFS) is updated. To achieve this, you can pass two callback functions when creating an AiCoder instance:
+*Note: The `createTscirciutCoder` function now accepts an optional `openaiClient` parameter to override the default openai client. This allows you to provide a custom client.
+The AI Coder supports streaming of AI responses and notifying you when the virtual file system (VFS) is updated. To achieve this, you can pass two callback functions when creating an TscircuitCoder instance:
 - **onStreamedChunk**: A callback function that receives streamed chunks from the AI. This is useful for logging or updating a UI with gradual progress.
 - **onVfsChanged**: A callback function that is invoked whenever the VFS is updated with new content. This is useful for refreshing a file view or triggering further processing.
 
 **Example Usage:**
 
 ```tsx
-import { createAiCoder } from "@tscircuit/prompt-benchmarks/lib/ai/aiCoder"
+import { createTscircuitCoder } from "@tscircuit/prompt-benchmarks/lib/ai/tscircuitCoder"
 
 // Define a callback for handling streamed chunks
 const handleStream = (chunk: string) => {
@@ -56,12 +56,12 @@ const handleVfsUpdate = () => {
   console.log("The virtual file system has been updated.")
 }
 
-// Create an instance of AiCoder with your callbacks
-const aiCoder = createAiCoder(handleStream, handleVfsUpdate)
+// Create an instance of TscircuitCoder with your callbacks
+const tscircuitCoder = createTscircuitCoder(handleStream, handleVfsUpdate)
 
 // Submit a prompt to generate a circuit.
 // The onStream callback logs streaming updates and onVfsChanged notifies when a new file is added to the VFS.
-aiCoder.submitPrompt("create a circuit that blinks an LED")
+tscircuitCoder.submitPrompt("create a circuit that blinks an LED")
 ```
 
 ## Running Benchmarks

benchmarks/benchmark-local-circuit-error-correction.eval.ts

@@ -2,7 +2,7 @@ import path from "node:path"
 import { createLocalCircuitPrompt } from "../lib/prompt-templates/create-local-circuit-prompt"
 import { evalite } from "evalite"
 import { CircuitScorer } from "./scorers/circuit-scorer"
-import { askAboutOutput } from "lib/ai/ask-about-output"
+import { askAiAboutOutput } from "lib/ask-ai/ask-ai-about-output"
 import { cleanupAttemptLogs } from "lib/utils/cleanup-attempt-logs"
 import { savePrompt } from "lib/utils/save-prompt"
 import { loadProblems } from "lib/utils/load-problems"
@@ -52,7 +52,7 @@ evalite("Reasoning Electronics Engineer", {
       output.code = codeBlock
       for (const question of input.questions) {
         output.results.push({
-          result: await askAboutOutput(code, question.text),
+          result: await askAiAboutOutput(code, question.text),
           expected: question.answer,
         })
       }

benchmarks/benchmark-local-circuit-random.eval.ts

@@ -4,7 +4,7 @@ import { evalite } from "evalite"
 import { AiCircuitScorer } from "./scorers/ai-circuit-scorer"
 import { cleanupAttemptLogs } from "lib/utils/cleanup-attempt-logs"
 import { savePrompt } from "lib/utils/save-prompt"
-import { runAiWithErrorCorrection } from "lib/ai/run-ai-with-error-correction"
+import { runAiWithErrorCorrection } from "lib/tscircuit-coder/run-ai-with-error-correction"
 import { generateRandomPrompts } from "lib/utils/generate-random-prompts"
 
 const logsDir = path.join(__dirname, "./attempt-logs")

benchmarks/benchmark-local-circuit.eval.ts

@@ -2,11 +2,11 @@ import path from "node:path"
 import { createLocalCircuitPrompt } from "../lib/prompt-templates/create-local-circuit-prompt"
 import { evalite } from "evalite"
 import { CircuitScorer } from "./scorers/circuit-scorer"
-import { askAboutOutput } from "lib/ai/ask-about-output"
+import { askAiAboutOutput } from "lib/ask-ai/ask-ai-about-output"
 import { savePrompt } from "lib/utils/save-prompt"
 import { loadProblems } from "lib/utils/load-problems"
-import { askAi } from "lib/ai/ask-ai"
-import { evaluateTscircuitCode } from "lib/ai/evaluate-tscircuit-code"
+import { askAi } from "lib/ask-ai/ask-ai"
+import { evaluateTscircuitCode } from "lib/utils/evaluate-tscircuit-code"
 
 let systemPrompt = ""
 
@@ -49,7 +49,7 @@ evalite("Electronics Engineer", {
       output.code = codeBlock
       for (const question of input.questions) {
         output.results.push({
-          result: await askAboutOutput(code, question.text),
+          result: await askAiAboutOutput(code, question.text),
           expected: question.answer,
         })
       }

lib/ai/index.ts

@@ -1,5 +1 @@
-export * from "./ask-ai"
-export * from "./ask-ai-with-previous-attempts"
-export * from "./run-ai-with-error-correction"
-export * from "./aiCoder"
 export * from "./openai"

lib/ai/ask-about-output.ts → lib/ask-ai/ask-ai-about-output.ts

@@ -1,6 +1,6 @@
 import { openai } from "lib/ai/openai"
 
-export const askAboutOutput = async (
+export const askAiAboutOutput = async (
   codefence: string,
   question: string,
 ): Promise<boolean> => {

lib/ask-ai/index.ts

@@ -0,0 +1,3 @@
+export * from "./ask-ai"
+export * from "./ask-ai-about-output"
+export * from "./ask-ai-with-previous-attempts"

lib/index.ts

@@ -1,3 +1,5 @@
 export * from "./ai/index"
 export * from "./prompt-templates/index"
+export * from "./ask-ai/index"
+export * from "./tscircuit-coder/index"
 export * from "./utils/get-imports-from-code"

lib/tscircuit-coder/index.ts

@@ -0,0 +1,2 @@
+export * from "./tscircuitCoder"
+export * from "./run-ai-with-error-correction"

lib/ai/run-ai-with-error-correction.ts → lib/tscircuit-coder/run-ai-with-error-correction.ts

@@ -1,7 +1,7 @@
-import { askAiWithPreviousAttempts } from "./ask-ai-with-previous-attempts"
+import { askAiWithPreviousAttempts } from "../ask-ai/ask-ai-with-previous-attempts"
 import { saveAttemptLog } from "lib/utils/save-attempt"
 import type OpenAI from "openai"
-import { evaluateTscircuitCode } from "./evaluate-tscircuit-code"
+import { evaluateTscircuitCode } from "../utils/evaluate-tscircuit-code"
 
 const createAttemptFile = ({
   fileName,

lib/ai/aiCoder.ts → lib/tscircuit-coder/tscircuitCoder.ts

@@ -3,25 +3,25 @@ import type { OpenAI } from "openai"
 import { runAiWithErrorCorrection } from "./run-ai-with-error-correction"
 import { createLocalCircuitPrompt } from "lib/prompt-templates/create-local-circuit-prompt"
 
-export interface AiCoderEvents {
+export interface TscircuitCoderEvents {
   streamedChunk: string
   vfsChanged: undefined
 }
 
-export interface AiCoder {
+export interface TscircuitCoder {
   vfs: { [filepath: string]: string }
   availableOptions: { name: string; options: string[] }[]
   submitPrompt: (
     prompt: string,
     options?: { selectedMicrocontroller?: string },
   ) => Promise<void>
-  on<K extends keyof AiCoderEvents>(
+  on<K extends keyof TscircuitCoderEvents>(
     event: K,
-    listener: (payload: AiCoderEvents[K]) => void,
+    listener: (payload: TscircuitCoderEvents[K]) => void,
   ): this
 }
 
-export class AiCoderImpl extends EventEmitter implements AiCoder {
+export class TscircuitCoderImpl extends EventEmitter implements TscircuitCoder {
   vfs: { [filepath: string]: string } = {}
   availableOptions = [{ name: "microController", options: ["pico", "esp32"] }]
   openaiClient: OpenAI | undefined
@@ -70,6 +70,6 @@ export class AiCoderImpl extends EventEmitter implements AiCoder {
   }
 }
 
-export const createAiCoder = (openaiClient?: OpenAI): AiCoder => {
-  return new AiCoderImpl({ openaiClient })
+export const createTscircuitCoder = (openaiClient?: OpenAI): TscircuitCoder => {
+  return new TscircuitCoderImpl({ openaiClient })
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tscircuit/prompt-benchmarks",
-  "version": "0.0.40",
+  "version": "0.0.41",
   "main": "dist/index.js",
   "repository": "github:tscircuit/prompt-benchmarks",
   "homepage": "https://github.com/tscircuit/prompt-benchmarks#readme",

tests/aiCoder.test.ts

@@ -0,0 +1,23 @@
+import { createTscircuitCoder } from "lib/tscircuit-coder/tscircuitCoder"
+import { expect, test } from "bun:test"
+
+test("TscircuitCoder submitPrompt streams and updates vfs", async () => {
+  const streamedChunks: string[] = []
+  let vfsUpdated = false
+  const tscircuitCoder = createTscircuitCoder()
+  tscircuitCoder.on("streamedChunk", (chunk: string) => {
+    streamedChunks.push(chunk)
+  })
+  tscircuitCoder.on("vfsChanged", () => {
+    vfsUpdated = true
+  })
+
+  await tscircuitCoder.submitPrompt(
+    "create a random complicated circuit that does something cool",
+  )
+
+  expect(streamedChunks.length).toBeGreaterThan(0)
+  const vfsKeys = Object.keys(tscircuitCoder.vfs)
+  expect(vfsKeys.length).toBeGreaterThan(0)
+  expect(vfsUpdated).toBe(true)
+})

tests/lib/evaluate-tscircuit-code.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from "bun:test"
-import { evaluateTscircuitCode } from "../../lib/ai/evaluate-tscircuit-code"
+import { evaluateTscircuitCode } from "../../lib/utils/evaluate-tscircuit-code"
 
 const validCircuit = `
 export const StrainGaugeAmplifier = () => (