Refactor code_embedder.ts to remove database dependencies and implement in-memory embeddings; add file_embeddings.json for storing embeddings; enhance get_skeleton.ts with improved file handling and embedding logic; update package.json to remove unused dependencies.

2026-05-03 05:23:57 +00:00 · 2025-05-10 22:44:43 +02:00 · 2025-05-10 22:44:43 +02:00 · 1693eeba22
commit 1693eeba22
parent 2732eb4d92
5 changed files with 134 additions and 367 deletions
--- a/.gitignore
+++ b/.gitignore
@ -33,4 +33,5 @@ into-one.sh
 skeleton/
 .DS_Store
 node_modules
-code_embeddings.db
+code_embeddings.db
+file_embeddings.json
--- a/code_embedder.ts
+++ b/code_embedder.ts
@ -1,25 +1,13 @@
 import { config } from "dotenv";
-import { createHash } from "node:crypto";
-import { readFileSync, readdirSync, statSync } from "node:fs";
+import { readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
 import path from "node:path";
 import { z } from "zod";
 import { OpenAI } from "openai";
-import { createClient } from "@libsql/client";
-import {
-	encode,
-	encodeChat,
-	decode,
-	isWithinTokenLimit,
-	encodeGenerator,
-	decodeGenerator,
-	decodeAsyncGenerator,
-} from "gpt-tokenizer";
+import { isWithinTokenLimit } from "gpt-tokenizer";

 // configuration
-const DB_PATH = "file:code_embeddings.db";
 const SOURCE_FILE_EXTENSIONS = [".c", ".cpp", ".h", ".hpp", ".hxx"];
 const OPENAI_MODEL = "text-embedding-3-large";
-const EMBEDDING_DIMENSION = 3072;

 // env
 config();
@ -28,24 +16,6 @@ const { OPENAI_API_KEY } = EnvSchema.parse(process.env);

 // clients
 const openai = new OpenAI({ apiKey: OPENAI_API_KEY });
-const db = createClient({ url: DB_PATH });
-
-await db.execute(
-	`CREATE TABLE IF NOT EXISTS file_embeddings (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    file_path TEXT UNIQUE NOT NULL,
-    checksum TEXT NOT NULL,
-    embedding F32_BLOB(${EMBEDDING_DIMENSION}) NOT NULL,
-    last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-  );
-	CREATE INDEX IF NOT EXISTS file_embeddings_idx ON file_embeddings (libsql_vector_idx(embedding));`,
-);
-
-const getChecksum = (filePath: string): string => {
-	const hash = createHash("sha256");
-	hash.update(readFileSync(filePath));
-	return hash.digest("hex");
-};

 const getEmbedding = async (content: string): Promise<number[]> => {
 	const token_limit = 8000;
@ -58,7 +28,7 @@ const getEmbedding = async (content: string): Promise<number[]> => {
 		input: c,
 		encoding_format: "float",
 	});
-	return data[0].embedding as unknown as number[];
+	return data[0].embedding;
 };

 const findFiles = (startDir: string): string[] => {
@ -79,33 +49,24 @@ const findFiles = (startDir: string): string[] => {
 	return files;
 };

-const processFile = async (filePath: string) => {
+const processFile = async (
+	filePath: string,
+	embeddings: Record<string, number[]>,
+) => {
 	const content = readFileSync(filePath, "utf8");
 	if (!content.trim()) {
 		console.log(`Skipping ${filePath} because it is empty`);
 		return;
 	}

-	const checksum = getChecksum(filePath);
-	const existing = await db.execute(
-		"SELECT checksum FROM file_embeddings WHERE file_path = ?",
-		[filePath],
-	);
-	const row = existing.rows[0];
-	if (row && row.checksum === checksum) {
+	if (embeddings[filePath] != null) {
 		console.log(`Skipping ${filePath} because it already exists`);
 		return;
 	}

 	const embedding = await getEmbedding(content);

-	// await db.execute(
-	// 	`INSERT INTO file_embeddings (file_path, checksum, embedding) VALUES ('${filePath}', '${checksum}', vector32('[${embedding.join(",")}]'))`,
-	// );
-	await db.execute(
-		"INSERT INTO file_embeddings (file_path, checksum, embedding) VALUES (?, ?, ?)",
-		[filePath, checksum, new Uint8Array(embedding)],
-	);
+	embeddings[filePath] = embedding;
 };

 const targetDirectory = process.argv[2] ?? ".";
@ -114,9 +75,12 @@ if (!statSync(targetDirectory).isDirectory()) {
 	process.exit(1);
 }
 const files = findFiles(targetDirectory);
+const embeddings: Record<string, number[]> = {};
 for (let n = 0; n < files.length; ++n) {
 	console.log(`Processing ${files[n]} (${n + 1}/${files.length})`);
-	await processFile(files[n]);
+	await processFile(files[n], embeddings);
 }
+
+writeFileSync("file_embeddings.json", JSON.stringify(embeddings, null, 2));
 console.log("Done");
 process.exit(0);
--- a/get_skeleton.ts
+++ b/get_skeleton.ts
@ -1,26 +1,83 @@
 import fs from "node:fs/promises";
 import path from "node:path";
-import { config } from "dotenv";
+import "dotenv/config";
 import OpenAI from "openai";
 import pLimit from "p-limit";
+import { isWithinTokenLimit } from "gpt-tokenizer";
+import { readdirSync, readFileSync } from "node:fs";

-config();
+const embeddings = JSON.parse(
+	readFileSync("file_embeddings.json", "utf8"),
+) as Record<string, number[]>;
+
+const cosineDistance = (a: number[], b: number[]): number => {
+	const dotProduct = a.reduce((acc, val, i) => acc + val * b[i], 0);
+	const magnitudeA = Math.sqrt(a.reduce((acc, val) => acc + val * val, 0));
+	const magnitudeB = Math.sqrt(b.reduce((acc, val) => acc + val * val, 0));
+	return dotProduct / (magnitudeA * magnitudeB);
+};
+
+const getRelevantFiles = (embedding: number[], count: number): string[] => {
+	const result = Object.entries(embeddings)
+		.filter(([file, _]) => !file.includes("skeleton"))
+		.map(([file, fileEmbedding]) => ({
+			file,
+			distance: cosineDistance(fileEmbedding, embedding),
+		}));
+	return result
+		.sort((a, b) => a.distance - b.distance)
+		.slice(0, count)
+		.map((r) => r.file);
+};

 const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

-const getSkeleton = async (header: string, source: string): Promise<string> => {
+const OPENAI_MODEL = "text-embedding-3-large";
+const getEmbedding = async (content: string): Promise<number[]> => {
+	const token_limit = 8000;
+	let c = content;
+	while (!isWithinTokenLimit(c, token_limit)) {
+		c = c.slice(0, c.length - 100);
+	}
+	const { data } = await client.embeddings.create({
+		model: OPENAI_MODEL,
+		input: c,
+		encoding_format: "float",
+	});
+	return data[0].embedding as unknown as number[];
+};
+
+const returnInstructions =
+	"Only return the header file with added Doxygen annotations, nothing else. Do NOT return the source file. Do NOT put it into a ``` markdown code block";
+
+const getSkeleton = async (
+	header: string,
+	source: string | null,
+): Promise<string> => {
+	const embedding = await getEmbedding(header);
+	const relevantFiles = getRelevantFiles(embedding, 5);
+
+	const files = await Promise.all(
+		relevantFiles.map(async (f) => ({
+			content: await fs.readFile(f, "utf8"),
+			filename: f,
+		})),
+	);
+
 	const response = await client.responses.create({
 		model: "gpt-4.1",
 		instructions: `You will receive the code of a header and source file. Add Doxygen annotations to it.

 Use the documentation and source files to understand what every class, member, method and function does. Add detailed explanations for everything in the annotations.

- Prefix everything you add with "[AI]" so that we can validate it later
+- Prefix everything you add with "[AI]" so that we can validate it later. Put [AI] always behind the annotation, e.g. @brief [AI]. Add this for every annotation you add: @brief [AI] / @param PARAM [AI] / @details [AI] and so on.
 - If there are already comment or annotations, add them to your annotations.
 - Ignore the cpp and reference files. They only there for you to give you some context for the annotations.
- Don't add generic annotations that are not helpful. For example, if the function is getPosition, don't add an annotation like "Gets the position"
- If you encounter a class/method or member that is unknown (e.g. m_unk0x10, FUN_10001510 etc.) and you know what it is or does, add a "[AI_SUGGESTED_NAME: <name>]" in the annotations to it.
- Only return the annotated header file, nothing else. Do NOT return the source file. Do NOT put it into a \`\`\` markdown code block.
+- Don't add generic annotations that are not helpful. For example, if the function is GetPosition, don't add an annotation like "Gets the position" or if the result is LegoResult, don't add an annotation like "Result status"—thats obvious.
+- Don't add useless @return annotations that only state the type, e.g. @return unsigned long [AI]
+- If you encounter a class/method or member that is unknown (e.g. m_unk0x10, FUN_10001510, LegoUnknown100db7f4, VTable0x04 etc.) and you know what it is or does, add a "[AI_SUGGESTED_NAME: <name>]" with your suggested name in the annotations to it. But only do this if you know what it is.
+- If you encounter a class/method or member that is named the wrong way (you assume the name is wrong, misleading or erroneous) add a "[AI_SUGGESTED_CORRECTED_NAME: <name>]" in the annotations to it. Use this sparingly and only if your are sure that something is wrong. 
+- ${returnInstructions}

 I will now give you a rough documentation of the codebase to use as a reference:

@ -312,14 +369,23 @@ Result:
 p_output = "456"
 Return value: TRUE

-Return just the annotated header file, nothing else. Do NOT return the source file. Do NOT put it into a \`\`\` markdown code block.
+Here are some relevant source and header files. They are just a reference for you to better understand the code:
+${files.map((f) => `### ${f.filename}\n${f.content}`).join("\n\n")}

-Here are some relevant source and header files. They are just a reference for you to better understand the code:`,
+${returnInstructions}`,
 		input: `Header: ${header}
-Source: $source
+${source ? `Source: ${source}` : ""}

-Now return just the annotated header file, nothing else. Do NOT return the source file. Do NOT put it into a \`\`\` markdown code block.`,
+${returnInstructions}`,
 	});
+	if (response.usage) {
+		console.log(
+			(
+				(response.usage.input_tokens / 1_000_000) * 2 +
+				(response.usage.output_tokens / 1_000_000) * 8
+			).toLocaleString("en-US", { style: "currency", currency: "USD" }),
+		);
+	}
 	return response.output_text;
 };

@ -356,6 +422,26 @@ const ensureDir = async (dir: string): Promise<void> => {
 	await fs.mkdir(dir, { recursive: true });
 };

+export const findFileBackward = (dir: string, s: string): string | null => {
+	const entries = readdirSync(dir, { withFileTypes: true }).sort((a, b) =>
+		b.name.localeCompare(a.name),
+	);
+	for (let n = 0; n < entries.length; ++n) {
+		const entry = entries[n];
+		const p = path.join(dir, entry.name);
+		if (entry.isFile() && entry.name === s) {
+			return p;
+		}
+		if (entry.isDirectory()) {
+			const r = findFileBackward(p, s);
+			if (r) {
+				return r;
+			}
+		}
+	}
+	return null;
+};
+
 const handle = async (
 	n: number,
 	h: string,
@ -363,26 +449,9 @@ const handle = async (
 	root: string,
 	outRoot: string,
 ): Promise<void> => {
-	let s = `${h.slice(0, -path.extname(h).length)}.cpp`;
-	if (!(await exists(s))) {
-		const rel = path.relative(root, h);
-		const parts = rel.split(path.sep);
-		const idx = parts.indexOf("include");
-		if (idx !== -1) {
-			parts[idx] = "source";
-			s = `${path.join(root, ...parts)}.cpp`;
-		}
-	}
-	if (!(await exists(s))) {
-		const base = `${path.basename(h, path.extname(h))}.cpp`;
-		const all = await walk(root);
-		const match = all.find((p) => path.basename(p) === base);
-		if (match) {
-			s = match;
-		}
-	}
-	if (!(await exists(s))) {
-		return;
+	const s = findFileBackward(root, `${path.basename(h, path.extname(h))}.cpp`);
+	if (!s) {
+		console.warn(`matching source file not found: ${h}`);
 	}
 	const relH = path.relative(root, h);
 	const oh = path.join(outRoot, relH);
@ -390,15 +459,17 @@ const handle = async (
 	if (!(await exists(oh))) {
 		const skeleton = await getSkeleton(
 			await fs.readFile(h, "utf8"),
-			await fs.readFile(s, "utf8"),
+			s ? await fs.readFile(s, "utf8") : null,
 		);
 		await fs.writeFile(oh, skeleton);
 	}
-	const relS = path.relative(root, s);
-	const os = path.join(outRoot, relS);
-	await ensureDir(path.dirname(os));
-	if (!(await exists(os))) {
-		await fs.copyFile(s, os);
+	if (s) {
+		const relS = path.relative(root, s);
+		const os = path.join(outRoot, relS);
+		await ensureDir(path.dirname(os));
+		if (!(await exists(os))) {
+			await fs.copyFile(s, os);
+		}
 	}
 	console.log(`processed ${n}/${total}`);
 };
@ -408,7 +479,7 @@ const main = async (): Promise<void> => {
 	const outRoot = path.join(root, "skeleton");
 	const headers = await walk(root);
 	const total = headers.length;
-	const limit = pLimit(1);
+	const limit = pLimit(20);
 	await Promise.all(
 		headers.map((h, i) => limit(() => handle(i + 1, h, total, root, outRoot))),
 	);
--- a/package-lock.json
+++ b/package-lock.json
@ -9,7 +9,7 @@
 			"version": "1.0.0",
 			"license": "ISC",
 			"dependencies": {
-				"@libsql/client": "^0.15.4",
+				"diff": "^7.0.0",
 				"dotenv": "^16.5.0",
 				"gpt-tokenizer": "^2.9.0",
 				"openai": "^4.97.0",
@ -20,156 +20,6 @@
 				"typescript": "^5.8.3"
 			}
 		},
-		"node_modules/@libsql/client": {
-			"version": "0.15.4",
-			"resolved": "https://registry.npmjs.org/@libsql/client/-/client-0.15.4.tgz",
-			"integrity": "sha512-m8a7giWlhLdfKVIZFd3UlBptWTS+H0toSOL09BxbqzBeFHwuVC+5ewyi4LMBxoy2TLNQGE4lO8cwpsTWmu695w==",
-			"license": "MIT",
-			"dependencies": {
-				"@libsql/core": "^0.15.4",
-				"@libsql/hrana-client": "^0.7.0",
-				"js-base64": "^3.7.5",
-				"libsql": "^0.5.6",
-				"promise-limit": "^2.7.0"
-			}
-		},
-		"node_modules/@libsql/core": {
-			"version": "0.15.4",
-			"resolved": "https://registry.npmjs.org/@libsql/core/-/core-0.15.4.tgz",
-			"integrity": "sha512-NMvh6xnn3vrcd7DNehj0HiJcRWB2a8hHhJUTkOBej3Pf3KB21HOmdOUjXxJ5pGbjWXh4ezQBmHtF5ozFhocXaA==",
-			"license": "MIT",
-			"dependencies": {
-				"js-base64": "^3.7.5"
-			}
-		},
-		"node_modules/@libsql/darwin-arm64": {
-			"version": "0.5.8",
-			"resolved": "https://registry.npmjs.org/@libsql/darwin-arm64/-/darwin-arm64-0.5.8.tgz",
-			"integrity": "sha512-dJRfwCHAKOIgysMbB+PBo3ZmCVuGC02fH57kFEFlqbbUv6wnAZV5g7GErQIv4IlC4VPKAS4RL20fjLUgXE+0Xg==",
-			"cpu": [
-				"arm64"
-			],
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"darwin"
-			]
-		},
-		"node_modules/@libsql/darwin-x64": {
-			"version": "0.5.8",
-			"resolved": "https://registry.npmjs.org/@libsql/darwin-x64/-/darwin-x64-0.5.8.tgz",
-			"integrity": "sha512-ua5ngqJy9o4lyjYDzF8c69YbOAwP2TQXPhDURs8l97b09HHFh5/8gWRNor7vYRpsziwp8TC77DdQ0C84+gP5tg==",
-			"cpu": [
-				"x64"
-			],
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"darwin"
-			]
-		},
-		"node_modules/@libsql/hrana-client": {
-			"version": "0.7.0",
-			"resolved": "https://registry.npmjs.org/@libsql/hrana-client/-/hrana-client-0.7.0.tgz",
-			"integrity": "sha512-OF8fFQSkbL7vJY9rfuegK1R7sPgQ6kFMkDamiEccNUvieQ+3urzfDFI616oPl8V7T9zRmnTkSjMOImYCAVRVuw==",
-			"license": "MIT",
-			"dependencies": {
-				"@libsql/isomorphic-fetch": "^0.3.1",
-				"@libsql/isomorphic-ws": "^0.1.5",
-				"js-base64": "^3.7.5",
-				"node-fetch": "^3.3.2"
-			}
-		},
-		"node_modules/@libsql/isomorphic-fetch": {
-			"version": "0.3.1",
-			"resolved": "https://registry.npmjs.org/@libsql/isomorphic-fetch/-/isomorphic-fetch-0.3.1.tgz",
-			"integrity": "sha512-6kK3SUK5Uu56zPq/Las620n5aS9xJq+jMBcNSOmjhNf/MUvdyji4vrMTqD7ptY7/4/CAVEAYDeotUz60LNQHtw==",
-			"license": "MIT",
-			"engines": {
-				"node": ">=18.0.0"
-			}
-		},
-		"node_modules/@libsql/isomorphic-ws": {
-			"version": "0.1.5",
-			"resolved": "https://registry.npmjs.org/@libsql/isomorphic-ws/-/isomorphic-ws-0.1.5.tgz",
-			"integrity": "sha512-DtLWIH29onUYR00i0GlQ3UdcTRC6EP4u9w/h9LxpUZJWRMARk6dQwZ6Jkd+QdwVpuAOrdxt18v0K2uIYR3fwFg==",
-			"license": "MIT",
-			"dependencies": {
-				"@types/ws": "^8.5.4",
-				"ws": "^8.13.0"
-			}
-		},
-		"node_modules/@libsql/linux-arm64-gnu": {
-			"version": "0.5.8",
-			"resolved": "https://registry.npmjs.org/@libsql/linux-arm64-gnu/-/linux-arm64-gnu-0.5.8.tgz",
-			"integrity": "sha512-6HHZlPbMu+cmCJafg/dwOcWFMu07hTB5teMKU5ke66kqeWLRBnOs5/DnZGVz6q0k+Z4L4UTRbdrnCklR3GmvFg==",
-			"cpu": [
-				"arm64"
-			],
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"linux"
-			]
-		},
-		"node_modules/@libsql/linux-arm64-musl": {
-			"version": "0.5.8",
-			"resolved": "https://registry.npmjs.org/@libsql/linux-arm64-musl/-/linux-arm64-musl-0.5.8.tgz",
-			"integrity": "sha512-QGhZadKk3gvrDHa63U7xQrsqET/43E6L7/15oh7I+SINl8meoZAJNTJNYfOUmPM2lPPfNDgr46v4p5ggo6su0A==",
-			"cpu": [
-				"arm64"
-			],
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"linux"
-			]
-		},
-		"node_modules/@libsql/linux-x64-gnu": {
-			"version": "0.5.8",
-			"resolved": "https://registry.npmjs.org/@libsql/linux-x64-gnu/-/linux-x64-gnu-0.5.8.tgz",
-			"integrity": "sha512-HUWxOvLE5W287O/vaHWFpZMqaaebEBZvcUqJJ/E+IcC9kmKc6GqDW+fJkfPfosrpGyPNbYDj0w9pIcck0l/oeA==",
-			"cpu": [
-				"x64"
-			],
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"linux"
-			]
-		},
-		"node_modules/@libsql/linux-x64-musl": {
-			"version": "0.5.8",
-			"resolved": "https://registry.npmjs.org/@libsql/linux-x64-musl/-/linux-x64-musl-0.5.8.tgz",
-			"integrity": "sha512-hfhkPwqzFroU01xUB7ZXFw3bP+jqcGolGLyhEkeh/Rsoune0ucm1KPrU2tqTBqQP4a7lL0nSL1A37nfjIO61Hw==",
-			"cpu": [
-				"x64"
-			],
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"linux"
-			]
-		},
-		"node_modules/@libsql/win32-x64-msvc": {
-			"version": "0.5.8",
-			"resolved": "https://registry.npmjs.org/@libsql/win32-x64-msvc/-/win32-x64-msvc-0.5.8.tgz",
-			"integrity": "sha512-8hKczus0swLEvXu8N0znWdyFo5QzFnE9mnz7G/sb+eVn+zpPlT6ZdFHZdhQzev9C0to7kvYDj03qESTUIwDiqg==",
-			"cpu": [
-				"x64"
-			],
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"win32"
-			]
-		},
-		"node_modules/@neon-rs/load": {
-			"version": "0.0.4",
-			"resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.4.tgz",
-			"integrity": "sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw==",
-			"license": "MIT"
-		},
 		"node_modules/@types/node": {
 			"version": "22.15.16",
 			"resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.16.tgz",
@ -189,15 +39,6 @@
 				"form-data": "^4.0.0"
 			}
 		},
-		"node_modules/@types/ws": {
-			"version": "8.18.1",
-			"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz",
-			"integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==",
-			"license": "MIT",
-			"dependencies": {
-				"@types/node": "*"
-			}
-		},
 		"node_modules/abort-controller": {
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
@ -253,15 +94,6 @@
 				"node": ">= 0.8"
 			}
 		},
-		"node_modules/data-uri-to-buffer": {
-			"version": "4.0.1",
-			"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
-			"integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
-			"license": "MIT",
-			"engines": {
-				"node": ">= 12"
-			}
-		},
 		"node_modules/delayed-stream": {
 			"version": "1.0.0",
 			"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
@ -271,13 +103,13 @@
 				"node": ">=0.4.0"
 			}
 		},
-		"node_modules/detect-libc": {
-			"version": "2.0.2",
-			"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz",
-			"integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==",
-			"license": "Apache-2.0",
+		"node_modules/diff": {
+			"version": "7.0.0",
+			"resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz",
+			"integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==",
+			"license": "BSD-3-Clause",
 			"engines": {
-				"node": ">=8"
+				"node": ">=0.3.1"
 			}
 		},
 		"node_modules/dotenv": {
@ -360,29 +192,6 @@
 				"node": ">=6"
 			}
 		},
-		"node_modules/fetch-blob": {
-			"version": "3.2.0",
-			"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
-			"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
-			"funding": [
-				{
-					"type": "github",
-					"url": "https://github.com/sponsors/jimmywarting"
-				},
-				{
-					"type": "paypal",
-					"url": "https://paypal.me/jimmywarting"
-				}
-			],
-			"license": "MIT",
-			"dependencies": {
-				"node-domexception": "^1.0.0",
-				"web-streams-polyfill": "^3.0.3"
-			},
-			"engines": {
-				"node": "^12.20 || >= 14.13"
-			}
-		},
 		"node_modules/form-data": {
 			"version": "4.0.2",
 			"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.2.tgz",
@ -426,18 +235,6 @@
 				"node": ">= 14"
 			}
 		},
-		"node_modules/formdata-polyfill": {
-			"version": "4.0.10",
-			"resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
-			"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
-			"license": "MIT",
-			"dependencies": {
-				"fetch-blob": "^3.1.2"
-			},
-			"engines": {
-				"node": ">=12.20.0"
-			}
-		},
 		"node_modules/function-bind": {
 			"version": "1.1.2",
 			"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
@ -550,41 +347,6 @@
 				"ms": "^2.0.0"
 			}
 		},
-		"node_modules/js-base64": {
-			"version": "3.7.7",
-			"resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.7.tgz",
-			"integrity": "sha512-7rCnleh0z2CkXhH67J8K1Ytz0b2Y+yxTPL+/KOJoa20hfnVQ/3/T6W/KflYI4bRHRagNeXeU2bkNGI3v1oS/lw==",
-			"license": "BSD-3-Clause"
-		},
-		"node_modules/libsql": {
-			"version": "0.5.8",
-			"resolved": "https://registry.npmjs.org/libsql/-/libsql-0.5.8.tgz",
-			"integrity": "sha512-+OopMI1wM/NvAJTHf3O3+beHd1YfKLnSVsOGBl3/7UBDZ4ydVadkbBk5Hjjs9d3ALC5rBaftMY59AvwyC8MzPw==",
-			"cpu": [
-				"x64",
-				"arm64",
-				"wasm32"
-			],
-			"license": "MIT",
-			"os": [
-				"darwin",
-				"linux",
-				"win32"
-			],
-			"dependencies": {
-				"@neon-rs/load": "^0.0.4",
-				"detect-libc": "2.0.2"
-			},
-			"optionalDependencies": {
-				"@libsql/darwin-arm64": "0.5.8",
-				"@libsql/darwin-x64": "0.5.8",
-				"@libsql/linux-arm64-gnu": "0.5.8",
-				"@libsql/linux-arm64-musl": "0.5.8",
-				"@libsql/linux-x64-gnu": "0.5.8",
-				"@libsql/linux-x64-musl": "0.5.8",
-				"@libsql/win32-x64-msvc": "0.5.8"
-			}
-		},
 		"node_modules/math-intrinsics": {
 			"version": "1.1.0",
 			"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
@ -641,24 +403,6 @@
 				"node": ">=10.5.0"
 			}
 		},
-		"node_modules/node-fetch": {
-			"version": "3.3.2",
-			"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
-			"integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
-			"license": "MIT",
-			"dependencies": {
-				"data-uri-to-buffer": "^4.0.0",
-				"fetch-blob": "^3.1.4",
-				"formdata-polyfill": "^4.0.10"
-			},
-			"engines": {
-				"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
-			},
-			"funding": {
-				"type": "opencollective",
-				"url": "https://opencollective.com/node-fetch"
-			}
-		},
 		"node_modules/openai": {
 			"version": "4.97.0",
 			"resolved": "https://registry.npmjs.org/openai/-/openai-4.97.0.tgz",
@ -739,12 +483,6 @@
 				"url": "https://github.com/sponsors/sindresorhus"
 			}
 		},
-		"node_modules/promise-limit": {
-			"version": "2.7.0",
-			"resolved": "https://registry.npmjs.org/promise-limit/-/promise-limit-2.7.0.tgz",
-			"integrity": "sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw==",
-			"license": "ISC"
-		},
 		"node_modules/tr46": {
 			"version": "0.0.3",
 			"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
@ -771,15 +509,6 @@
 			"integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
 			"license": "MIT"
 		},
-		"node_modules/web-streams-polyfill": {
-			"version": "3.3.3",
-			"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz",
-			"integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==",
-			"license": "MIT",
-			"engines": {
-				"node": ">= 8"
-			}
-		},
 		"node_modules/webidl-conversions": {
 			"version": "3.0.1",
 			"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
@ -801,6 +530,8 @@
 			"resolved": "https://registry.npmjs.org/ws/-/ws-8.18.2.tgz",
 			"integrity": "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==",
 			"license": "MIT",
+			"optional": true,
+			"peer": true,
 			"engines": {
 				"node": ">=10.0.0"
 			},
--- a/package.json
+++ b/package.json
@ -15,7 +15,7 @@
 		"typescript": "^5.8.3"
 	},
 	"dependencies": {
-		"@libsql/client": "^0.15.4",
+		"diff": "^7.0.0",
 		"dotenv": "^16.5.0",
 		"gpt-tokenizer": "^2.9.0",
 		"openai": "^4.97.0",