From 10cad10ac96e24097be7a151cbcae3cf247e0113 Mon Sep 17 00:00:00 2001 From: Florian Kaiser Date: Thu, 8 May 2025 21:11:06 +0200 Subject: [PATCH] Add code embedder scripts and database setup; update .gitignore --- .gitignore | 5 +- code_embedder.py | 221 ++++++++++ code_embedder.ts | 104 +++++ get_skeleton.py | 379 ++++++++++++++++++ package-lock.json | 995 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 23 ++ 6 files changed, 1726 insertions(+), 1 deletion(-) create mode 100644 code_embedder.py create mode 100644 code_embedder.ts create mode 100644 get_skeleton.py create mode 100644 package-lock.json create mode 100644 package.json diff --git a/.gitignore b/.gitignore index fe5185b8..a111d00e 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,7 @@ ISLE.EXE LEGO1.DLL all_code.txt ask.py -into-one.sh \ No newline at end of file +into-one.sh +skeleton/ +.DS_Store +node_modules \ No newline at end of file diff --git a/code_embedder.py b/code_embedder.py new file mode 100644 index 00000000..fff503f6 --- /dev/null +++ b/code_embedder.py @@ -0,0 +1,221 @@ +import hashlib +import json +import os +import sqlite3 + +from dotenv import load_dotenv +from openai import OpenAI + +# Configuration +DATABASE_NAME = "code_embeddings.db" +# Define relevant file extensions (add more as needed) +SOURCE_FILE_EXTENSIONS = [ + ".c", + ".cpp", + ".h", + ".hpp", + ".py", + ".js", + ".ts", + ".java", + ".go", + ".rs", + ".swift", + ".kt", + ".m", + ".mm", + ".cs", + ".rb", + ".php", + ".pl", + ".sh", + ".lua", + ".sql", +] # Add header extensions too +ENV_FILE_PATH = ".env" +OPENAI_MODEL = "text-embedding-3-large" + + +def load_api_key(env_path: str) -> str: + """Loads the OpenAI API key from the .env file.""" + load_dotenv(dotenv_path=env_path) + api_key = os.getenv("OPENAI_API_KEY") + if not api_key: + raise ValueError( + "OPENAI_API_KEY not found in .env file or environment variables." + ) + return api_key + + +def get_file_checksum(file_path: str) -> str: + """Calculates the SHA256 checksum of a file.""" + sha256_hash = hashlib.sha256() + try: + with open(file_path, "rb") as f: + for byte_block in iter(lambda: f.read(4096), b""): + sha256_hash.update(byte_block) + return sha256_hash.hexdigest() + except FileNotFoundError: + print(f"Error: File not found at {file_path}") + return "" + except IOError: + print(f"Error: Could not read file at {file_path}") + return "" + + +def get_openai_embedding(text: str, client: OpenAI) -> list[float] | None: + """Gets the embedding for the given text using OpenAI API.""" + if not text.strip(): # Avoid sending empty strings to OpenAI + print("Warning: Empty content, skipping embedding.") + return None + try: + response = client.embeddings.create(input=text, model=OPENAI_MODEL) + return response.data[0].embedding + except Exception as e: + print(f"Error getting embedding from OpenAI: {e}") + return None + + +def find_files(start_path: str, extensions: list[str]) -> list[str]: + """Recursively finds all files with given extensions in the start_path.""" + found_files = [] + for root, _, files in os.walk(start_path): + for file in files: + if any(file.endswith(ext) for ext in extensions): + found_files.append(os.path.join(root, file)) + return found_files + + +def init_db(db_name: str) -> sqlite3.Connection: + """Initializes the SQLite database and creates the table if it doesn't exist.""" + conn = sqlite3.connect(db_name) + cursor = conn.cursor() + # Embedding dimension for text-embedding-3-large is 3072 + # Storing embeddings as TEXT (JSON string) or BLOB. TEXT is easier for inspection. + cursor.execute(""" + CREATE TABLE IF NOT EXISTS file_embeddings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_path TEXT UNIQUE NOT NULL, + checksum TEXT NOT NULL, + embedding TEXT NOT NULL, -- Store as JSON string + last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_file_path ON file_embeddings (file_path); + """) + conn.commit() + return conn + + +def process_file(file_path: str, client: OpenAI, conn: sqlite3.Connection): + """Processes a single file: calculates checksum, gets embedding, and stores in DB.""" + print(f"Processing {file_path}...") + try: + with open(file_path, "r", encoding="utf-8", errors="ignore") as f: + content = f.read() + except Exception as e: + print(f"Error reading file {file_path}: {e}") + return + + current_checksum = get_file_checksum(file_path) + if not current_checksum: # Error occurred in checksum calculation + return + + cursor = conn.cursor() + cursor.execute( + "SELECT checksum FROM file_embeddings WHERE file_path = ?", (file_path,) + ) + result = cursor.fetchone() + + if result and result[0] == current_checksum: + print(f"File {file_path} is unchanged. Skipping.") + return + + embedding = get_openai_embedding(content, client) + if not embedding: + print(f"Could not get embedding for {file_path}. Skipping.") + return + + embedding_json = json.dumps(embedding) # Convert list to JSON string + + if result: # File exists but checksum differs, so update + print(f"File {file_path} has changed. Updating embedding.") + cursor.execute( + """ + UPDATE file_embeddings + SET checksum = ?, embedding = ?, last_updated = CURRENT_TIMESTAMP + WHERE file_path = ? + """, + (current_checksum, embedding_json, file_path), + ) + else: # New file, insert + print(f"New file {file_path}. Adding embedding.") + cursor.execute( + """ + INSERT INTO file_embeddings (file_path, checksum, embedding) + VALUES (?, ?, ?) + """, + (file_path, current_checksum, embedding_json), + ) + conn.commit() + print(f"Successfully processed and stored embedding for {file_path}.") + + +def main(): + """Main function to orchestrate the embedding process.""" + # 0. Create .env if it doesn't exist + if not os.path.exists(ENV_FILE_PATH): + with open(ENV_FILE_PATH, "w") as f: + f.write("OPENAI_API_KEY='YOUR_API_KEY_HERE'\\n") + print(f"Created {ENV_FILE_PATH}. Please add your OpenAI API key to it.") + return + + # 1. Load API Key + try: + api_key = load_api_key(ENV_FILE_PATH) + except ValueError as e: + print(e) + return + + # 2. Initialize OpenAI client + try: + client = OpenAI(api_key=api_key) + except Exception as e: + print(f"Failed to initialize OpenAI client: {e}") + return + + # 3. Initialize DB + conn = init_db(DATABASE_NAME) + + # 4. Get target directory + target_directory = input("Enter the root directory of your codebase to scan: ") + if not os.path.isdir(target_directory): + print(f"Error: Directory '{target_directory}' not found.") + conn.close() + return + + print( + f"Scanning for files in {target_directory} with extensions: {', '.join(SOURCE_FILE_EXTENSIONS)}" + ) + + # 5. Find files + files_to_process = find_files(target_directory, SOURCE_FILE_EXTENSIONS) + if not files_to_process: + print("No relevant files found to process.") + conn.close() + return + + print(f"Found {len(files_to_process)} files to process.") + + # 6. Process each file + for n, file_path in enumerate(files_to_process): + print(f"--- Processing file {n+1}/{len(files_to_process)} ---") + process_file(file_path, client, conn) + + conn.close() + print("\nEmbedding process completed.") + + +if __name__ == "__main__": + main() diff --git a/code_embedder.ts b/code_embedder.ts new file mode 100644 index 00000000..d56e13ba --- /dev/null +++ b/code_embedder.ts @@ -0,0 +1,104 @@ +import { config } from "dotenv"; +import { createHash } from "node:crypto"; +import { readFileSync, readdirSync, statSync } from "node:fs"; +import path from "node:path"; +import { z } from "zod"; +import { OpenAI } from "openai"; +import { createClient } from "@libsql/client"; + +// configuration +const DB_PATH = "file:code_embeddings.db"; +const SOURCE_FILE_EXTENSIONS = [".c", ".cpp", ".h", ".hpp", ".hxx"]; +const OPENAI_MODEL = "text-embedding-3-large"; +const EMBEDDING_DIMENSION = 3072; + +// env +config(); +const EnvSchema = z.object({ OPENAI_API_KEY: z.string() }); +const { OPENAI_API_KEY } = EnvSchema.parse(process.env); + +// clients +const openai = new OpenAI({ apiKey: OPENAI_API_KEY }); +const db = createClient({ url: DB_PATH }); + +await db.execute( + `CREATE TABLE IF NOT EXISTS file_embeddings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_path TEXT UNIQUE NOT NULL, + checksum TEXT NOT NULL, + embedding F32_BLOB(${EMBEDDING_DIMENSION}) NOT NULL, + last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP + )`, +); + +const getChecksum = (filePath: string): string => { + const hash = createHash("sha256"); + hash.update(readFileSync(filePath)); + return hash.digest("hex"); +}; + +const getEmbedding = async (content: string): Promise => { + const { data } = await openai.embeddings.create({ + model: OPENAI_MODEL, + input: content, + encoding_format: "float", + }); + return data[0].embedding as unknown as number[]; +}; + +const findFiles = (startDir: string): string[] => { + const stack = [startDir]; + const files: string[] = []; + while (stack.length) { + const current = stack.pop() as string; + for (const entry of readdirSync(current)) { + const fullPath = path.join(current, entry); + const stats = statSync(fullPath); + if (stats.isDirectory()) { + stack.push(fullPath); + } else if (SOURCE_FILE_EXTENSIONS.some((ext) => entry.endsWith(ext))) { + files.push(fullPath); + } + } + } + return files; +}; + +const processFile = async (filePath: string) => { + const content = readFileSync(filePath, "utf8"); + if (!content.trim()) { + console.log(`Skipping ${filePath} because it is empty`); + return; + } + + const checksum = getChecksum(filePath); + const existing = await db.execute( + "SELECT checksum FROM file_embeddings WHERE file_path = ?", + [filePath], + ); + const row = existing.rows[0]; + if (row && row.checksum === checksum) { + console.log(`Skipping ${filePath} because it already exists`); + return; + } + + const embedding = await getEmbedding(content); + + await db.execute( + "INSERT INTO file_embeddings (file_path, checksum, embedding) VALUES (?, ?, ?)", + [filePath, checksum, embedding], + ); +}; + +const targetDirectory = process.argv[2] ?? "."; +if (!statSync(targetDirectory).isDirectory()) { + console.error(`Directory '${targetDirectory}' not found`); + process.exit(1); +} +const files = findFiles(targetDirectory); +for (let n = 0; n < files.length; ++n) { + console.log(`Processing ${files[n]} (${n + 1}/${files.length})`); + await processFile(files[n]); +} +console.log("Done"); +process.exit(0); diff --git a/get_skeleton.py b/get_skeleton.py new file mode 100644 index 00000000..2987d10e --- /dev/null +++ b/get_skeleton.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 +import os +import shutil +from pathlib import Path + +import dotenv +from openai import OpenAI + +dotenv.load_dotenv() + +client = OpenAI(api_key=os.environ["OPENAI_API_KEY"]) + + +def get_skeleton(header: str, source: str) -> str: + response = client.responses.create( + model="gpt-4.1", + instructions="""You will receive the code of a header and source file. Add Doxygen annotations to it. + +Use the documentation and source files to understand what every class, member, method and function does. Add detailed explanations for everything in the annotations. + +- Prefix everything you add with "[AI]" so that we can validate it later +- If there are already comment or annotations, add them to your annotations. +- Ignore the cpp and reference files. They only there for you to give you some context for the annotations. +- Don't add generic annotations that are not helpful. For example, if the function is getPosition, don't add an annotation like "Gets the position" +- If you encounter a class/method or member that is unknown (e.g. m_unk0x10, FUN_10001510 etc.) and you know what it is or does, add a "[AI_SUGGESTED_NAME: ]" in the annotations to it. +- Only return the annotated header file, nothing else. Do NOT return the source file. Do NOT put it into a ``` markdown code block. + +I will now give you a rough documentation of the codebase to use as a reference: + +# Documentation + +This codebase is a decompilation of the 1997 game LEGO Island and implements the game's rendering, user input, game logic, and resource management. It uses a combination of custom C++ classes and the Microsoft DirectX 5 API for 3D graphics and sound. + +## Terminology and Core Concepts: +* **Script**: SI File (it appears they wanted to use SI much more as actual scripts, but ended up hard-coding almost everything) +* **ROI (Real-time Object Instance):** The base class for any object that can be placed in the 3D world. It handles visibility, world bounding volumes (bounding box and sphere), intrinsic importance (for LOD), and contains a list of LOD objects. +* **Mx (Mindscape):** Prefix for general utility classes provided by Mindscape, the game developer, like `MxString`, `MxMatrix`, `MxList`, etc. +* **Omni:** Name of the engine. + +## Basic Architecture: + +The code follows a component-based architecture, with separate modules for rendering, input, game logic, and resource management. The `LegoOmni` class acts as the central hub, coordinating the different subsystems. + +The code uses a combination of event-driven programming (for input and notifications) and a tick-based update loop (for game logic and animation). It makes heavy use of the retained mode features of Direct3DRM for scene management. + +## Core Concepts + +### Tickling + +MxCore objects can be registered to be tickled by the MxTickleManager. This is used to update them in a set interval. This is used for loading the next frame of a video, update 3D sound positions and more. + +## Classes + +### MxAtom +Key (string) / Value (U16) pair. + +Inc() +Dec() + +### MxAtomId +String with lookup method (upper case/lower case/exact). Used for IDs. + +MxOmni holds an AtomSet that contains MxAtoms for every MxAtomId created + a counter of how many instances exists (purpose unclear). + +### MxString +Typical string class with utility functions. + +### MxCore +Virtual base class. + +Tickle() +Notify(MyParam) +GetId() +ClassName() + +IsA() +Checks ALL parents. + +### MxTickleManager : MxCore +Holds a list of MxTickleClient*. Goes though them on Tickle() and calls Tickle() if interval time has passed. + +### MxTickleClient +Holds a MxCore*, Interval, LastUpdateTime and Flags (only used for TICKLE_MANAGER_FLAG_DESTROY?). + +### IsleApp +Main class and the entry point of the game. + +### MxOmni + +#### Start(MxDSAction*) + + +### MxDSObject : MxCore +Base Object for extracted objects from SI files. + +Adds AtomId, ObjectId, Type, Flags, ObjectName and SourceName as well as a Deserialize method. + +Deserializes SourceName and ObjectName. It also saves the flags provided as a param. + +Also provides DeserializeDSObjectDispatch which deserializes an MxOb chunk into the corresponding MxDSObject child. + +### MxDSAction : MxDSObject +Deserializes Flags (ignores param), StartTime, Duration, LoopCount, Location (Vec3), Direction (Vec3) and Up (Vec3). + +Also if extra is available it appends it to ExtraData. + +### MxDSMediaAction : MxDSAction +Deserializes MediaSrcPath, two U32 into an unknown struct, FramesPerSecond, MediaFormat, PaletteManagement, SustainTime. + +### MxDSMultiAction : MxDSAction +Deserializes multiple chunks? into MxDSActions using DeserializeDSObjectDispatch + +### MxDSParallelAction : MxDSMultiAction +Just a rename from MxDSMultiAction. + +### MxDSSelectAction : MxDSParallelAction +Deserializes Unknown0x9c, checks if it starts with "RANDOM_" and if so, parses the number after "RANDOM_", gets a number from 0 to number. If not reads a string and saves it in the VariableTable. + +Then reads a list of strings (presumably numbers) into a list. Then reads the same number of chunks into objects (Actions), chooses nth one where N is the index of the string that equals to the random number. + +### MxDSSound : MxDSMediaAction +Deserializes a volume. + +### MxDSObjectAction : MxDSMediaAction +Adds nothing. + +### MxVariableTable : MxHashTable +MxOmni holds a VariableTable that is just a key/value store string/string. + +### MxPresenter : MxCore +Abstract base class for all presenters. Separates the tickle down to ReadyTickle(), StartingTickle(), StreamingTickle(), RepeatingTickle(), FreezingTickle() and DoneTickle() + +Similar to DeserializeDSObjectDispatch, there is a PresenterNameDispatch() that reads the media format (" FLC", " SMK", " MID", " WAV") and returns the corresponding HandlerClassName(). + +### MxMediaPresenter : MxPresenter +Hold a MxDSSubscriber* and reads data from it on the tickles. + +### LegoBuildingManager : MxCore + +#### CreateBuilding() + +### MxEntity : MxCore +Adds EntityId (S32) and AtomID (MxAtomId). + +### LegoEntity : MxEntity +Adds WorldDirection, WorldUp, WorldPostion, WorldSpeed, ROI, CameraFlag, Flags, + +Virtual methods: +ClickSound(bool) +ClickAnimation() +SwitchVariant() +SwitchSound() +SwitchMove() +SwitchColor(LegoROI*) +SwitchMood() + +#### ParseAction(char*) +Parses a string like ""ACTION:; ; " + +If action is not 7 (e_exit), it stores the filename into m_siFile, and if the action is not 6 (e_run) it stores the ID into m_targetEntityId. + +### ROI +Has LODlist (a list of LODObject), Visibility and ROIList (a list of ROI-pointers, via CompoundObject). + +Provides (deleted) functions for world velocity, bounding box and bounding sphere. + +### OrientableROI : ROI +Adds Local2World 4x4-matrix, WorldBoundingBox (and WorldBoundingSphere), WorldVelocity, ParentROI (another OrientableROI). +Also has an unknown bounding box and u32. The u32 can be enabled/disabled which either sets bit 1 and 3 or clears only bit 1. + +WorldUp, WorldDirection and WorldPosition are within `local2world`'s second, third and forth row. + +### ViewROI : OrientableROI +Adds Geometry saved withing a Tgl::Group and Unknown int. + +Uses the lod list with its own type ViewLODList (ref counted). + +### LegoROI : ViewROI +Adds Name, Entity, BoundingSphere. Provides functions to color/texture every lod (also based on global handlers). + +### LegoWorld : LegoEntity + +#### PlaceActor(...) +Goes through all controllers in m_controllerList and calls PlaceActor(). + +### LegoActor : LegoEntity +Adds Controller (LegoPathController*), Boundary, CollideBox, LastTime, ActorTime +Has UserNavFlag which presumably defines if user controls this character. Also has an ActorState: + +UpdatePlane(LegoNamedPlane&) +PlaceActor(LegoNamedPlane&) + +#### ParseAction(char*) +Parses a string like ""ATTACH_CAMERA: location direction up", "SPEED speed", "SOUND sound" , "MUTE" and "VISIBILITY". Saves it into the corresponding member. + +### LegoPathController : MxCore +Has PathBoundary (LegoPathBoundary*) a set of actors and many other things. Presumably it controls the movement of actors along paths. + +#### PlaceActor(LegoPathActor*) +Removes actor from current controller, and set it to this. + +#### PlaceActor(LegoPathActor*, LegoAnimPresenter*, ...) +Removes actor from current controller, does through all boundaries, goes through all presenters of them + + +### LegoPathActor : LegoActor + +### MxStreamer : MxCore +MxMisc holds a MxStreamer singleton. Also holds a list of MxStreamController. + +#### Open(const char*, MxU16 p_lookupType) +Creates and calls Open() on a MxDiskStreamController or MxRAMStreamController depending on lookupType if not already exists. + +### MxDSSubscriber : MxCore + +#### Create(MxStreamController* p_controller, MxU32 p_objectId, MxS16 p_unk0x48) +Calls MxStreamController::AddSubscriber() and sets some properties on itself. + +### MxStreamController : MxCore +Holds a list of subscriber. + +#### AddSubscriber(MxDSSubscriber*) +Puts it into the subscriber list. + +#### Open(const char* p_filename) +Removes ":" and ".SI" from filename and stores it in m_atom. + +### MxRAMStreamController : MxStreamController +Holds an MxDSBuffer. + +### MxDSBuffer : MxCore + +### MxStreamProvider : MxCore +Abstract base class. Holds an MxDSFile. + +### MxRAMStreamProvider : MxStreamProvider + +#### SetResourceToGet(MxStreamController*) +Gets the stream controllers Atom, adds ".SI". Tries to load it first from HDD and then from disk. Sets BufferSize to MxDSFile.BufferSize. Then reads the entire file into m_pContentsOfFile. + +#### MxU32 ReadData(MxU8* p_buffer, MxU32 p_size) +Return total size of MxOb. Rearranged p_buffer so that split chunks are merged. + +### MxDSStreamingAction : MxDSAction +Mostly unknown. + +### MxDiskStreamProvider : MxStreamProvider +Holds a list of MxDSStreamingAction. + +#### SetResourceToGet(MxStreamController*) +Gets the stream controllers Atom, adds ".SI". Tries to load it first from HDD and then from disk. Then starts a MxDiskStreamProviderThread with target this. + +#### MxDiskStreamProvider::WaitForWorkToComplete() +Called by the thread. Run indefinitely until object is destroyed. Streams data, code mostly unknown. + +### MxThread +Abstract base class for threads. Starts and manages one. Has abstract Run() method. + +### MxDiskStreamProviderThread : MxThread +Calls MxDiskStreamProvider::WaitForWorkToComplete. + +### MxDSChunk : MxCore +Holds Flags, ObjectId, Time, Data (U8*) and Length. Also some static utility functions. + +### MxDSSource : MxCore +Holds a buffer, length and position and offers abstract function to read and write. + +### MxDSFile : MxDSSource +Presumably this represents an SI file. Holds a MXIOINFO and on Open() opens m_filename and starts reading the starting chunks ("OMNI" etc.) also checks SI version (2.2). Then it reads the length of the MxOf chunk and puts it into m_pBuffer from parent class. + +Also holds the header chunk as ChunkHeader. GetBufferSize() returns the buffer size from the header. + +### LegoEdge +Has FaceA (LegoWEEdge*), FaceB (LegoWEEdge*), PointA (Vector3), PointB (Vector3). Also utility functions like CWVertex (LegoWEEdge&), CCWVertex (LegoWEEdge&), GetClockwiseEdge(LegoWEEdge&) and GetCounterclockwiseEdge(LegoWEEdge&). + +### LegoUnknown100db7f4 : LegoEdge +Adds Flags, a Mx3DPointFloat and a float and some utility functions like DistanceToMidpoint. + +### LegoWEEdge +Has Edges (LegoUnknown100db7f4*) + +### LegoWEGEdge : LegoWEEdge +Adds EdgeNormal, Flags and other lots of other stuff. + +### LegoPathBoundary : LegoWEGEdge +Adds actors and presenters. + +### LegoNamedPlane +Has Name (char*), Position, Direction and Up. Can be serialized. + +### LegoStorage +Abstract base class for a file-file object with positioning, reading/writing basic data types etc. + +### LegoMemory : LegoStorage +LegoStorage operating on a U8 pointer. + +### LegoFile : LegoStorage +LegoStorage operating on a File. + +### Mx3DPointFloat : Vector3 +Just a Vector3, doesn't add much. + +## Global Functions + +### KeyValueStringParse(char* p_output, const char* p_command, const char* p_string) +The function KeyValueStringParse searches a text (p_string) for a keyword (p_command). +If it finds the keyword, it copies the value immediately after that keyword into p_output. +It returns TRUE if it found the keyword, otherwise FALSE. + +Example: +p_string = "foo:123, bar:456, baz:789" +p_command = "bar" + +Result: +p_output = "456" +Return value: TRUE + +Return just the annotated header file, nothing else. Do NOT return the source file. Do NOT put it into a ``` markdown code block. + +Here are some relevant source and header files. They are just a reference for you to better understand the code: +""", + input=f"Header: {header}\nSource: {source}\n\nNow return just the annotated header file, nothing else. Do NOT return the source file. Do NOT put it into a ``` markdown code block.", + ) + return response.output_text + + +root = Path.cwd() +out_root = root / "skeleton" + +import asyncio +from concurrent.futures import ThreadPoolExecutor + +headers = [ + p + for p in root.rglob("*") + if p.is_file() + and p.suffix in {".h", ".hpp", ".hh", ".hxx"} + and "skeleton" not in str(p) +] +total = len(headers) + + +def handle(n, h): + s = h.with_suffix(".cpp") + if not s.exists(): + try: + r = h.relative_to(root) + parts = list(r.parts) + if "include" in parts: + parts[parts.index("include")] = "source" + s = root / Path(*parts).with_suffix(".cpp") + except ValueError: + pass + if not s.exists(): + m = list(root.rglob(f"{h.stem}.cpp")) + if m: + s = m[0] + if not s.exists(): + return + oh = out_root / h.relative_to(root) + oh.parent.mkdir(parents=True, exist_ok=True) + if not oh.exists(): + oh.write_text(get_skeleton(h.read_text(), s.read_text())) + os = out_root / s.relative_to(root) + os.parent.mkdir(parents=True, exist_ok=True) + if not os.exists(): + shutil.copy2(s, os) + print(f"processed {n}/{total}") + + +async def main(): + loop = asyncio.get_running_loop() + with ThreadPoolExecutor(max_workers=20) as ex: + await asyncio.gather( + *(loop.run_in_executor(ex, handle, n, h) for n, h in enumerate(headers, 1)) + ) + + +asyncio.run(main()) diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..3ee13261 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,995 @@ +{ + "name": "isle", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "isle", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@libsql/client": "^0.15.4", + "dotenv": "^16.5.0", + "openai": "^4.97.0", + "zod": "^3.24.4" + }, + "devDependencies": { + "ts-node": "^10.9.2", + "typescript": "^5.8.3" + } + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@libsql/client": { + "version": "0.15.4", + "resolved": "https://registry.npmjs.org/@libsql/client/-/client-0.15.4.tgz", + "integrity": "sha512-m8a7giWlhLdfKVIZFd3UlBptWTS+H0toSOL09BxbqzBeFHwuVC+5ewyi4LMBxoy2TLNQGE4lO8cwpsTWmu695w==", + "license": "MIT", + "dependencies": { + "@libsql/core": "^0.15.4", + "@libsql/hrana-client": "^0.7.0", + "js-base64": "^3.7.5", + "libsql": "^0.5.6", + "promise-limit": "^2.7.0" + } + }, + "node_modules/@libsql/core": { + "version": "0.15.4", + "resolved": "https://registry.npmjs.org/@libsql/core/-/core-0.15.4.tgz", + "integrity": "sha512-NMvh6xnn3vrcd7DNehj0HiJcRWB2a8hHhJUTkOBej3Pf3KB21HOmdOUjXxJ5pGbjWXh4ezQBmHtF5ozFhocXaA==", + "license": "MIT", + "dependencies": { + "js-base64": "^3.7.5" + } + }, + "node_modules/@libsql/darwin-arm64": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/@libsql/darwin-arm64/-/darwin-arm64-0.5.8.tgz", + "integrity": "sha512-dJRfwCHAKOIgysMbB+PBo3ZmCVuGC02fH57kFEFlqbbUv6wnAZV5g7GErQIv4IlC4VPKAS4RL20fjLUgXE+0Xg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@libsql/darwin-x64": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/@libsql/darwin-x64/-/darwin-x64-0.5.8.tgz", + "integrity": "sha512-ua5ngqJy9o4lyjYDzF8c69YbOAwP2TQXPhDURs8l97b09HHFh5/8gWRNor7vYRpsziwp8TC77DdQ0C84+gP5tg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@libsql/hrana-client": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@libsql/hrana-client/-/hrana-client-0.7.0.tgz", + "integrity": "sha512-OF8fFQSkbL7vJY9rfuegK1R7sPgQ6kFMkDamiEccNUvieQ+3urzfDFI616oPl8V7T9zRmnTkSjMOImYCAVRVuw==", + "license": "MIT", + "dependencies": { + "@libsql/isomorphic-fetch": "^0.3.1", + "@libsql/isomorphic-ws": "^0.1.5", + "js-base64": "^3.7.5", + "node-fetch": "^3.3.2" + } + }, + "node_modules/@libsql/isomorphic-fetch": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@libsql/isomorphic-fetch/-/isomorphic-fetch-0.3.1.tgz", + "integrity": "sha512-6kK3SUK5Uu56zPq/Las620n5aS9xJq+jMBcNSOmjhNf/MUvdyji4vrMTqD7ptY7/4/CAVEAYDeotUz60LNQHtw==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@libsql/isomorphic-ws": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/@libsql/isomorphic-ws/-/isomorphic-ws-0.1.5.tgz", + "integrity": "sha512-DtLWIH29onUYR00i0GlQ3UdcTRC6EP4u9w/h9LxpUZJWRMARk6dQwZ6Jkd+QdwVpuAOrdxt18v0K2uIYR3fwFg==", + "license": "MIT", + "dependencies": { + "@types/ws": "^8.5.4", + "ws": "^8.13.0" + } + }, + "node_modules/@libsql/linux-arm64-gnu": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/@libsql/linux-arm64-gnu/-/linux-arm64-gnu-0.5.8.tgz", + "integrity": "sha512-6HHZlPbMu+cmCJafg/dwOcWFMu07hTB5teMKU5ke66kqeWLRBnOs5/DnZGVz6q0k+Z4L4UTRbdrnCklR3GmvFg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@libsql/linux-arm64-musl": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/@libsql/linux-arm64-musl/-/linux-arm64-musl-0.5.8.tgz", + "integrity": "sha512-QGhZadKk3gvrDHa63U7xQrsqET/43E6L7/15oh7I+SINl8meoZAJNTJNYfOUmPM2lPPfNDgr46v4p5ggo6su0A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@libsql/linux-x64-gnu": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/@libsql/linux-x64-gnu/-/linux-x64-gnu-0.5.8.tgz", + "integrity": "sha512-HUWxOvLE5W287O/vaHWFpZMqaaebEBZvcUqJJ/E+IcC9kmKc6GqDW+fJkfPfosrpGyPNbYDj0w9pIcck0l/oeA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@libsql/linux-x64-musl": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/@libsql/linux-x64-musl/-/linux-x64-musl-0.5.8.tgz", + "integrity": "sha512-hfhkPwqzFroU01xUB7ZXFw3bP+jqcGolGLyhEkeh/Rsoune0ucm1KPrU2tqTBqQP4a7lL0nSL1A37nfjIO61Hw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@libsql/win32-x64-msvc": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/@libsql/win32-x64-msvc/-/win32-x64-msvc-0.5.8.tgz", + "integrity": "sha512-8hKczus0swLEvXu8N0znWdyFo5QzFnE9mnz7G/sb+eVn+zpPlT6ZdFHZdhQzev9C0to7kvYDj03qESTUIwDiqg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@neon-rs/load": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.4.tgz", + "integrity": "sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw==", + "license": "MIT" + }, + "node_modules/@tsconfig/node10": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", + "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.15.16", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.16.tgz", + "integrity": "sha512-3pr+KjwpVujqWqOKT8mNR+rd09FqhBLwg+5L/4t0cNYBzm/yEiYGCxWttjaPBsLtAo+WFNoXzGJfolM1JuRXoA==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/node-fetch": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" + } + }, + "node_modules/@types/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/acorn": { + "version": "8.14.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.1.tgz", + "integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", + "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/agentkeepalive": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", + "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", + "license": "MIT", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true, + "license": "MIT" + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/detect-libc": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", + "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/dotenv": { + "version": "16.5.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz", + "integrity": "sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/form-data": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.2.tgz", + "integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", + "license": "MIT" + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "license": "MIT", + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, + "node_modules/formdata-node/node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.0.0" + } + }, + "node_modules/js-base64": { + "version": "3.7.7", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.7.tgz", + "integrity": "sha512-7rCnleh0z2CkXhH67J8K1Ytz0b2Y+yxTPL+/KOJoa20hfnVQ/3/T6W/KflYI4bRHRagNeXeU2bkNGI3v1oS/lw==", + "license": "BSD-3-Clause" + }, + "node_modules/libsql": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/libsql/-/libsql-0.5.8.tgz", + "integrity": "sha512-+OopMI1wM/NvAJTHf3O3+beHd1YfKLnSVsOGBl3/7UBDZ4ydVadkbBk5Hjjs9d3ALC5rBaftMY59AvwyC8MzPw==", + "cpu": [ + "x64", + "arm64", + "wasm32" + ], + "license": "MIT", + "os": [ + "darwin", + "linux", + "win32" + ], + "dependencies": { + "@neon-rs/load": "^0.0.4", + "detect-libc": "2.0.2" + }, + "optionalDependencies": { + "@libsql/darwin-arm64": "0.5.8", + "@libsql/darwin-x64": "0.5.8", + "@libsql/linux-arm64-gnu": "0.5.8", + "@libsql/linux-arm64-musl": "0.5.8", + "@libsql/linux-x64-gnu": "0.5.8", + "@libsql/linux-x64-musl": "0.5.8", + "@libsql/win32-x64-msvc": "0.5.8" + } + }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true, + "license": "ISC" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/openai": { + "version": "4.97.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.97.0.tgz", + "integrity": "sha512-LRoiy0zvEf819ZUEJhgfV8PfsE8G5WpQi4AwA1uCV8SKvvtXQkoWUFkepD6plqyJQRghy2+AEPQ07FrJFKHZ9Q==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + }, + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/openai/node_modules/@types/node": { + "version": "18.19.99", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.99.tgz", + "integrity": "sha512-tNGqoGjjI4vY5jfm3lnqgR6yS8wyT76SfsWefLWRyh/cEK4UHmPVyqHZdafI/SNu1PQzfo2JLBWfG8eMmD7KrQ==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/openai/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/openai/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "license": "MIT" + }, + "node_modules/promise-limit": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/promise-limit/-/promise-limit-2.7.0.tgz", + "integrity": "sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw==", + "license": "ISC" + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "node_modules/typescript": { + "version": "5.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true, + "license": "MIT" + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/ws": { + "version": "8.18.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.2.tgz", + "integrity": "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/zod": { + "version": "3.24.4", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.4.tgz", + "integrity": "sha512-OdqJE9UDRPwWsrHjLN2F8bPxvwJBK22EHLWtanu0LSYr5YqzsaaW3RMgmjwr8Rypg5k+meEJdSPXJZXE/yqOMg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 00000000..7f3f18e2 --- /dev/null +++ b/package.json @@ -0,0 +1,23 @@ +{ + "name": "isle", + "version": "1.0.0", + "description": "[Development Vlog](https://www.youtube.com/playlist?list=PLbpl-gZkNl2COf_bB6cfgTapD5WduAfPz) | [Contributing](/CONTRIBUTING.md) | [Matrix](https://matrix.to/#/#isledecomp:matrix.org) | [Forums](https://forum.mattkc.com/viewforum.php?f=1) | [Patreon](https://www.patreon.com/mattkc)", + "main": "index.js", + "scripts": { + "start": "ts-node code_embedder.ts", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "ts-node": "^10.9.2", + "typescript": "^5.8.3" + }, + "dependencies": { + "@libsql/client": "^0.15.4", + "dotenv": "^16.5.0", + "openai": "^4.97.0", + "zod": "^3.24.4" + } +}