Nomad changes
All checks were successful
Deploy fil (kreuzberg) / deploy (push) Successful in 49s

This commit is contained in:
Henrik Jess Nielsen
2026-06-01 23:40:55 +02:00
parent 72b1a0a6ed
commit b4c07d3693
5723 changed files with 1130655 additions and 0 deletions

View File

@@ -0,0 +1,33 @@
import { extractBytes, initWasm } from "@kreuzberg/wasm";
interface DocumentJob {
name: string;
bytes: Uint8Array;
mimeType: string;
}
async function _processBatch(documents: DocumentJob[], concurrency: number = 3) {
await initWasm();
const results: Record<string, string> = {};
const queue = [...documents];
const workers = Array(concurrency)
.fill(null)
.map(async () => {
while (queue.length > 0) {
const doc = queue.shift();
if (!doc) break;
try {
const result = await extractBytes(doc.bytes, doc.mimeType);
results[doc.name] = result.content;
} catch (error) {
console.error(`Failed to process ${doc.name}:`, error);
}
}
});
await Promise.all(workers);
return results;
}