This commit is contained in:
33
docs/snippets/wasm/getting-started/batch-processing.ts
Normal file
33
docs/snippets/wasm/getting-started/batch-processing.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
import { extractBytes, initWasm } from "@kreuzberg/wasm";
|
||||
|
||||
interface DocumentJob {
|
||||
name: string;
|
||||
bytes: Uint8Array;
|
||||
mimeType: string;
|
||||
}
|
||||
|
||||
async function _processBatch(documents: DocumentJob[], concurrency: number = 3) {
|
||||
await initWasm();
|
||||
|
||||
const results: Record<string, string> = {};
|
||||
const queue = [...documents];
|
||||
|
||||
const workers = Array(concurrency)
|
||||
.fill(null)
|
||||
.map(async () => {
|
||||
while (queue.length > 0) {
|
||||
const doc = queue.shift();
|
||||
if (!doc) break;
|
||||
|
||||
try {
|
||||
const result = await extractBytes(doc.bytes, doc.mimeType);
|
||||
results[doc.name] = result.content;
|
||||
} catch (error) {
|
||||
console.error(`Failed to process ${doc.name}:`, error);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all(workers);
|
||||
return results;
|
||||
}
|
||||
Reference in New Issue
Block a user