hjess/fil
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Nomad changes
All checks were successful
Deploy fil (kreuzberg) / deploy (push) Successful in 49s
Details

Browse Source
This commit is contained in:
Henrik Jess Nielsen
2026-06-01 23:40:55 +02:00
parent 72b1a0a6ed
commit b4c07d3693
5723 changed files with 1130655 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

304
.ai-rulez/.generated-manifest.json Normal file
View File

@@ -0,0 +1,304 @@
{
"version": "1",
"files": [
".agents/agents/c-ffi-specialist.md",
".agents/agents/code-reviewer.md",
".agents/agents/csharp-specialist.md",
".agents/agents/dart-specialist.md",
".agents/agents/devops-engineer.md",
".agents/agents/docs-writer.md",
".agents/agents/e2e-generator-engineer.md",
".agents/agents/elixir-specialist.md",
".agents/agents/extraction-engineer.md",
".agents/agents/ffi-engineer.md",
".agents/agents/go-specialist.md",
".agents/agents/java-specialist.md",
".agents/agents/jni-specialist.md",
".agents/agents/kotlin-android-specialist.md",
".agents/agents/kreuzberg-developer.md",
".agents/agents/ocr-engineer.md",
".agents/agents/performance-engineer.md",
".agents/agents/php-specialist.md",
".agents/agents/plugin-engineer.md",
".agents/agents/polyglot-architect.md",
".agents/agents/python-specialist.md",
".agents/agents/r-specialist.md",
".agents/agents/release-engineer.md",
".agents/agents/ruby-specialist.md",
".agents/agents/rust-core-engineer.md",
".agents/agents/security-auditor.md",
".agents/agents/swift-specialist.md",
".agents/agents/test-writer.md",
".agents/agents/typescript-specialist.md",
".agents/agents/wasm-specialist.md",
".agents/agents/zig-specialist.md",
".agents/settings.json",
".agents/skills/add-language-generator/SKILL.md",
".agents/skills/alef/SKILL.md",
".agents/skills/alef/references/adapters.md",
".agents/skills/alef/references/backends.md",
".agents/skills/alef/references/cli-reference.md",
".agents/skills/alef/references/configuration.md",
".agents/skills/alef/references/designing-alef-toml.md",
".agents/skills/alef/references/e2e-testing.md",
".agents/skills/alef/references/troubleshooting.md",
".agents/skills/api-server-mcp/SKILL.md",
".agents/skills/chunking-embeddings/SKILL.md",
".agents/skills/common-task-commands/SKILL.md",
".agents/skills/create-e2e-fixture/SKILL.md",
".agents/skills/extraction-pipeline-patterns/SKILL.md",
".agents/skills/format-specific-extraction/SKILL.md",
".agents/skills/plugin-architecture-patterns/SKILL.md",
".agents/skills/quick-start/SKILL.md",
".claude/agents/c-ffi-specialist.md",
".claude/agents/code-reviewer.md",
".claude/agents/csharp-specialist.md",
".claude/agents/dart-specialist.md",
".claude/agents/devops-engineer.md",
".claude/agents/docs-writer.md",
".claude/agents/e2e-generator-engineer.md",
".claude/agents/elixir-specialist.md",
".claude/agents/extraction-engineer.md",
".claude/agents/ffi-engineer.md",
".claude/agents/go-specialist.md",
".claude/agents/java-specialist.md",
".claude/agents/jni-specialist.md",
".claude/agents/kotlin-android-specialist.md",
".claude/agents/kreuzberg-developer.md",
".claude/agents/ocr-engineer.md",
".claude/agents/performance-engineer.md",
".claude/agents/php-specialist.md",
".claude/agents/plugin-engineer.md",
".claude/agents/polyglot-architect.md",
".claude/agents/python-specialist.md",
".claude/agents/r-specialist.md",
".claude/agents/release-engineer.md",
".claude/agents/ruby-specialist.md",
".claude/agents/rust-core-engineer.md",
".claude/agents/security-auditor.md",
".claude/agents/swift-specialist.md",
".claude/agents/test-writer.md",
".claude/agents/typescript-specialist.md",
".claude/agents/wasm-specialist.md",
".claude/agents/zig-specialist.md",
".claude/settings.json",
".claude/skills/add-language-generator/SKILL.md",
".claude/skills/alef/SKILL.md",
".claude/skills/alef/references/adapters.md",
".claude/skills/alef/references/backends.md",
".claude/skills/alef/references/cli-reference.md",
".claude/skills/alef/references/configuration.md",
".claude/skills/alef/references/designing-alef-toml.md",
".claude/skills/alef/references/e2e-testing.md",
".claude/skills/alef/references/troubleshooting.md",
".claude/skills/api-server-mcp/SKILL.md",
".claude/skills/chunking-embeddings/SKILL.md",
".claude/skills/common-task-commands/SKILL.md",
".claude/skills/create-e2e-fixture/SKILL.md",
".claude/skills/extraction-pipeline-patterns/SKILL.md",
".claude/skills/format-specific-extraction/SKILL.md",
".claude/skills/iterate/SKILL.md",
".claude/skills/parallelize/SKILL.md",
".claude/skills/plugin-architecture-patterns/SKILL.md",
".claude/skills/quick-start/SKILL.md",
".codex/agents/c-ffi-specialist.toml",
".codex/agents/code-reviewer.toml",
".codex/agents/csharp-specialist.toml",
".codex/agents/dart-specialist.toml",
".codex/agents/devops-engineer.toml",
".codex/agents/docs-writer.toml",
".codex/agents/e2e-generator-engineer.toml",
".codex/agents/elixir-specialist.toml",
".codex/agents/extraction-engineer.toml",
".codex/agents/ffi-engineer.toml",
".codex/agents/go-specialist.toml",
".codex/agents/java-specialist.toml",
".codex/agents/jni-specialist.toml",
".codex/agents/kotlin-android-specialist.toml",
".codex/agents/kreuzberg-developer.toml",
".codex/agents/ocr-engineer.toml",
".codex/agents/performance-engineer.toml",
".codex/agents/php-specialist.toml",
".codex/agents/plugin-engineer.toml",
".codex/agents/polyglot-architect.toml",
".codex/agents/python-specialist.toml",
".codex/agents/r-specialist.toml",
".codex/agents/release-engineer.toml",
".codex/agents/ruby-specialist.toml",
".codex/agents/rust-core-engineer.toml",
".codex/agents/security-auditor.toml",
".codex/agents/swift-specialist.toml",
".codex/agents/test-writer.toml",
".codex/agents/typescript-specialist.toml",
".codex/agents/wasm-specialist.toml",
".codex/agents/zig-specialist.toml",
".codex/commands/iterate.md",
".codex/commands/parallelize.md",
".codex/skills/add-language-generator/SKILL.md",
".codex/skills/alef/SKILL.md",
".codex/skills/alef/references/adapters.md",
".codex/skills/alef/references/backends.md",
".codex/skills/alef/references/cli-reference.md",
".codex/skills/alef/references/configuration.md",
".codex/skills/alef/references/designing-alef-toml.md",
".codex/skills/alef/references/e2e-testing.md",
".codex/skills/alef/references/troubleshooting.md",
".codex/skills/api-server-mcp/SKILL.md",
".codex/skills/chunking-embeddings/SKILL.md",
".codex/skills/common-task-commands/SKILL.md",
".codex/skills/create-e2e-fixture/SKILL.md",
".codex/skills/extraction-pipeline-patterns/SKILL.md",
".codex/skills/format-specific-extraction/SKILL.md",
".codex/skills/plugin-architecture-patterns/SKILL.md",
".codex/skills/quick-start/SKILL.md",
".cursor/commands/iterate.md",
".cursor/commands/parallelize.md",
".cursor/rules/agent-workflow.mdc",
".cursor/rules/alef-generated-bindings.mdc",
".cursor/rules/alef-workflow.mdc",
".cursor/rules/anti-patterns.mdc",
".cursor/rules/api-compatibility.mdc",
".cursor/rules/async-and-concurrency.mdc",
".cursor/rules/atomic-commits.mdc",
".cursor/rules/avoid-duplication.mdc",
".cursor/rules/batch-operations.mdc",
".cursor/rules/bindings.mdc",
".cursor/rules/branch-hygiene.mdc",
".cursor/rules/cache-and-performance.mdc",
".cursor/rules/cgo-bindings.mdc",
".cursor/rules/cicd-pipeline-standards.mdc",
".cursor/rules/commit-messages.mdc",
".cursor/rules/communication-style.mdc",
".cursor/rules/complexity-limits.mdc",
".cursor/rules/containerization-docker.mdc",
".cursor/rules/context-config-loading-precedence.mdc",
".cursor/rules/context-crate-structure.mdc",
".cursor/rules/context-kreuzberg-brand-and-docs.mdc",
".cursor/rules/context-mime-detection-routing.mdc",
".cursor/rules/context-owasp-quick-reference.mdc",
".cursor/rules/context-polyrepo-structure.mdc",
".cursor/rules/context-pre-commit-tooling.mdc",
".cursor/rules/context-prek.mdc",
".cursor/rules/context-preservation.mdc",
".cursor/rules/context-taskfile-structure.mdc",
".cursor/rules/context-wasm-constraints.mdc",
".cursor/rules/csharp-conventions.mdc",
".cursor/rules/dead-code.mdc",
".cursor/rules/dependency-awareness.mdc",
".cursor/rules/e2e-generator-conventions.mdc",
".cursor/rules/elixir-conventions.mdc",
".cursor/rules/error-handling.mdc",
".cursor/rules/explain-reasoning.mdc",
".cursor/rules/ext-php-rs-bindings.mdc",
".cursor/rules/extendr-bindings.mdc",
".cursor/rules/extraction-quality.mdc",
".cursor/rules/extraction-safety.mdc",
".cursor/rules/feature-flag-policy.mdc",
".cursor/rules/ffi-and-language-interop.mdc",
".cursor/rules/fixture-schema-design.mdc",
".cursor/rules/gcloud-conventions.mdc",
".cursor/rules/generated-code-policy.mdc",
".cursor/rules/gh-workflows.mdc",
".cursor/rules/go-conventions.mdc",
".cursor/rules/incremental-approach.mdc",
".cursor/rules/input-validation.mdc",
".cursor/rules/java-conventions.mdc",
".cursor/rules/least-privilege.mdc",
".cursor/rules/magnus-bindings.mdc",
".cursor/rules/meaningful-assertions.mdc",
".cursor/rules/minimal-changes.mdc",
".cursor/rules/monitoring-observability.mdc",
".cursor/rules/napi-rs-bindings.mdc",
".cursor/rules/no-ai-signatures.mdc",
".cursor/rules/ocr-backend-standards.mdc",
".cursor/rules/ocr-language-and-config.mdc",
".cursor/rules/ocr-performance.mdc",
".cursor/rules/ocr-quality.mdc",
".cursor/rules/ocr-table-and-hocr.mdc",
".cursor/rules/output-awareness.mdc",
".cursor/rules/php-conventions.mdc",
".cursor/rules/plugin-extensibility.mdc",
".cursor/rules/plugin-interface-contract.mdc",
".cursor/rules/plugin-registry-and-selection.mdc",
".cursor/rules/plugin-testing.mdc",
".cursor/rules/pyo3-bindings.mdc",
".cursor/rules/python-conventions.mdc",
".cursor/rules/python-ffi-plugins.mdc",
".cursor/rules/r-conventions.mdc",
".cursor/rules/read-before-write.mdc",
".cursor/rules/readability-first.mdc",
".cursor/rules/ruby-conventions.mdc",
".cursor/rules/rust-conventions.mdc",
".cursor/rules/rust-polyglot-conventions.mdc",
".cursor/rules/rustler-bindings.mdc",
".cursor/rules/safe-git-operations.mdc",
".cursor/rules/secrets-handling.mdc",
".cursor/rules/systematic-debugging.mdc",
".cursor/rules/task-automation-build.mdc",
".cursor/rules/task-runner.mdc",
".cursor/rules/tdd-workflow.mdc",
".cursor/rules/test-alongside-code.mdc",
".cursor/rules/test-independence.mdc",
".cursor/rules/test-naming.mdc",
".cursor/rules/testing-anti-patterns.mdc",
".cursor/rules/typescript-conventions.mdc",
".cursor/rules/verification-before-completion.mdc",
".cursor/rules/verify-before-acting.mdc",
".cursor/rules/wasm-bindings.mdc",
".github/agents/c-ffi-specialist.agent.md",
".github/agents/code-reviewer.agent.md",
".github/agents/csharp-specialist.agent.md",
".github/agents/dart-specialist.agent.md",
".github/agents/devops-engineer.agent.md",
".github/agents/docs-writer.agent.md",
".github/agents/e2e-generator-engineer.agent.md",
".github/agents/elixir-specialist.agent.md",
".github/agents/extraction-engineer.agent.md",
".github/agents/ffi-engineer.agent.md",
".github/agents/go-specialist.agent.md",
".github/agents/java-specialist.agent.md",
".github/agents/jni-specialist.agent.md",
".github/agents/kotlin-android-specialist.agent.md",
".github/agents/kreuzberg-developer.agent.md",
".github/agents/ocr-engineer.agent.md",
".github/agents/performance-engineer.agent.md",
".github/agents/php-specialist.agent.md",
".github/agents/plugin-engineer.agent.md",
".github/agents/polyglot-architect.agent.md",
".github/agents/python-specialist.agent.md",
".github/agents/r-specialist.agent.md",
".github/agents/release-engineer.agent.md",
".github/agents/ruby-specialist.agent.md",
".github/agents/rust-core-engineer.agent.md",
".github/agents/security-auditor.agent.md",
".github/agents/swift-specialist.agent.md",
".github/agents/test-writer.agent.md",
".github/agents/typescript-specialist.agent.md",
".github/agents/wasm-specialist.agent.md",
".github/agents/zig-specialist.agent.md",
".github/commands/iterate.md",
".github/commands/parallelize.md",
".github/copilot-instructions.md",
".github/skills/add-language-generator/SKILL.md",
".github/skills/alef/SKILL.md",
".github/skills/alef/references/adapters.md",
".github/skills/alef/references/backends.md",
".github/skills/alef/references/cli-reference.md",
".github/skills/alef/references/configuration.md",
".github/skills/alef/references/designing-alef-toml.md",
".github/skills/alef/references/e2e-testing.md",
".github/skills/alef/references/troubleshooting.md",
".github/skills/api-server-mcp/SKILL.md",
".github/skills/chunking-embeddings/SKILL.md",
".github/skills/common-task-commands/SKILL.md",
".github/skills/create-e2e-fixture/SKILL.md",
".github/skills/extraction-pipeline-patterns/SKILL.md",
".github/skills/format-specific-extraction/SKILL.md",
".github/skills/plugin-architecture-patterns/SKILL.md",
".github/skills/quick-start/SKILL.md",
".mcp.json",
"AGENTS.md",
"CLAUDE.md",
"GEMINI.md"
]
}

17
.ai-rulez/agents/kreuzberg-developer.md Normal file
View File

@@ -0,0 +1,17 @@
---
name: kreuzberg-developer
description: General kreuzberg development guidance and cross-cutting concerns
model: haiku
---
When working on kreuzberg:
1. Rust core is the single source of truth — all business logic in crates/kreuzberg/src/
2. Bindings (Python, TypeScript, Ruby, PHP, etc.) are thin wrappers — never duplicate core logic
3. Use `task` commands for all operations: `task build` is core-only; use `task build:bindings` or `task build:all` explicitly when bindings are needed
4. Build FFI layer first if needed: `task build:bindings`
5. For ONNX features: ensure ORT_LIB_LOCATION is set or use download-binaries feature
6. All unsafe blocks require SAFETY comments. No .unwrap() in production code.
7. Coverage targets: 95% for Rust core, 80% for bindings
8. WASM builds are sync-only — implement SyncExtractor for WASM-compatible extractors
9. Version in root Cargo.toml is the single source of truth for all binding packages

71
.ai-rulez/config.toml Normal file
View File

@@ -0,0 +1,71 @@
# AI-Rulez Configuration (migrated to V4 TOML format)
# Documentation: https://github.com/Goldziher/ai-rulez
version = '4.0'
name = 'Kreuzberg'
description = 'Rust document intelligence library with active Python, TypeScript/Node, Ruby, PHP, Go, Java, C#, Elixir, R, WebAssembly, Dart, Kotlin Android, Swift, Zig, and C FFI bindings'
gitignore = true
presets = ['claude', 'copilot', 'cursor', 'antigravity', 'codex']
builtins = [
'rust',
'python',
'go',
'java',
'ruby',
'php',
'csharp',
'elixir',
'r',
'wasm',
'pyo3',
'napi-rs',
'magnus',
'ext-php-rs',
'rustler',
'cgo',
'extendr',
'default-commands',
]
[[includes]]
name = 'kreuzberg-core'
source = 'https://github.com/kreuzberg-dev/ai-rulez.git'
path = 'modules/core'
merge_strategy = 'local-override'
[[includes]]
name = 'kreuzberg-languages'
source = 'https://github.com/kreuzberg-dev/ai-rulez.git'
path = 'modules/languages'
merge_strategy = 'local-override'
[[includes]]
name = 'kreuzberg-cicd'
source = 'https://github.com/kreuzberg-dev/ai-rulez.git'
path = 'modules/cicd'
merge_strategy = 'local-override'
[[includes]]
name = 'kreuzberg-infrastructure'
source = 'https://github.com/kreuzberg-dev/ai-rulez.git'
path = 'modules/infrastructure'
merge_strategy = 'local-override'
[[includes]]
name = 'kreuzberg-e2e-generator'
source = 'https://github.com/kreuzberg-dev/ai-rulez.git'
path = 'modules/e2e-generator'
merge_strategy = 'local-override'
[[installed_skills]]
name = 'alef'
source = 'https://github.com/kreuzberg-dev/alef.git'
[[mcp_servers]]
name = 'playwright'
description = 'Playwright browser automation for E2E testing and docs verification'
command = 'npx'
args = ['-y', '@playwright/mcp@latest']
[defaults]
effort = 'medium'

65
.ai-rulez/context/config-loading-precedence.md Normal file
View File

@@ -0,0 +1,65 @@
---
summary: Configuration loading precedence for CLI and server modes
---
# Configuration Loading & Precedence
## CLI Mode Precedence (highest to lowest)
1. Individual CLI flags (`--ocr`, `--output-format`, `--chunk`)
2. Inline JSON config (`--config-json` or `--config-json-base64`)
3. Config file (`--config path.toml`)
4. Auto-discovered config (`kreuzberg.{toml,yaml,json}` in cwd/parents)
5. Default values
## Server/MCP Mode Precedence
1. CLI arguments (`--host`, `--port`)
2. Environment variables (`KREUZBERG_HOST`, `KREUZBERG_PORT`)
3. Config file `[server]` section
4. Defaults (`127.0.0.1:8000`)
## Config File Discovery
Searches current directory and parents for `kreuzberg.toml`, `kreuzberg.yaml`, or `kreuzberg.json`. Stops at first match.
## Inline JSON Config
Field-level merge (not whole-object replacement):
```rust
fn merge_json_into_config(base: &ExtractionConfig, json: Value) -> Result<ExtractionConfig> {
let mut config_json = serde_json::to_value(base)?;
// Merge fields from json into config_json
serde_json::from_value(merged)?
}
```
Use `--config-json-base64` for shell escaping.
## Config File Formats
**TOML** (`kreuzberg.toml`):
```toml
use_cache = true
[ocr]
backend = "tesseract"
languages = ["eng", "deu"]
[security_limits]
max_archive_size = 524288000
```
**YAML** and **JSON** follow equivalent structure.
## CLI Flag Overrides
In `commands.rs`: `apply_extraction_overrides()` applies individual flags on top of merged config.
## Critical Rules
1. CLI flags always win over config file
2. JSON merge is field-level, not whole-object
3. Auto-discovery stops at first config file found
4. `--config-json-base64` for shell-safe JSON passing
5. Server config uses `[server]` section + extraction config

36
.ai-rulez/context/crate-structure.md Normal file
View File

@@ -0,0 +1,36 @@
---
priority: high
---
# Crate Structure
Version source of truth: root `Cargo.toml` `[workspace.package] version`.
## Workspace crates (`crates/`)
- `kreuzberg` — core library: extraction engine, MIME detection, plugin system, OCR, chunking, embeddings, API/MCP server
- `kreuzberg-cli` — CLI binary; thin wrapper over core with `cli` feature set
- `kreuzberg-ffi` — C FFI layer (`#[no_mangle] extern "C"`); opaque handles, cbindgen headers; used by Go, Java, C# bindings
- `kreuzberg-node` — NAPI-RS Node.js/TypeScript bindings
- `kreuzberg-py` — PyO3 Python bindings
- `kreuzberg-php` — ext-php-rs PHP bindings
- `kreuzberg-wasm` — wasm-bindgen WASM bindings; uses `wasm-target` feature set
- `kreuzberg-paddle-ocr` — PaddleOCR via ONNX Runtime; not available on WASM or Windows
- `kreuzberg-tesseract` — Rust bindings for Tesseract OCR
## Out-of-workspace bindings (`packages/`)
- `packages/python/` — PyPI (maturin + PyO3)
- `packages/typescript/` — npm type declarations
- `packages/ruby/` — RubyGems (Magnus); native ext compiled by `rake`
- `packages/php/` — Composer (ext-php-rs)
- `packages/go/v5/` — Go module; cgo over kreuzberg-ffi
- `packages/java/` — Maven; Foreign Function & Memory API over kreuzberg-ffi
- `packages/csharp/` — NuGet; P/Invoke over kreuzberg-ffi
- `packages/elixir/` — Hex; Rustler NIF (workspace member at `packages/elixir/native/kreuzberg_rustler`)
- `packages/r/` — CRAN; extendr (excluded from workspace)
## Tools (`tools/`)
- `tools/e2e-generator` — reads JSON fixtures, generates runnable test suites per language into `e2e/`
- `tools/benchmark-harness` — criterion-based benchmark runner

56
.ai-rulez/context/mime-detection-routing.md Normal file
View File

@@ -0,0 +1,56 @@
---
summary: MIME type detection and extractor routing logic
---
# MIME Detection & Routing
## Detection Flow
```text
Extension -> EXT_TO_MIME map -> validate -> Registry lookup -> Extractor
```
## Key Functions
| Function | Location | Purpose |
| ------------------------------------ | -------------- | --------------------------------------- |
| `detect_mime_type(path, inspect)` | `core/mime.rs` | Extension + optional content inspection |
| `detect_mime_type_from_bytes(bytes)` | `core/mime.rs` | Magic number detection (infer crate) |
| `validate_mime_type(mime)` | `core/mime.rs` | Check if any extractor supports it |
## Extension Mapping
118+ extensions mapped in `EXT_TO_MIME` (`core/mime.rs`). Case-insensitive.
Key mappings: `.pdf` -> `application/pdf`, `.docx` -> `application/vnd.openxmlformats-officedocument.wordprocessingml.document`, `.xlsx` -> spreadsheet variant, `.png`/`.jpg` -> `image/*`
## Registry Selection
```rust
// In core/extractor/bytes.rs
fn select_extractor_for_mime(mime_type: &str) -> Result<Arc<dyn DocumentExtractor>> {
let registry = get_document_extractor_registry();
let registry_guard = registry.read()?;
registry_guard.get_for_mime_type(mime_type)
.ok_or_else(|| KreuzbergError::UnsupportedFormat(mime_type.into()))
}
```
Selects highest-priority extractor registered for that MIME type.
## Adding New MIME Types
1. Add extension mapping: `m.insert("ext", "application/x-new");` in `core/mime.rs`
2. Implement `DocumentExtractor` with `supported_mime_types()` returning the MIME
3. Register in `register_default_extractors()`
## Wildcard Support
Extractors can register for MIME type families: `"image/*"` matches `image/png`, `image/jpeg`, etc.
## Critical Rules
1. Always `validate_mime_type()` before extraction
2. Extension mapping is case-insensitive
3. Content inspection (infer crate) is fallback for extension-less files
4. Registry validation is final authority on supported types

78
.ai-rulez/context/wasm-constraints.md Normal file
View File

@@ -0,0 +1,78 @@
---
summary: WASM build constraints and patterns for kreuzberg-wasm crate
---
# WASM Build Constraints
## Overview
WASM target in `crates/kreuzberg-wasm/`. Uses wasm-bindgen with sync-only internal APIs.
## Feature Flags
```toml
[features]
wasm-target = ["pdf", "html", "xml", "email", "language-detection", "chunking", "quality", "office"]
wasm-threads = ["dep:wasm-bindgen-rayon"] # Optional
```
## Critical Constraints
### 1. No Tokio Runtime
All operations must be synchronous internally. Use `#[cfg(not(feature = "tokio-runtime"))]` paths.
### 2. SyncExtractor Required
Every WASM-compatible extractor MUST implement `SyncExtractor`:
```rust
impl SyncExtractor for MyExtractor {
fn extract_sync(&self, content: &[u8], mime_type: &str, config: &ExtractionConfig)
-> Result<ExtractionResult> { /* sync implementation */ }
}
impl DocumentExtractor for MyExtractor {
fn as_sync_extractor(&self) -> Option<&dyn SyncExtractor> {
Some(self) // MUST return Some for WASM
}
}
```
### 3. HTML Size Limit
```rust
const MAX_HTML_SIZE: usize = 2 * 1024 * 1024; // 2MB - stack constraint
```
## Build Config
```toml
[lib]
crate-type = ["cdylib", "rlib"]
[profile.release.package.kreuzberg-wasm]
opt-level = "z" # Size optimization
codegen-units = 1
```
## API Pattern
```rust
#[wasm_bindgen]
pub async fn extract_from_bytes(content: Vec<u8>, config: JsValue) -> Result<JsValue, JsValue> {
let config: ExtractionConfig = serde_wasm_bindgen::from_value(config)?;
let result = extract_bytes_sync(&content, mime_type, &config)?;
Ok(serde_wasm_bindgen::to_value(&result)?)
}
```
Functions can be `async` for JS compatibility, but internal extraction is sync.
## Critical Rules
1. **No tokio** -- all operations synchronous
2. **Implement SyncExtractor** for all WASM-compatible extractors
3. **HTML limited to 2MB** due to stack constraints
4. **Size optimization** via `opt-level = "z"`
5. **Feature gate** with `#[cfg(target_arch = "wasm32")]`

12
.ai-rulez/domains/document-extraction/DOMAIN.md Normal file
View File

@@ -0,0 +1,12 @@
---
description: Document extraction pipeline architecture
---
- Pipeline: file input → MIME detection (magic bytes + extension) → extractor routing → extraction → post-processing → ExtractionResult
- Extractors are plugins implementing the Extractor trait: extract(&self, source: &ExtractionSource) → ExtractionResult
- Fallback chains: if primary extractor fails, try next in priority order (e.g., native PDF → Tesseract OCR → error)
- Cache-first: check extraction cache before running extractors, cache results keyed by content hash
- ExtractionResult contains: text content, metadata (page count, language, confidence), optional structured data (tables, images)
- Async-first: all extraction paths are async, use spawn_blocking for CPU-bound work (OCR, image processing)
- Memory limits: streaming for large files, configurable max file size, depth limits for nested archives
- Format coverage: 91+ formats — PDF, DOCX, XLSX, PPTX, HTML, images, email (EML/MSG), archives, plain text

16
.ai-rulez/domains/document-extraction/agents/extraction-engineer.md Normal file
View File

@@ -0,0 +1,16 @@
---
name: extraction-engineer
description: Document extraction pipeline development and maintenance
model: haiku
---
When working on document extraction code:
1. Key source paths: crates/kreuzberg/src/core/ (extractor.rs, mime.rs, config.rs), crates/kreuzberg/src/extraction/
2. The extraction pipeline: Input -> Cache Check -> MIME Detection -> Format Conversion -> Extractor Selection (priority-based) -> Extraction -> Fallback Chain -> Post-Processing -> Caching -> Output
3. For MIME detection: use EXT_TO_MIME map + magic bytes fallback via infer crate. Always validate_mime_type() before extraction.
4. For caching: keys based on content hash, invalidate on config changes
5. For errors: implement fallback chains (try next-priority extractor), preserve partial results, return structured error info
6. For new formats: add to EXT_TO_MIME, implement DocumentExtractor trait, register in register_default_extractors()
7. Always use SecurityLimits validators for user content (ZipBombValidator, DepthValidator, StringGrowthValidator)
8. Run `task test` after changes. Target 95% coverage on core extraction code.

9
.ai-rulez/domains/document-extraction/rules/api-compatibility.md Normal file
View File

@@ -0,0 +1,9 @@
---
priority: high
---
- Follow semantic versioning — breaking changes require major version bump
- Document all public API changes in CHANGELOG.md
- Maintain backward compatibility for at least one minor version before removing deprecated APIs
- All public types must be FFI-friendly or have FFI-compatible equivalents
- Version in Cargo.toml is the single source of truth for all binding packages

9
.ai-rulez/domains/document-extraction/rules/async-and-concurrency.md Normal file
View File

@@ -0,0 +1,9 @@
---
priority: high
---
- All extraction paths must be fully async using tokio
- Never block the async runtime — use spawn_blocking for CPU-intensive work
- All public types must be Send + Sync
- Use tokio::select! for timeout handling on extraction operations
- Cross-platform: test on Linux (amd64, arm64) and macOS at minimum

10
.ai-rulez/domains/document-extraction/rules/cache-and-performance.md Normal file
View File

@@ -0,0 +1,10 @@
---
priority: high
---
- Cache keys: content-hash based (hash of file bytes + config), not path-based
- Invalidate cache when extraction config changes (output format, OCR settings, etc.)
- Check cache before any extraction — cache hits should skip all processing
- Concurrent batch processing: use configurable worker pool, default to CPU count
- Stream large files instead of loading into memory — use AsyncRead where possible
- Monitor cache hit rates — target >80% for repeated extractions

10
.ai-rulez/domains/document-extraction/rules/extraction-quality.md Normal file
View File

@@ -0,0 +1,10 @@
---
priority: high
---
- 95% test coverage on core extraction code, 80% on bindings
- Test all format categories: text, office, PDF, images, archives, markup
- Test corrupted/malformed documents — extraction must fail gracefully, never panic
- Benchmark extraction speeds per format — track regressions in CI
- Test both success and error paths for every extractor
- Use property-based testing for parsers with wide input ranges

10
.ai-rulez/domains/document-extraction/rules/extraction-safety.md Normal file
View File

@@ -0,0 +1,10 @@
---
priority: critical
---
- Always use `SecurityLimits` to cap archive size, compression ratio, file count, and nesting depth for user content. Use `ZipBombValidator` for archive extraction.
- Validate MIME type before extraction — never trust file extensions alone
- Implement fallback chains: if primary extractor fails, try next-priority extractor
- Preserve partial results on failure — return what was extracted with error context
- All errors must include: operation name, input description, root cause, and suggestion
- Never expose internal file paths or system details in error messages returned to users

13
.ai-rulez/domains/ocr-integration/DOMAIN.md Normal file
View File

@@ -0,0 +1,13 @@
---
description: OCR backend integration and image processing
---
- Multiple backends: Tesseract (C FFI via leptonica/tesseract-sys), PaddleOCR (ONNX Runtime), Python backends (EasyOCR, Surya) via FFI
- Backend selection: priority-based with fallback — Tesseract default, PaddleOCR for CJK, Python backends as fallback
- Image preprocessing: deskew, binarization, noise removal, contrast enhancement — applied before OCR
- PSM modes: configurable page segmentation (single block, single line, sparse text) per use case
- Table detection: identify table regions → cell extraction → row/column reconstruction → Markdown table output
- hOCR: parse Tesseract hOCR output for word-level bounding boxes, confidence scores, reading order
- Language management: auto-detect document language, load appropriate Tesseract traineddata, support multi-language documents
- Caching: cache OCR results by image hash + backend + language + PSM mode
- Confidence tracking: per-word and per-page confidence scores, flag low-confidence regions for review

16
.ai-rulez/domains/ocr-integration/agents/ocr-engineer.md Normal file
View File

@@ -0,0 +1,16 @@
---
name: ocr-engineer
description: OCR pipeline development, backend integration, and table reconstruction
model: haiku
---
When working on OCR code:
1. Key source paths: crates/kreuzberg/src/ocr/ (processor.rs, tesseract_backend.rs, hocr.rs, cache.rs, language_registry.rs, table/)
2. The OCR pipeline: Image Detection -> Preprocessing (denoise, deskew, binarize) -> Backend Selection -> OCR Execution -> hOCR Parsing -> Table Reconstruction -> Caching -> Return
3. Backends: Tesseract (default, native C FFI via leptess), PaddleOCR (ONNX via ort), EasyOCR (Python via PyO3)
4. For Python backends: use tokio::task::spawn_blocking, minimize GIL hold time with py.allow_threads(), cache Python data in Rust fields
5. For table detection: detect via line/cell boundary detection, validate grid structure, OCR each cell, output as markdown
6. For language management: validate against LanguageRegistry, check tessdata availability
7. Cache OCR results with key = hash(image_bytes + language + config)
8. hOCR parsing: use the hocr module to extract word-level bounding boxes and confidence scores

11
.ai-rulez/domains/ocr-integration/rules/ocr-backend-standards.md Normal file
View File

@@ -0,0 +1,11 @@
---
priority: critical
---
- Pluggable backend architecture: all backends implement the OcrBackend trait
- Backend independence: switching backends must not require API changes
- Tesseract is the default backend (native C FFI via leptess)
- Python backends (EasyOCR, PaddleOCR): use tokio::task::spawn_blocking, release GIL for Rust work
- Graceful degradation: if preferred backend unavailable, fall back to next available
- All backends must return structured results with confidence scores
- Document installation requirements and troubleshooting for each backend

9
.ai-rulez/domains/ocr-integration/rules/ocr-language-and-config.md Normal file
View File

@@ -0,0 +1,9 @@
---
priority: medium
---
- Validate language packs exist before OCR execution — fail fast with helpful message
- Support ISO 639 language codes, map to backend-specific formats
- Configuration cascade: CLI args > environment > config file > defaults
- Provide troubleshooting guides for common issues (missing tessdata, backend not found)
- Language pack installation: document per-platform instructions

10
.ai-rulez/domains/ocr-integration/rules/ocr-performance.md Normal file
View File

@@ -0,0 +1,10 @@
---
priority: high
---
- Cache OCR results: key = hash(image_bytes + language + config)
- Invalidate cache when OCR config changes (backend, language, PSM mode)
- Batch processing: process multiple images concurrently with configurable parallelism
- Resource management: limit concurrent OCR operations to avoid memory exhaustion
- Performance targets: <2s for single page, <10s for 10-page document
- Monitor and log OCR processing times for regression detection

10
.ai-rulez/domains/ocr-integration/rules/ocr-quality.md Normal file
View File

@@ -0,0 +1,10 @@
---
priority: high
---
- Track confidence scores on all OCR results — expose in API
- Image preprocessing (denoise, deskew, binarize) should improve accuracy by 10-30%
- PSM mode selection: auto-detect layout, allow user override (single block, single line, sparse text, etc.)
- Language detection: validate requested languages are available, provide install hints if not
- Multi-language support: allow multiple languages per OCR request
- Test OCR accuracy against ground-truth documents in CI

10
.ai-rulez/domains/ocr-integration/rules/ocr-table-and-hocr.md Normal file
View File

@@ -0,0 +1,10 @@
---
priority: high
---
- hOCR parsing: extract word-level bounding boxes, confidence scores, and text content
- Preserve spatial relationships from hOCR output for layout reconstruction
- Table detection: use cell boundary detection (line detection + intersection analysis)
- Validate grid structure before treating detected regions as tables
- OCR each cell individually for better accuracy
- Convert tables to markdown format with proper column alignment

13
.ai-rulez/domains/plugin-system/DOMAIN.md Normal file
View File

@@ -0,0 +1,13 @@
---
description: Plugin trait system and Python FFI integration
---
- Core traits: Extractor, PostProcessor, MetadataExtractor — each with async extract/process methods returning Result
- Discovery: static registration (Rust plugins compiled in) + dynamic discovery (Python plugins via PyO3 FFI)
- Priority selection: plugins declare priority per MIME type, registry selects highest-priority match, fallback to next
- Registry: PluginRegistry holds all discovered plugins, provides lookup by MIME type, supports hot-reload for Python plugins
- Python FFI: Python plugins implement a Python class matching the trait interface, called via PyO3 with GIL management
- GIL management: acquire GIL only for Python calls, release immediately after, use py.allow_threads() for Rust-side work
- Plugin lifecycle: init → register → validate → ready. Plugins validate their dependencies (e.g., Tesseract binary, Python packages) at startup
- Error handling: plugin errors are wrapped in PluginError with source plugin name, converted to ExtractionError at boundary
- Testing: test plugins with real files (not mocks), test fallback chains, test Python plugin loading/unloading

16
.ai-rulez/domains/plugin-system/agents/plugin-engineer.md Normal file
View File

@@ -0,0 +1,16 @@
---
name: plugin-engineer
description: Plugin system architecture, registry management, and Python FFI
model: haiku
---
When working on the plugin system:
1. Key source paths: crates/kreuzberg/src/plugins/ (mod.rs, extractor.rs, ocr.rs, postprocessor.rs, validator.rs, registry.rs), crates/kreuzberg-py/src/plugins.rs
2. Plugin types: DocumentExtractor, OcrBackend, PostProcessor, Validator — all extend base Plugin trait (Send + Sync required)
3. Priority system: 0-255, default 50, custom override > 50, fallback < 50. Registry selects highest priority for MIME type.
4. Registries use Arc<RwLock<>> with MIME type indexing for O(log n) lookup
5. Python plugins: validate protocol compliance, use py.allow_threads() for expensive Rust ops, tokio::task::spawn_blocking for async calls
6. For new plugin types: define trait extending Plugin, create typed registry, add registration functions, implement priority-based selection
7. GIL optimization: cache frequently-accessed Python data in Rust fields, measure GIL overhead
8. All plugins must handle errors gracefully — return Result, never panic

10
.ai-rulez/domains/plugin-system/rules/plugin-extensibility.md Normal file
View File

@@ -0,0 +1,10 @@
---
priority: medium
---
- API stability: plugin interfaces are versioned, breaking changes require major version bump
- Plugin discovery: support both static (compile-time) and dynamic (runtime) registration
- Plugin validation: check capabilities, supported formats, and version compatibility before registration
- Plugin chaining: post-processors can be composed in sequence
- Configuration: plugins accept typed configuration, validated at registration time
- Documentation: every plugin type must have a development guide with examples

10
.ai-rulez/domains/plugin-system/rules/plugin-interface-contract.md Normal file
View File

@@ -0,0 +1,10 @@
---
priority: critical
---
- All plugins must implement the base Plugin trait: Send + Sync + 'static required
- Plugin types: DocumentExtractor, OcrBackend, PostProcessor, Validator
- Async execution: use async trait methods for non-blocking operations
- Lifecycle: init() -> process() -> cleanup(). Init must validate all requirements.
- Never panic in plugin code — all errors must be returned as Result
- Consistent result format: all extractors return ExtractionResult with text, metadata, and confidence

12
.ai-rulez/domains/plugin-system/rules/plugin-registry-and-selection.md Normal file
View File

@@ -0,0 +1,12 @@
---
priority: critical
---
- Separate typed registry per plugin type (ExtractorRegistry, OcrRegistry, etc.)
- Thread safety: Arc<RwLock<>> for all registries
- Priority system: 0-255, default 50, custom > 50, fallback < 50
- Selection: highest priority plugin matching the MIME type wins
- MIME type indexing for O(log n) lookup
- Conflict resolution: if equal priority, prefer Rust-native over FFI plugins
- Dynamic registration: plugins can be added/removed at runtime
- Validate plugin before registration (check trait compliance, supported formats)

10
.ai-rulez/domains/plugin-system/rules/plugin-testing.md Normal file
View File

@@ -0,0 +1,10 @@
---
priority: high
---
- Mock plugin testing: create test doubles for unit tests
- Real plugin testing: integration tests with actual backends
- Thread safety tests: run concurrent plugin operations to detect race conditions
- Performance baselines: measure and track plugin overhead vs direct calls
- Test all error paths: invalid input, backend failure, timeout, resource exhaustion
- Test plugin lifecycle: register, use, unregister, verify cleanup

11
.ai-rulez/domains/plugin-system/rules/python-ffi-plugins.md Normal file
View File

@@ -0,0 +1,11 @@
---
priority: high
---
- GIL management: use py.allow_threads() for expensive Rust operations
- Cache frequently-accessed Python data in Rust fields to minimize GIL acquisitions
- Use tokio::task::spawn_blocking for async calls to Python backends
- Python exception translation: convert Python exceptions to Rust errors with full context
- Data type mapping: Python str <-> Rust String, Python bytes <-> Rust Vec<u8>, Python dict <-> Rust HashMap
- Validate Python plugin protocol compliance on registration
- Target GIL overhead: 5-55us per acquisition

45
.ai-rulez/ground-truth-generation.md Normal file
View File

@@ -0,0 +1,45 @@
# Ground Truth Generation
## Pandoc Commands
```bash
pandoc <source_file> -t gfm --wrap=none -o <gt_file.md>
pandoc <source_file> -t plain --wrap=none -o <gt_file.txt>
```
## Artifact Removal
```bash
sed -i '' 's/ {#[^}]*}//g' "$file" # Remove {#id} attributes
sed -i '' 's/ {[^}]*}//g' "$file" # Remove {.class} attributes
sed -i '' '/^:::/d' "$file" # Remove fenced div markers
sed -i '' 's/\\\$/$/g' "$file" # Unescape dollar signs
sed -i '' "s/\\\\'/'/g" "$file" # Unescape quotes
```
## Cleanup Rules
1. Convert ALL HTML to markdown equivalents where possible
2. For colspan/rowspan, put content in first cell, leave others empty
3. Remove `<!-- -->` comments
4. Strip `<u>`, `<sup>`, `<sub>` tags (keep text content)
5. Convert `<img>` to `![alt](src)`
6. Collapse 3+ consecutive blank lines to 2
7. Never use our own extractor output as GT
## Fixture JSON Structure
```json
{
"document": "relative/path/to/source.ext",
"file_type": "docx",
"file_size": 12345,
"expected_frameworks": ["kreuzberg"],
"metadata": { "description": "...", "source": "pandoc-generated" },
"ground_truth": {
"text_file": "relative/path/to/gt.txt",
"markdown_file": "relative/path/to/gt.md",
"source": "pandoc"
}
}
```

34
.ai-rulez/rules/alef-generated-bindings.md Normal file
View File

@@ -0,0 +1,34 @@
---
priority: critical
---
# Alef-Generated Bindings
Files in `packages/*/` and binding crates are generated by Alef — DO NOT manually edit.
## Workflow
1. Check `alef.toml` before editing anything in `packages/*/` or binding crates
2. Modify Rust source files or `alef.toml` itself
3. Run `task alef:generate` to regenerate all Alef-managed output without formatting (`alef all --clean --format=false`)
4. Run `task alef:format` explicitly if Alef post-generation formatting is needed
5. Run `task e2e:test` or `task e2e:all` to verify
6. Commit Rust source + `alef.toml` + regenerated bindings atomically
## Rename Mappings (from `alef.toml`)
- **Go**: `DocumentExtractor` → `Extractor`
- All other languages: no renames
## Freshness Check
`task alef:generate && git diff --exit-code packages/ crates/kreuzberg-node/ crates/kreuzberg-wasm/ crates/kreuzberg-ffi/`
## Key `alef.toml` Sections
- `[crate]` — Rust source files parsed for type/function extraction
- `[languages.*]` — per-language output path, module name, rename mappings
- `[e2e]` — e2e test generation: fixtures dir, output dir, per-language call overrides
- `[readme]` — README template generation per language package
Canonical e2e tasks are `task e2e:generate`, `task e2e:build`, `task e2e:test`, and `task e2e:all`. Do not add legacy aliases.

65
.ai-rulez/rules/feature-flag-policy.md Normal file
View File

@@ -0,0 +1,65 @@
---
priority: high
---
# Feature Flag Policy
All features in `crates/kreuzberg/Cargo.toml`.
## ORT-Incompatible Targets (WASM, Android x86_64 emulator)
Only ORT-dependent paths are incompatible. The same paths block both WASM (no native ORT linkage at all) and the `x86_64-linux-android` emulator triple (no pyke prebuilt; `aarch64-linux-android` does ship a prebuilt and gets full ORT):
- `paddle-ocr` — ONNX Runtime + native C++ deps: not WASM-safe; no Android x86_64 prebuilt
- `layout-detection` — depends on ONNX Runtime layout models: not WASM-safe; no Android x86_64 prebuilt
- `embeddings` — depends on ONNX Runtime sentence-transformer models: not WASM-safe; no Android x86_64 prebuilt
- `auto-rotate` — depends on ONNX Runtime orientation classifier: not WASM-safe; no Android x86_64 prebuilt
Pure-Rust **type-only** companion features expose the public config/result types for the above without pulling in ORT:
- `layout-types` — `LayoutDetectionConfig`, `TableModel`, `BBox`, `DetectionResult`, `LayoutClass`, `LayoutDetection`, `RecognizedTable`. `layout-detection` implies `layout-types`.
- `auto-rotate-types` — `OrientationResult`. `auto-rotate` implies `auto-rotate-types`.
- `embedding-presets` — `EmbeddingPreset` (already existed; pure-Rust preset metadata).
WASM/Android-safe variants:
- `ocr` (native) → `ocr-wasm` (uses `tesseract-wasm` + safe image deps) — Android keeps native `ocr`
- `excel` (native) → `excel-wasm` (drops `tokio-runtime`) — Android keeps native `excel`
- `tree-sitter` (native dlopen) → `tree-sitter-wasm` (statically-linked grammar pack) — Android keeps native `tree-sitter`
- `liter-llm` — works on WASM via the upstream `wasm-http` feature; included in `no-ort-target`
- `stopwords` — pure-Rust, included in `no-ort-target`
- `keywords` — pure-Rust YAKE/RAKE, included in `no-ort-target`
The `no-ort-target` aggregate is the shared no-ORT base used by both `wasm-target` and `android-target`. `wasm-target = no-ort-target + excel-wasm + tree-sitter-wasm + ocr-wasm`. `android-target = no-ort-target + excel + tree-sitter + ocr + api + mcp`.
## Experimental (NOT in `full`)
- `pdf-oxide` — pure-Rust PDF text extraction; opt-in only, excluded from both `full` and `formats`
## ORT Variants (Mutually Exclusive)
- `ort-bundled` — downloads official Microsoft ORT binaries; default when OCR/ML features active
- `ort-dynamic` — load ORT from system; only when system ORT is guaranteed present
## Platform-Conditional
- `kreuzberg-paddle-ocr`, `hf-hub`, `pprof` — excluded on `wasm32`
- `ureq`: `rustls` on non-Windows; `native-tls` on Windows
- `kreuzberg-ffi` and `kreuzberg-dart` cargo dependencies are target-conditional: `cfg(all(target_os = "android", target_arch = "x86_64"))` selects `android-target`; all other targets (including arm64 Android phones) get the full ORT-enabled feature set.
## Aggregate Sets
| Feature | Description |
| ---------------- | -------------------------------------------------------------------------------------------------- |
| `formats` | All document formats + api/mcp/otel/chunking; no OCR, no ML |
| `full` | `formats` + ocr + paddle-ocr + layout + embeddings + tree-sitter + liter-llm; excludes `pdf-oxide` |
| `no-ort-target` | Pure-Rust base: every capability that does not depend on ONNX Runtime |
| `wasm-target` | `no-ort-target` + excel-wasm + tree-sitter-wasm + ocr-wasm |
| `android-target` | `no-ort-target` + excel + tree-sitter + ocr + api + mcp (for x86_64-linux-android emulator) |
## Build Profiles
- `release` — LTO thin, codegen-units=1, strip
- `profiling` — inherits release, retains debug info
- `kreuzberg-wasm` override: `opt-level="z"` (size-optimized)
- `sevenz-rust2`, `zip` override: `opt-level=2` (prevents SIGBUS on macOS ARM64)

15
.ai-rulez/rules/typescript-conventions.md Normal file
View File

@@ -0,0 +1,15 @@
---
priority: high
---
- `strict: true` + `noUncheckedIndexedAccess` in tsconfig, never `any` — use `unknown` with type guards.
- ESM imports only, `const` over `let`, `as const` for literals, `interface` over `type` for objects.
- `import type` for type-only imports to avoid runtime overhead. Discriminated unions for type-safe state.
- Formatting/linting: `oxfmt` + `oxlint`. Type checking: `tsc --noEmit` in CI.
- Testing: `vitest` (80%+ coverage). Runtime validation at system boundaries with `zod`.
- Error handling: discriminated unions for expected errors, throw only for unexpected.
- Package manager: `pnpm` with `pnpm-lock.yaml` committed, build: `tsup` or `esbuild`.
- Monorepo: workspace protocol (`workspace:*`), shared tsconfig base, `pnpm-workspace.yaml`.
- Node.js: `node:` prefix for core modules, `fetch` over `axios`.
- Security: `pnpm audit` for dependency CVE scanning. Zero tolerance for critical/high vulnerabilities.
- Anti-patterns: non-null assertions (`!`), type assertions (`as`), `enum` (use unions), `@ts-ignore`.

212
.ai-rulez/skills/api-server-mcp/SKILL.md Normal file
View File

@@ -0,0 +1,212 @@
---
description: "REST API server and MCP protocol integration"
name: api-server-mcp
priority: critical
---
# API Server & MCP Protocol
**Axum server design for document extraction endpoints, middleware, async processing, and Model Context Protocol integration for AI agents**
## Kreuzberg API Architecture
**Location**: `crates/kreuzberg/src/api/`, `crates/kreuzberg-cli/`
Kreuzberg provides a dual REST API + MCP server built with Axum + Tokio.
```text
Request Flow:
HTTP Client / AI Agent (Claude)
|
[Transport Layer]
├── REST API (Axum HTTP)
└── MCP Protocol (HTTP or Stdio)
|
[Middleware Layer]
├── CORS, Request Logging (TraceLayer)
├── Request/Response size limits
└── Rate limiting (optional)
|
[Router]
├── REST Endpoints
│ ├── POST /extract - File upload extraction
│ ├── POST /extract-url - URL-based extraction
│ ├── GET /formats - List supported formats
│ ├── GET /health - Server health check
│ ├── POST /batch - Batch document processing
│ ├── GET /cache/stats - Cache statistics
│ └── DELETE /cache - Clear extraction cache
├── MCP Endpoints
│ ├── POST /mcp/tools - List available tools
│ ├── POST /mcp/tools/call - Call a tool
│ ├── GET /mcp/resources - List resources
│ ├── GET /mcp/resources/:uri - Read resource
│ ├── GET /mcp/prompts - List prompts
│ └── GET /mcp/prompts/:name - Get prompt
|
[Handler / Tool Layer]
├── extract_handler / extract_file tool
├── batch_handler / batch_extract tool
├── health_handler / get_capabilities tool
└── format_handler
|
[Extraction Core]
├── Format detection
├── Extraction pipeline
├── Post-processing (chunking, embeddings)
└── Result formatting
|
JSON Response / MCP ToolResult
```
## Server Setup & Configuration
**Location**: `crates/kreuzberg/src/api/server.rs`
Server initialization pattern: Create `ApiState` (holds `ExtractionConfig` + `ExtractionCache`), build Axum `Router` with all REST + MCP routes, apply middleware layers (body limits, CORS, tracing), serve via `tokio::net::TcpListener`.
Key middleware layers applied in order:
- `DefaultBodyLimit::max(100MB)` + `RequestBodyLimitLayer` -- configurable via env vars
- `CorsLayer::permissive()` -- restrict in production via `CORS_ALLOWED_ORIGINS`
- `TraceLayer::new_for_http()` -- request/response logging
## Core REST Handlers
**Location**: `crates/kreuzberg/src/api/handlers.rs`
| Handler | Method | Description |
| --------------------- | ----------------- | ------------------------------------------------------------------------------------------------------ |
| `extract_handler` | POST /extract | Multipart upload: parse file + optional config JSON, check cache, call `extract_bytes()`, cache result |
| `extract_url_handler` | POST /extract-url | Fetch URL via reqwest, extract bytes |
| `batch_handler` | POST /batch | Parallel extraction with `Semaphore`-limited concurrency (default: CPU count) |
| `health_handler` | GET /health | Report status, version, uptime, feature availability (OCR, embeddings), cache stats |
| `formats_handler` | GET /formats | Return supported format categories (office, pdf, images, web, email, archives, academic) |
| `cache_stats_handler` | GET /cache/stats | Hit/miss counts and hit rate |
| `cache_clear_handler` | DELETE /cache | Clear LRU cache |
## Caching Strategy
**Location**: `crates/kreuzberg/src/cache/mod.rs`
LRU cache keyed by `SHA256(file_content)`, stores `Arc<ExtractionResult>`. Default 1000 entries. Thread-safe via `RwLock`. Tracks hit/miss counters with `AtomicU64` for stats endpoint.
## Error Handling
**Location**: `crates/kreuzberg/src/api/error.rs`
`ApiError` enum maps to HTTP status codes:
- `MissingFile` -> 400, `FileNotFound` -> 404
- `OnnxRuntimeMissing` / `TesseractMissing` -> 503 (with remediation message)
- `PayloadTooLarge` -> 413
- `ExtractionFailed` / `InvalidConfig` / `UnsupportedFormat` -> 500
## MCP Server Implementation
**Location**: `crates/kreuzberg/src/mcp/server.rs`
The MCP server allows Claude and other AI agents to call Kreuzberg extraction functions through the Model Context Protocol.
### MCP Tools (Callable Functions)
Three tools are registered:
| Tool | Purpose | Required Params |
| ------------------ | --------------------------------------------------------- | --------------- |
| `extract_file` | Extract text/tables/metadata from documents (75+ formats) | `file_path` |
| `batch_extract` | Extract from multiple documents in parallel | `file_paths[]` |
| `get_capabilities` | List supported formats, features, backends | (none) |
**Tool registration pattern** (example: `extract_file`):
```rust
// Define Tool with name, description, JSON Schema inputSchema
// Register with server.register_tool(tool, handler_fn)
// Handler: parse params -> build ExtractionConfig -> call extract_file() -> return ToolResult as JSON
```
`extract_file` optional params: `format`, `extract_tables`, `extract_images`, `ocr_enabled`, `extract_metadata`, `chunking_preset`, `generate_embeddings`.
### MCP Resources (Static Knowledge)
Three resources provide static information to agents:
- `kreuzberg://formats` -- Supported format list as JSON
- `kreuzberg://features` -- Cross-binding feature matrix (from `FEATURE_MATRIX.md`)
- `kreuzberg://api-reference` -- Generated API documentation
### MCP Prompts (Agent Templates)
Two prompts guide agent extraction workflows:
- `extract_for_rag` -- Document type-specific RAG extraction guidance (research paper, contract, report). Recommends chunking preset and embedding config.
- `batch_document_processing` -- Optimal concurrency, grouping, and error handling for batch workflows.
### MCP Transport Protocols
- **HTTP/REST**: MCP routes mounted alongside REST API on separate `/mcp/` prefix
- **Stdio**: JSON-RPC 2.0 over stdin/stdout for local CLI integration (e.g., Claude Desktop)
### Integration with Claude Desktop
```json
{
"mcpServers": {
"kreuzberg": {
"command": "kreuzberg-mcp",
"env": {
"KREUZBERG_API_BASE": "http://localhost:8000",
"KREUZBERG_MCP_TRANSPORT": "stdio"
}
}
}
}
```
### MCP Error Handling
`ToolError` variants: `FileNotFound`, `UnsupportedFormat`, `ExtractionFailed`, `OnnxRuntimeMissing`, `TesseractMissing`, `Timeout`. Each maps to an MCP `ToolResultError` with descriptive code and message.
## Environment Configuration
See `.env.example` for all configurable variables. Key categories:
- **Server**: `KREUZBERG_HOST`, `KREUZBERG_PORT`
- **Size limits**: `KREUZBERG_MAX_REQUEST_BODY_BYTES` (default 100MB), `KREUZBERG_MAX_MULTIPART_FIELD_BYTES`
- **Features**: `KREUZBERG_ENABLE_OCR`, `KREUZBERG_ENABLE_EMBEDDINGS`, `KREUZBERG_ENABLE_KEYWORDS`
- **Cache**: `KREUZBERG_CACHE_ENABLED`, `KREUZBERG_CACHE_SIZE`
- **CORS**: `CORS_ALLOWED_ORIGINS` (comma-separated)
- **MCP**: `KREUZBERG_MCP_HOST`, `KREUZBERG_MCP_PORT`, `KREUZBERG_MCP_TRANSPORT` (stdio/http)
- **Logging**: `RUST_LOG=kreuzberg=info,tower_http=debug`
## Critical Rules
### REST API Rules
1. **Always validate multipart file uploads** - Check MIME type, size, magic bytes
2. **Timeout long-running extractions** - Set per-handler timeout (5 min default)
3. **Stream large files** - Never buffer entire multi-GB file in memory
4. **Cache aggressively** - Identical files should return from cache in <1ms
5. **Parallel extraction is CPU-bound** - Limit workers to CPU count + 1
6. **Error responses must be actionable** - Include error code and remediation suggestion
7. **Health checks must verify features** - Report missing dependencies (ONNX, Tesseract)
8. **Size limits are configurable** - Allow override via env var for large deployments
9. **CORS is permissive by default** - Restrict in production via env var
10. **Logging all requests** - Track extraction metrics for observability
### MCP Rules
1. **All tools must have timeout** - Prevent hanging on large files (default 5 min)
2. **Error responses must be detailed** - Include suggestions for missing dependencies
3. **Feature gates must be checked** - Return helpful message if feature unavailable (embeddings, OCR)
4. **Resources should be static** - Don't query external services in resource handlers
5. **Prompts guide agents** - Provide clear examples and best practices
6. **Batch tools must support cancellation** - Allow agent to stop long-running batch operations
7. **Logging all tool calls** - Track usage for analytics and debugging
## Related Skills
- **extraction-pipeline-patterns** - Core extraction called by handlers and MCP tools
- **chunking-embeddings** - Optional chunking/embedding parameters in extraction
- **ocr-backend-management** - OCR engine selection and image preprocessing

120
.ai-rulez/skills/chunking-embeddings/SKILL.md Normal file
View File

@@ -0,0 +1,120 @@
---
description: "Chunking, embeddings, and RAG pipeline integration"
name: chunking-embeddings
priority: critical
---
# Chunking & Embeddings
**Text splitting strategies, embedding generation with FastEmbed, RAG pipeline integration**
## Chunking Architecture Overview
**Location**: `crates/kreuzberg/src/chunking/`, `crates/kreuzberg/src/embeddings.rs`
```text
Extracted Text
|
[1. Normalization] -> Clean whitespace, remove control chars
|
[2. Chunk Strategy Selection] -> Fixed-size, semantic, syntax-aware, recursive
|
[3. Overlap Management] -> Control context window overlap
|
[4. Optional Embedding] -> Generate vectors with FastEmbed
|
Output: Vec<Chunk> with text, vectors, metadata
```
## Chunking Strategies
**Location**: `crates/kreuzberg/src/chunking/mod.rs`
| Strategy | Pattern | Best For |
| --------------------------------- | ------------------------------------------------------- | ------------------------------------------------------------------ |
| **Fixed-Size** | Sliding window with configurable overlap | Uniform chunks for embedding models with fixed token limits |
| **Semantic** | Split by sentences, merge/split by similarity threshold | Smart context preservation for LLM consumption and semantic search |
| **Syntax-Aware** | Split by paragraph/section/heading/code-block structure | Preserving document structure (sections, code blocks) in RAG |
| **Recursive** (LangChain pattern) | Try separators in order: `\n\n`, `\n`, `,` | Best general-purpose chunking; auto-finds optimal split points |
Key config fields per strategy (see struct definitions in `chunking/mod.rs`):
- Fixed-Size: `chunk_size`, `overlap`, `trim_whitespace`
- Semantic: `target_chunk_size`, `min/max_chunk_size`, `semantic_threshold`, `use_sentence_boundaries`
- Syntax-Aware: `chunk_by` (Paragraph/Section/Heading/Sentence/CodeBlock), `max_chunk_size`, `respect_code_blocks`
- Recursive: `separators[]`, `chunk_size`, `overlap`
## Chunking Configuration Presets
**Location**: `crates/kreuzberg/src/chunking/mod.rs`
| Preset | Chunk Size | Overlap | Strategy | Use Case |
| ------------ | ----------- | ------- | ---------- | ---------------------- |
| **Balanced** | 512 tokens | 50 | Semantic | RAG sweet spot |
| **Compact** | 256 tokens | 32 | Fixed-Size | Dense vectors |
| **Extended** | 1024 tokens | 100 | Recursive | Full context |
| **Minimal** | 128 tokens | 16 | (default) | Lightweight embeddings |
Usage: set `config.chunking.preset = Some("balanced")` in `ExtractionConfig`.
## Embedding Generation with FastEmbed
**Location**: `crates/kreuzberg/src/embeddings.rs`
### Model Selection
| Model | Dimensions | Notes |
| ----------------------------------- | ---------- | -------------------------------- |
| `BAAI/bge-small-en-v1.5` (default) | 384 | Fast, excellent for RAG |
| `BAAI/bge-small-zh-v1.5` | 384 | Chinese optimized |
| `BAAI/bge-base-en-v1.5` | 768 | Better quality, slower |
| `jinaai/jina-embeddings-v2-base-en` | 768 | Long context (up to 8192 tokens) |
| `Custom(path)` | varies | Custom ONNX model path |
### Embedding Pattern
`TextEmbeddingManager` provides singleton-cached models per config. Pattern:
1. `get_or_init_model()` -- lazy-loads ONNX model (downloads if needed), caches in `Arc<RwLock<HashMap>>`
2. `embed_chunks()` -- collects chunk texts, calls `model.embed(texts, batch_size)`, zips results back to `ChunkWithEmbedding`
Default config: `batch_size=256`, `device=CPU`, `parallel_requests=4`.
### ONNX Runtime Requirement
Embeddings require ONNX Runtime. Feature-gated via:
```toml
[features]
embeddings = ["dep:fastembed", "dep:ort"]
```
Install: `brew install onnxruntime` (macOS) / `apt install libonnxruntime libonnxruntime-dev` (Linux). Verify: `echo $ORT_DYLIB_PATH`.
## RAG Integration Pattern
The full extraction-to-RAG pipeline:
1. **Extract**: `extract_file(path, config)` -> `ExtractionResult`
2. **Chunk**: Apply preset strategy to `result.content` -> `Vec<Chunk>`
3. **Embed**: If embedding config present, `TextEmbeddingManager::embed_chunks()` -> `Vec<ChunkWithEmbedding>`
4. **Output**: `RagDocument { file_path, metadata, chunks }` ready for vector DB ingestion
See `ChunkWithEmbedding` struct in `types.rs`: contains `text`, `embedding: Vec<f32>`, `dimensions`, `norm`, `metadata`.
## Critical Rules
1. **Chunking is preprocessing** - Always apply before embedding to ensure consistent vector sizes
2. **Overlap prevents information loss** - Set overlap to 15-20% of chunk size
3. **Embedding models are stateful** - Lazy load and cache to avoid repeated initialization
4. **ONNX Runtime is required** - Gracefully degrade if not available (skip embeddings)
5. **Batch embedding for performance** - Never embed single chunks; batch 50-1000 chunks
6. **Normalize embeddings for search** - Use L2 norm for cosine similarity
7. **Cache embedding results** - Don't re-embed identical text chunks
8. **Model selection impacts quality** - bge-small (384) for speed, bge-base (768) for quality
## Related Skills
- **extraction-pipeline-patterns** - Text extraction preceding chunking
- **api-server-mcp** - Endpoint for chunking + embedding operations
- **ocr-backend-management** - OCR text quality affects chunking success

126
.ai-rulez/skills/extraction-pipeline-patterns/SKILL.md Normal file
View File

@@ -0,0 +1,126 @@
---
description: "Document extraction pipeline architecture and patterns"
name: extraction-pipeline-patterns
priority: critical
---
# Extraction Pipeline Patterns
**Kreuzberg's format detection -> extraction -> fallback orchestration for 75+ file formats**
## Core Pipeline Architecture
The extraction pipeline (`crates/kreuzberg/src/core/pipeline.rs`, `crates/kreuzberg/src/extraction/`) orchestrates:
1. **Format Detection** - MIME type inference + extension validation -> select appropriate extractor
2. **Intelligent Extraction** - Route to format-specific extractors (PDF, DOCX, Excel, HTML, images, archives, etc.)
3. **Fallback Strategies** - Password-protected PDFs, OCR for images, nested archive handling, corrupted file recovery
4. **Post-Processing Pipeline** - Validators, quality processing, chunking, custom hooks (see `core/pipeline.rs`)
## Format Detection Strategy
**Location**: `crates/kreuzberg/src/core/mime.rs`, `crates/kreuzberg/src/core/formats.rs`
Pattern: detect via magic bytes, validate extension alignment (prevent spoofing), route to extractor. Multiple extractors for same format -> choose highest confidence/specificity.
```rust
// Pseudocode: core/mime.rs
match (magic_bytes(content), extension) {
(Some(fmt), Some(ext)) if aligned -> Ok(fmt),
(Some(fmt), Some(ext)) if misaligned -> Err(FormatMismatch),
(Some(fmt), None) -> Ok(fmt), // magic bytes only
(None, Some(ext)) -> Ok(from_extension(ext)),
_ -> Err(UnknownFormat),
}
```
## Extraction Modules (75 Formats)
| Category | Extractors | Key Modules |
| ------------ | ------------------------------------------------ | ---------------------------------------------------- |
| **Office** | DOCX, XLSX, XLSM, XLSB, XLS, PPTX, ODP, ODS | `extraction/{docx,excel,pptx}.rs` |
| **PDF** | Standard + encrypted, password attempts | `pdf/` subdirectory (13 files) |
| **Images** | PNG, JPG, TIFF, WebP, JP2, SVG (OCR-enabled) | `extraction/image.rs` + `ocr/` |
| **Web** | HTML, XHTML, XML, SVG (DOM parsing) | `extraction/html.rs` (67KB - complex table handling) |
| **Email** | EML, MSG (headers, body, attachments, threading) | `extraction/email.rs` |
| **Archives** | ZIP, TAR, GZ, 7Z (recursive extraction) | `extraction/archive.rs` (31KB) |
| **Markdown** | MD, TXT, RST, Org Mode, RTF | `extraction/markdown.rs` |
| **Academic** | LaTeX, BibTeX, JATS, Jupyter, DocBook | `extraction/{structured,xml}.rs` |
## Extraction Dispatcher
```rust
// Pseudocode: extraction/mod.rs
let format = detect_format(source.bytes, source.extension);
let result = match format {
Pdf -> extract_pdf(source, config),
Docx -> extract_docx(source, config),
Image -> extract_image_with_ocr_fallback(source, config),
Archive -> extract_archive_recursive(source, config),
_ -> extract_with_plugin(format, source, config),
};
run_pipeline(result, config) // post-processing always runs
```
## Fallback Strategies
- **Password-Protected PDFs**: Try primary password -> secondary password list -> return `is_encrypted=true` in metadata on failure
- **OCR Fallback**: If image text extraction confidence < threshold, trigger OCR backend; return both results with scores
- **Nested Archives**: Recursive extraction with configurable depth limit; flatten or preserve hierarchy
- **Corrupted File Recovery**: Stream-based parsing, emit content up to error point, include error location in metadata
## Configuration Integration
**Location**: `crates/kreuzberg/src/core/config.rs`, `crates/kreuzberg/src/core/config_validation.rs`
`ExtractionConfig` holds format-specific configs (`pdf`, `image`, `html`, `office`), fallback orchestration (`fallback`), and post-processing (`postprocessor`, `chunking`, `keywords`). See struct definition in `config.rs`.
## Plugin System Integration
**Location**: `crates/kreuzberg/src/plugins/`
- **CustomExtractor**: Override built-in format extractors
- **PostProcessor**: Modify results after extraction (Early/Middle/Late stages)
- **Validator**: Fail-fast validation (e.g., minimum text length)
- **OCRBackend**: Swap OCR engine
Plugin registry loaded at startup, cached for zero-cost lookup.
## Feature Flag Strategy
**Location**: `Cargo.toml` (workspace), `crates/kreuzberg/Cargo.toml`, `FEATURE_MATRIX.md`
20+ features across 9 language bindings. Key feature groups:
| Group | Features | Notes |
| -------- | ------------------------------------------------------------------------------------ | --------------------------------- |
| OCR | `tesseract` (default), `tesseract-static`, `ocr-minimal` | Mutually exclusive recommendation |
| Formats | `pdf`, `pdf-minimal`, `office`, `office-minimal` | |
| AI/ML | `embeddings` (requires ONNX), `keywords-yake`, `keywords-rake`, `language-detection` | |
| Server | `api` (Axum), `mcp`, `tokio-runtime`, `lite-runtime` | |
| Bindings | `python-bindings`, `ruby-bindings`, `php-bindings`, `node-bindings`, `wasm` | |
Conditional compilation: modules gated with `#[cfg(feature = "...")]`. Runtime `validate_config()` warns if requested feature not compiled in.
### Feature Flag Critical Rules
1. **Never mix conflicting features** - e.g., `ocr-minimal` + `tesseract` should error at compile time
2. **Always provide feature diagnostics** - Config validation must warn if feature unavailable
3. **Default to maximum feature set** - Unless embedded/minimal specifically requested
4. **Test all feature combinations** - Matrix testing in CI catches regressions
5. **WASM incompatible** with embeddings, keywords, OCR
## Critical Rules
1. **Always use format detection** before routing to extractors (prevent confusion attacks)
2. **Stream-based parsing** for PDFs/archives to handle multi-GB files
3. **Post-pipeline is mandatory**: All extraction results flow through `run_pipeline()` for validators/hooks
4. **Plugin overrides are order-dependent**: Plugins registered first take priority
5. **Fallback timeouts**: Set reasonable OCR/archive extraction timeouts (config-driven)
6. **Metadata preservation**: Include format detection confidence, extraction method used, any fallbacks applied
## Related Skills
- **ocr-backend-management** - OCR engine selection and image preprocessing
- **chunking-embeddings** - Post-extraction text splitting with FastEmbed
- **api-server-mcp** - Axum endpoint for extraction pipeline exposure and MCP server

78
.ai-rulez/skills/format-specific-extraction/SKILL.md Normal file
View File

@@ -0,0 +1,78 @@
---
name: format-specific-extraction
description: "Format-specific document extraction workflows"
priority: high
---
# Format-Specific Extraction Workflows
## Office XML (DOCX/PPTX/ODT)
```text
ZIP archive → Security validation → XML parsing → Text + tables + metadata
```
1. `ZipBombValidator::new(limits).validate(&mut archive)?`
2. Extract XML files from archive (`word/document.xml`, `ppt/slides/*.xml`, `content.xml`)
3. Parse with `quick-xml::Reader` (streaming) + `DepthValidator` + `StringGrowthValidator`
4. Extract metadata via `crate::extraction::office_metadata::extract_metadata()`
5. See: `extractors/docx.rs`, `extractors/pptx.rs`, `extractors/odt.rs`
## PDF
```text
Bytes → pdf_oxide → Per-page text + OCR fallback → Tables → Metadata
```
1. `pdf_oxide::PdfDocument::from_bytes(content)?`
2. Check if needs OCR: `config.force_ocr || !has_searchable_text()`
3. Extract text per page, tables if `config.pages` enabled
4. Feature-gated: `#[cfg(feature = "pdf")]`
5. See: `extractors/pdf/mod.rs`
## Archives (ZIP/TAR/7z/GZIP)
```text
Validate → Extract metadata → Extract plaintext files only
```
1. `ZipBombValidator` BEFORE any extraction
2. Extract metadata (file list, sizes)
3. Extract text content from plaintext files
4. Use `build_archive_result()` helper
5. See: `extractors/archive.rs`, `extraction/archive/*.rs`
## Structured Text (JSON/YAML/TOML/XML)
```text
Detect format from MIME → Parse → Pretty-print → Metadata
```
Single `StructuredExtractor` handles multiple MIME types. Parse with format-specific library, pretty-print to text.
See: `extractors/structured.rs`
## Email (EML/MSG)
```text
Parse headers → Extract body (text/html) → Process attachments
```
See: `extraction/email.rs`, `extractors/email.rs`
## Common Helpers
| Helper | Location | Purpose |
| ------------------------------------- | --------------------------- | ------------------------------ |
| `office_metadata::extract_metadata()` | `extraction/office.rs` | Office XML metadata |
| `cells_to_markdown()` | `extraction/mod.rs` | Convert cell grid to GFM table |
| `build_archive_result()` | `extraction/archive/mod.rs` | Standard archive result |
## Adding a New Format
1. Add MIME type to `EXT_TO_MIME` in `core/mime.rs`
2. Create extractor implementing `DocumentExtractor` trait
3. Set `supported_mime_types()` and `priority()` (default: 50)
4. Register in `extractors/mod.rs` → `register_default_extractors()`
5. Feature-gate if optional: `#[cfg(feature = "my-format")]`
6. Apply security validators for user content
7. Add tests with fixture files

97
.ai-rulez/skills/plugin-architecture-patterns/SKILL.md Normal file
View File

@@ -0,0 +1,97 @@
---
name: plugin-architecture-patterns
description: "Plugin architecture, registration, and trait patterns"
priority: critical
---
# Plugin Architecture & Registration
## Plugin Types
| Type | Trait | Location |
| ------------------ | --------------------------- | ---------------------------- |
| Document Extractor | `DocumentExtractor: Plugin` | `plugins/extractor/trait.rs` |
| OCR Backend | `OcrBackend: Plugin` | `plugins/ocr/trait.rs` |
| Post Processor | `PostProcessor: Plugin` | `plugins/processor/trait.rs` |
| Validator | `Validator: Plugin` | `plugins/validator/trait.rs` |
## DocumentExtractor Implementation
```rust
use crate::plugins::{DocumentExtractor, Plugin};
use async_trait::async_trait;
pub struct MyExtractor;
impl Plugin for MyExtractor {
fn name(&self) -> &str { "my-extractor" }
fn version(&self) -> String { env!("CARGO_PKG_VERSION").to_string() }
}
#[async_trait]
impl DocumentExtractor for MyExtractor {
async fn extract_bytes(&self, content: &[u8], mime_type: &str, config: &ExtractionConfig)
-> Result<ExtractionResult> { /* ... */ }
fn supported_mime_types(&self) -> &[&str] { &["application/x-custom"] }
fn priority(&self) -> i32 { 50 }
// WASM support (optional)
fn as_sync_extractor(&self) -> Option<&dyn SyncExtractor> { None }
}
```
## Priority System
| Range | Use |
| ------ | ------------------------- |
| 0-25 | Fallback/low-quality |
| 26-49 | Alternative extractors |
| **50** | **Default (built-in)** |
| 51-75 | Premium/enhanced |
| 76-100 | Specialized/high-priority |
Registry selects **highest priority** extractor for each MIME type. Override built-ins with priority > 50.
## Registration
```rust
// In extractors/mod.rs → register_default_extractors()
let registry = get_document_extractor_registry();
let mut registry = registry.write()
.map_err(|e| KreuzbergError::Other(format!("Registry lock poisoned: {}", e)))?;
registry.register(Arc::new(MyExtractor::new()))?;
```
## Feature-Gated Registration
```rust
#[cfg(feature = "office")]
{
registry.register(Arc::new(DocxExtractor::new()))?;
registry.register(Arc::new(PptxExtractor::new()))?;
}
```
## PostProcessor Pattern
```rust
impl PostProcessor for MyProcessor {
async fn process(&self, result: &mut ExtractionResult, config: &ExtractionConfig)
-> Result<()> {
result.content = process_content(&result.content);
Ok(())
}
fn stage(&self) -> ProcessorStage { ProcessorStage::Middle }
}
```
Stages: `Early` → `Middle` → `Late`. Failures isolated (don't block others).
## Critical Rules
1. All plugins **MUST be `Send + Sync`**
2. Feature gate with `#[cfg(feature = "...")]` for optional formats
3. Use `#[async_trait]` for `DocumentExtractor`
4. Initialization via `ensure_initialized()` (lazy, called before first extraction)
5. Plugin names: kebab-case (e.g., `"pdf-extractor"`)

34
.cargo/config.toml Normal file
View File

@@ -0,0 +1,34 @@
# This file is auto-generated by alef. DO NOT EDIT.
# alef:hash:4e15143f4af1ae8bafbdb1506ef057da924484c66a19483966333558ad437e75
# Re-generate with: alef scaffold
[build]
incremental = true
jobs = 4
[net]
git-fetch-with-cli = true
[registries.crates-io]
protocol = "sparse"
# Required for PyO3 / ext-php-rs cdylibs: Python and Zend C-API symbols are
# resolved at runtime when the host loads the extension, not at link time.
# macOS ld is strict and rejects unresolved symbols by default.
[target.'cfg(target_os = "macos")']
rustflags = ["-C", "link-arg=-Wl,-undefined,dynamic_lookup"]
[target.x86_64-pc-windows-msvc]
linker = "rust-lld"
[target.i686-pc-windows-msvc]
linker = "rust-lld"
[target.aarch64-unknown-linux-gnu]
linker = "aarch64-linux-gnu-gcc"
[target.x86_64-unknown-linux-musl]
linker = "musl-gcc"
[target.wasm32-unknown-unknown]
rustflags = ["-C", "target-feature=+bulk-memory", "--cfg", "getrandom_backend=\"wasm_js\""]

8
.clang-format Normal file
View File

@@ -0,0 +1,8 @@
---
BasedOnStyle: LLVM
IndentWidth: 4
ColumnLimit: 100
BreakBeforeBraces: Attach
AllowShortFunctionsOnASingleLine: Empty
AllowShortIfStatementsOnASingleLine: false
SortIncludes: true

32
.dockerignore Normal file
View File

@@ -0,0 +1,32 @@
# =============================================================================
# .dockerignore — Docker builds only need Cargo.toml, Cargo.lock, and crates/
# Exclude everything else to minimize context and speed up builds.
# =============================================================================
# Start by ignoring everything
*
# Allow only what Docker builds need
!Cargo.toml
!Cargo.lock
!crates/kreuzberg/
!crates/kreuzberg-cli/
!crates/kreuzberg-tesseract/
!crates/kreuzberg-paddle-ocr/
!crates/kreuzberg-ffi/
!crates/kreuzberg-py/
!crates/kreuzberg-node/
!crates/kreuzberg-wasm/
!packages/elixir/native/kreuzberg_nif/
!docker/
# Re-exclude build artifacts inside allowed dirs
**/target/
**/.git/
**/node_modules/
**/__pycache__/
**/*.pyc
**/*.so
**/*.dylib
**/*.dll
**/*.node

182
.editorconfig Normal file
View File

@@ -0,0 +1,182 @@
# EditorConfig is awesome: https://EditorConfig.org
# top-most EditorConfig file
root = true
# All files
[*]
charset = utf-8
insert_final_newline = true
trim_trailing_whitespace = true
end_of_line = lf
# Code files
[*.{cs,go,rs,py,js,ts,tsx,jsx,php,rb}]
indent_style = space
# C# files
[*.cs]
indent_size = 4
# Organize usings
dotnet_sort_system_directives_first = true
dotnet_separate_import_directive_groups = false
# this. and Me. preferences
dotnet_style_qualification_for_field = false:warning
dotnet_style_qualification_for_property = false:warning
dotnet_style_qualification_for_method = false:warning
dotnet_style_qualification_for_event = false:warning
# Language keywords vs BCL types preferences
dotnet_style_predefined_type_for_locals_parameters_members = true:warning
dotnet_style_predefined_type_for_member_access = true:warning
# Parentheses preferences
dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:suggestion
dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:suggestion
dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:suggestion
dotnet_style_parentheses_in_other_operators = never_if_unnecessary:suggestion
# Modifier preferences
dotnet_style_require_accessibility_modifiers = always:warning
dotnet_style_readonly_field = true:warning
csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:suggestion
# Expression-level preferences
dotnet_style_object_initializer = true:suggestion
dotnet_style_collection_initializer = true:suggestion
dotnet_style_explicit_tuple_names = true:warning
dotnet_style_prefer_inferred_tuple_names = true:suggestion
dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion
dotnet_style_prefer_auto_properties = true:suggestion
dotnet_style_prefer_conditional_expression_over_assignment = true:silent
dotnet_style_prefer_conditional_expression_over_return = true:silent
dotnet_style_prefer_compound_assignment = true:suggestion
dotnet_style_prefer_simplified_interpolation = true:suggestion
dotnet_style_prefer_simplified_boolean_expressions = true:suggestion
# Null-checking preferences
dotnet_style_coalesce_expression = true:warning
dotnet_style_null_propagation = true:warning
dotnet_style_prefer_is_null_check_over_reference_equality_method = true:warning
# C# Code Style Rules
# var preferences
csharp_style_var_for_built_in_types = true:suggestion
csharp_style_var_when_type_is_apparent = true:suggestion
csharp_style_var_elsewhere = true:suggestion
# Expression-bodied members
csharp_style_expression_bodied_methods = when_on_single_line:suggestion
csharp_style_expression_bodied_constructors = false:silent
csharp_style_expression_bodied_operators = when_on_single_line:suggestion
csharp_style_expression_bodied_properties = when_on_single_line:suggestion
csharp_style_expression_bodied_indexers = when_on_single_line:suggestion
csharp_style_expression_bodied_accessors = when_on_single_line:suggestion
csharp_style_expression_bodied_lambdas = when_on_single_line:suggestion
csharp_style_expression_bodied_local_functions = when_on_single_line:suggestion
# Pattern matching preferences
csharp_style_pattern_matching_over_is_with_cast_check = true:warning
csharp_style_pattern_matching_over_as_with_null_check = true:warning
csharp_style_prefer_switch_expression = true:suggestion
csharp_style_prefer_pattern_matching = true:suggestion
csharp_style_prefer_not_pattern = true:suggestion
# Null-checking preferences
csharp_style_throw_expression = true:suggestion
csharp_style_conditional_delegate_call = true:warning
# Code block preferences
csharp_prefer_braces = true:warning
csharp_prefer_simple_using_statement = true:suggestion
# Expression preferences
csharp_prefer_simple_default_expression = true:suggestion
csharp_style_pattern_local_over_anonymous_function = true:suggestion
csharp_style_inlined_variable_declaration = true:suggestion
csharp_style_deconstructed_variable_declaration = true:suggestion
csharp_style_prefer_index_operator = true:suggestion
csharp_style_prefer_range_operator = true:suggestion
csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion
# C# Formatting Rules
# New line preferences
csharp_new_line_before_open_brace = all
csharp_new_line_before_else = true
csharp_new_line_before_catch = true
csharp_new_line_before_finally = true
csharp_new_line_before_members_in_object_initializers = true
csharp_new_line_before_members_in_anonymous_types = true
csharp_new_line_between_query_expression_clauses = true
# Indentation preferences
csharp_indent_case_contents = true
csharp_indent_switch_labels = true
csharp_indent_labels = no_change
csharp_indent_block_contents = true
csharp_indent_braces = false
csharp_indent_case_contents_when_block = false
# Space preferences
csharp_space_after_cast = false
csharp_space_after_keywords_in_control_flow_statements = true
csharp_space_between_parentheses = false
csharp_space_before_colon_in_inheritance_clause = true
csharp_space_after_colon_in_inheritance_clause = true
csharp_space_around_binary_operators = before_and_after
csharp_space_between_method_declaration_parameter_list_parentheses = false
csharp_space_between_method_declaration_empty_parameter_list_parentheses = false
csharp_space_between_method_declaration_name_and_open_parenthesis = false
csharp_space_between_method_call_parameter_list_parentheses = false
csharp_space_between_method_call_empty_parameter_list_parentheses = false
csharp_space_between_method_call_name_and_opening_parenthesis = false
csharp_space_after_comma = true
csharp_space_after_dot = false
csharp_space_after_semicolon_in_for_statement = true
csharp_space_before_semicolon_in_for_statement = false
csharp_space_around_declaration_statements = false
csharp_space_before_open_square_brackets = false
csharp_space_between_empty_square_brackets = false
csharp_space_between_square_brackets = false
# Wrap preferences
csharp_preserve_single_line_statements = false
csharp_preserve_single_line_blocks = true
# Using directive preferences
csharp_using_directive_placement = outside_namespace:warning
# Go files
[*.go]
indent_style = tab
indent_size = 4
# Rust files
[*.rs]
indent_size = 4
# Python files
[*.py]
indent_size = 4
# JavaScript/TypeScript files
[*.{js,ts,tsx,jsx}]
indent_size = 2
# Ruby files
[*.rb]
indent_size = 2
# PHP files
[*.php]
indent_size = 4
# YAML files
[*.{yml,yaml}]
indent_size = 2
# Markdown files
[*.md]
trim_trailing_whitespace = false

37
.gh-actions-updater.toml Normal file
View File

@@ -0,0 +1,37 @@
[scan]
include = [
".github/workflows/**/*.yml",
".github/workflows/**/*.yaml",
".github/actions/**/action.yml",
".github/actions/**/action.yaml",
"action.yml",
"action.yaml",
]
exclude = []
recursive = false
[cache]
enabled = true
ttl = "6h"
[update]
mode = "latest-tag"
# erlef/setup-beam: publisher exposes the major-minor float tag (e.g. v1.24)
# but its `dist/index.js` ships only on the patch tag (v1.24.0). The float
# resolves to a commit with no bundled JS, so the action exits in <1s with
# `failure` and no log message. Pin to patch tags only.
exclude = ["erlef/setup-beam"]
include_prereleases = false
preserve_major = true
missing_ref = "warn"
[output]
format = "human"
color = "auto"
[github]
api_url = "https://api.github.com"
[performance]
# Omit threads to use Rayon\'s available-core default.
# threads = 8

20
.gitattributes vendored Normal file
View File

@@ -0,0 +1,20 @@
# Generated by alef scaffold.
crates/kreuzberg-ffi/** linguist-generated=true
crates/kreuzberg-node/** linguist-generated=true
crates/kreuzberg-php/** linguist-generated=true
crates/kreuzberg-php/src/** linguist-generated=true
crates/kreuzberg-py/** linguist-generated=true
crates/kreuzberg-wasm/** linguist-generated=true
e2e/** linguist-generated=true
packages/csharp/src/** linguist-generated=true
packages/dart/** linguist-generated=true
packages/elixir/** linguist-generated=true
packages/go/v5/** linguist-generated=true
packages/java/** linguist-generated=true
packages/kotlin-android/** linguist-generated=true
packages/python/** linguist-generated=true
packages/r/** linguist-generated=true
packages/ruby/** linguist-generated=true
packages/swift/** linguist-generated=true
packages/zig/** linguist-generated=true
test_apps/** linguist-generated=true

10
.github/CODEOWNERS vendored Normal file
View File

@@ -0,0 +1,10 @@
# Default owner — everything
* @Goldziher
# Zensical config and documentation
/zensical.toml @Goldziher @pratik-mahalle @v-tan
/docs/ @Goldziher @pratik-mahalle @v-tan
*.md @Goldziher @pratik-mahalle @v-tan
# Rust crates
/crates/ @Goldziher @kh3rld

28
.github/ISSUE_TEMPLATE/bug_report.yml vendored Normal file
View File

@@ -0,0 +1,28 @@
name: Bug Report
description: Report a bug or unexpected behavior
title: "bug: "
labels: ["bug"]
projects: ["kreuzberg-dev/1"]
body:
- type: textarea
id: description
attributes:
label: Description
description: What happened? What did you expect to happen?
validations:
required: true
- type: textarea
id: steps-to-reproduce
attributes:
label: Steps to reproduce
description: Minimal steps to reproduce the issue.
validations:
required: true
- type: textarea
id: reproduction-files
attributes:
label: Relevant files and configuration
description: >-
Any configuration files, input files, or code snippets needed to
reproduce the issue.
render: text

1
.github/ISSUE_TEMPLATE/config.yml vendored Normal file
View File

@@ -0,0 +1 @@
blank_issues_enabled: true

20
.github/ISSUE_TEMPLATE/documentation.yml vendored Normal file
View File

@@ -0,0 +1,20 @@
name: Documentation Issue
description: Report missing, unclear, or incorrect documentation
title: "docs: "
labels: ["documentation"]
projects: ["kreuzberg-dev/1"]
body:
- type: textarea
id: what
attributes:
label: What
description: What documentation is missing, unclear, or incorrect?
validations:
required: true
- type: textarea
id: why
attributes:
label: Why
description: Why does this need to change?
validations:
required: true

18
.github/ISSUE_TEMPLATE/feature_request.yml vendored Normal file
View File

@@ -0,0 +1,18 @@
name: Feature Request
description: Suggest a new feature or improvement
title: "feat: "
labels: ["enhancement"]
projects: ["kreuzberg-dev/1"]
body:
- type: textarea
id: what
attributes:
label: What is the proposed feature?
validations:
required: true
- type: textarea
id: why
attributes:
label: Why would this be a good addition?
validations:
required: true

12
.github/PULL_REQUEST_TEMPLATE.md vendored Normal file
View File

@@ -0,0 +1,12 @@
## Related
<!-- Link issues or discussions if applicable -->
## Description
<!-- What does this PR do? -->
## Checklist
- [ ] CI passing
- [ ] Tests added where applicable

9
.github/actionlint.yaml vendored Normal file
View File

@@ -0,0 +1,9 @@
self-hosted-runner:
labels:
- runner-small
- runner-medium
- runner-medium-arm64
- runner-large
- runner-large-spot
- runner-medium-arm64-spot
- runner-gpu-l4

313
.github/actions/cache-benchmark-harness/action.yml vendored Normal file
View File

@@ -0,0 +1,313 @@
name: Cache Benchmark Harness Binary
description: >
Build and cache the benchmark-harness binary with intelligent caching based on source hashes.
Generates cache keys based on harness source + kreuzberg dependency + Cargo files,
restores from cache if available, builds if needed, and saves to cache.
Validates artifacts after restore or build to ensure integrity.
inputs:
cache-version:
description: "Manual version for cache invalidation"
required: false
default: "v1"
build-profile:
description: "Build profile (release, debug)"
required: false
default: "release"
outputs:
cache-hit:
description: "Boolean indicating exact cache hit"
value: ${{ steps.cache-restore.outputs.cache-hit }}
cache-key:
description: "The cache key used"
value: ${{ steps.generate-cache-key.outputs.cache-key }}
binary-path:
description: "Path to the built/cached benchmark-harness binary"
value: ${{ steps.validate-binary.outputs.binary-path }}
runs:
using: composite
steps:
# Validate inputs
- name: Validate inputs
shell: bash
env:
BUILD_PROFILE: ${{ inputs.build-profile }}
run: |
set -euo pipefail
# Validate build profile
valid_profiles=("release" "debug")
if [[ ! " ${valid_profiles[@]} " =~ " ${BUILD_PROFILE} " ]]; then
echo "❌ Error: build-profile must be one of: ${valid_profiles[*]}"
exit 1
fi
echo "✓ Validation passed"
echo " Build profile: $BUILD_PROFILE"
echo " Cache version: ${{ inputs.cache-version }}"
# Compute hash for benchmark-harness sources
- name: Compute benchmark-harness source hash
id: harness-hash
shell: bash
run: |
set -euo pipefail
echo "=== Computing Benchmark Harness Source Hash ==="
# Compute hash for harness source files and Cargo.toml
HARNESS_HASH=$(scripts/ci/cache/compute-hash.sh \
"tools/benchmark-harness/src/**" \
"tools/benchmark-harness/Cargo.toml" \
2>&1 | grep "^[a-f0-9]*$")
if [[ -z "$HARNESS_HASH" ]]; then
echo "❌ Failed to compute harness source hash"
exit 1
fi
echo "harness-hash=$HARNESS_HASH" >> "$GITHUB_OUTPUT"
echo "✓ Harness source hash: $HARNESS_HASH"
# Compute hash for kreuzberg dependency
- name: Compute kreuzberg dependency hash
id: kreuzberg-hash
shell: bash
run: |
set -euo pipefail
echo "=== Computing Kreuzberg Dependency Hash ==="
# Compute hash for kreuzberg crate (dependency)
KREUZBERG_HASH=$(scripts/ci/cache/compute-hash.sh --dirs \
"crates/kreuzberg" \
2>&1 | grep "^[a-f0-9]*$")
if [[ -z "$KREUZBERG_HASH" ]]; then
echo "❌ Failed to compute kreuzberg dependency hash"
exit 1
fi
echo "kreuzberg-hash=$KREUZBERG_HASH" >> "$GITHUB_OUTPUT"
echo "✓ Kreuzberg dependency hash: $KREUZBERG_HASH"
# Compute hash for Cargo files
- name: Compute Cargo files hash
id: cargo-hash
shell: bash
run: |
set -euo pipefail
echo "=== Computing Cargo Files Hash ==="
# Compute hash for Cargo.lock
CARGO_HASH=$(scripts/ci/cache/compute-hash.sh --files Cargo.lock 2>&1 | grep "^[a-f0-9]*$")
if [[ -z "$CARGO_HASH" ]]; then
echo "❌ Failed to compute Cargo files hash"
exit 1
fi
echo "cargo-hash=$CARGO_HASH" >> "$GITHUB_OUTPUT"
echo "✓ Cargo files hash: $CARGO_HASH"
# Generate cache key
- name: Generate cache key
id: generate-cache-key
shell: bash
env:
BUILD_PROFILE: ${{ inputs.build-profile }}
HARNESS_HASH: ${{ steps.harness-hash.outputs.harness-hash }}
KREUZBERG_HASH: ${{ steps.kreuzberg-hash.outputs.kreuzberg-hash }}
CARGO_HASH: ${{ steps.cargo-hash.outputs.cargo-hash }}
CACHE_VERSION: ${{ inputs.cache-version }}
run: |
set -euo pipefail
echo "=== Cache Key Generated ==="
# Build cache key following format:
# harness-{profile}-{platform}-src-{harness-hash}-kreuzberg-{kreuzberg-hash}-cargo-{cargo-hash}-v{version}
CACHE_KEY="harness-${BUILD_PROFILE}-$(uname -m)-src-${HARNESS_HASH}-kreuzberg-${KREUZBERG_HASH}-cargo-${CARGO_HASH}-${CACHE_VERSION}"
echo "cache-key=$CACHE_KEY" >> "$GITHUB_OUTPUT"
echo "Full key: $CACHE_KEY"
echo ""
echo "Key components:"
echo " Profile: $BUILD_PROFILE"
echo " Platform: $(uname -m)"
echo " Harness hash: $HARNESS_HASH"
echo " Kreuzberg hash: $KREUZBERG_HASH"
echo " Cargo hash: $CARGO_HASH"
echo " Cache version: $CACHE_VERSION"
# Determine target path based on profile
- name: Determine target paths
id: target-paths
shell: bash
env:
BUILD_PROFILE: ${{ inputs.build-profile }}
run: |
set -euo pipefail
echo "=== Determining Target Paths ==="
case "$BUILD_PROFILE" in
release)
TARGET_DIR="target/release"
;;
debug)
TARGET_DIR="target/debug"
;;
*)
echo "❌ Invalid build profile: $BUILD_PROFILE"
exit 1
;;
esac
echo "target-dir=$TARGET_DIR" >> "$GITHUB_OUTPUT"
echo "✓ Target directory: $TARGET_DIR"
# Detect architecture for cache keys (shell expansion doesn't work in YAML with: context)
- name: Detect architecture
id: detect-arch
shell: bash
run: echo "arch=$(uname -m)" >> "$GITHUB_OUTPUT"
# Restore from cache
- name: Restore benchmark-harness binary from cache
id: cache-restore
uses: kreuzberg-dev/actions/cache-binding-artifact@v1
with:
binding-name: benchmark-harness
cache-key: ${{ steps.generate-cache-key.outputs.cache-key }}
cache-restore-keys: |
harness-${{ inputs.build-profile }}-${{ steps.detect-arch.outputs.arch }}-src-
harness-${{ inputs.build-profile }}-${{ steps.detect-arch.outputs.arch }}-
harness-${{ inputs.build-profile }}-
cache-paths: |
${{ steps.target-paths.outputs.target-dir }}/benchmark-harness
operation: restore
# Log cache hit status
- name: Log cache hit status
shell: bash
run: |
set -euo pipefail
if [[ "${{ steps.cache-restore.outputs.cache-hit }}" == "true" ]]; then
echo "✓ Cache HIT - benchmark-harness binary found in cache"
else
echo "✗ Cache MISS - Building benchmark-harness from source"
fi
# Build if cache miss
- name: Build benchmark-harness
id: build
if: steps.cache-restore.outputs.cache-hit != 'true'
shell: bash
env:
BUILD_PROFILE: ${{ inputs.build-profile }}
run: |
set -euo pipefail
echo "=== Building Benchmark Harness ==="
echo "Profile: $BUILD_PROFILE"
# Determine cargo build profile argument
case "$BUILD_PROFILE" in
release)
BUILD_ARG="--release"
;;
debug)
# Debug is default, no flag needed
BUILD_ARG=""
;;
*)
echo "❌ Invalid build profile: $BUILD_PROFILE"
exit 1
;;
esac
# Build benchmark-harness
echo "Running: cargo build --manifest-path tools/benchmark-harness/Cargo.toml $BUILD_ARG"
if ! cargo build --manifest-path tools/benchmark-harness/Cargo.toml $BUILD_ARG; then
echo "❌ Build failed for benchmark-harness"
exit 1
fi
echo "✓ Build succeeded"
# Validate binary exists and is executable
- name: Validate benchmark-harness binary
id: validate-binary
shell: bash
env:
BUILD_PROFILE: ${{ inputs.build-profile }}
TARGET_DIR: ${{ steps.target-paths.outputs.target-dir }}
run: |
set -euo pipefail
echo "=== Validating Benchmark Harness Binary ==="
BINARY_PATH="${TARGET_DIR}/benchmark-harness"
# Check if binary exists
if [[ ! -f "$BINARY_PATH" ]]; then
echo "❌ Binary not found at: $BINARY_PATH"
exit 1
fi
# Check if binary is executable
if [[ ! -x "$BINARY_PATH" ]]; then
echo "❌ Binary is not executable: $BINARY_PATH"
exit 1
fi
# Get binary size and info
BINARY_SIZE=$(ls -lh "$BINARY_PATH" | awk '{print $5}')
BINARY_PERMS=$(ls -l "$BINARY_PATH" | awk '{print $1}')
echo "binary-path=$BINARY_PATH" >> "$GITHUB_OUTPUT"
echo "✓ Binary validation passed"
echo " Path: $BINARY_PATH"
echo " Size: $BINARY_SIZE"
echo " Permissions: $BINARY_PERMS"
# Save to cache if build occurred
- name: Save benchmark-harness binary to cache
if: steps.cache-restore.outputs.cache-hit != 'true'
uses: kreuzberg-dev/actions/cache-binding-artifact@v1
with:
binding-name: benchmark-harness
cache-key: ${{ steps.generate-cache-key.outputs.cache-key }}
cache-paths: |
${{ steps.target-paths.outputs.target-dir }}/benchmark-harness
operation: save
# Summary
- name: Summary
if: always()
shell: bash
run: |
set -euo pipefail
echo ""
echo "=== Build and Cache Summary ==="
echo "Build Profile: ${{ inputs.build-profile }}"
echo "Platform: $(uname -m)"
echo "Cache Hit: ${{ steps.cache-restore.outputs.cache-hit == 'true' && 'Yes' || 'No' }}"
echo "Cache Key: ${{ steps.generate-cache-key.outputs.cache-key }}"
echo "Binary Path: ${{ steps.validate-binary.outputs.binary-path }}"
echo ""
echo "Hashes:"
echo " Harness: ${{ steps.harness-hash.outputs.harness-hash }}"
echo " Kreuzberg: ${{ steps.kreuzberg-hash.outputs.kreuzberg-hash }}"
echo " Cargo: ${{ steps.cargo-hash.outputs.cargo-hash }}"

105
.github/actions/install-system-deps/action.yml vendored Normal file
View File

@@ -0,0 +1,105 @@
name: Install System Dependencies
description: |
Install and cache platform-specific dependencies required for document conversion.
Includes: Tesseract OCR, fonts, and build tools.
Features robust caching with architecture/version awareness, timeout handling, and retry logic.
inputs:
enable-retry:
description: Enable retry logic with exponential backoff
required: false
default: "true"
runs:
using: composite
steps:
- name: Detect Tesseract version (macOS)
if: runner.os == 'macOS'
id: detect-tesseract-macos
shell: bash
run: scripts/ci/install-system-deps/detect-tesseract-macos.sh
- name: Cache Tesseract & tessdata (macOS)
if: runner.os == 'macOS'
id: cache-tesseract-macos
uses: actions/cache@v5
with:
path: |
/usr/local/opt/tesseract/
/usr/local/Cellar/tesseract/
/opt/homebrew/opt/tesseract/
/opt/homebrew/Cellar/tesseract/
key: tesseract-macos-${{ runner.arch }}-v5-${{ steps.detect-tesseract-macos.outputs.version }}
restore-keys: |
tesseract-macos-${{ runner.arch }}-v5-
tesseract-macos-${{ runner.arch }}-
- name: Install dependencies (macOS)
if: runner.os == 'macOS'
shell: bash
run: scripts/ci/install-system-deps/install-macos.sh
- name: Detect Tesseract version (Linux)
if: runner.os == 'Linux'
id: detect-tesseract-linux
shell: bash
run: scripts/ci/install-system-deps/detect-tesseract-linux.sh
- name: Cache Tesseract data (Linux)
if: runner.os == 'Linux'
id: cache-tesseract-linux
uses: actions/cache@v5
with:
path: |
/usr/share/tesseract-ocr/5/tessdata/
/usr/share/tesseract-ocr/tessdata/
key: tesseract-linux-${{ runner.arch }}-v5-${{ steps.detect-tesseract-linux.outputs.version }}
restore-keys: |
tesseract-linux-${{ runner.arch }}-v5-
tesseract-linux-${{ runner.arch }}-
- name: Install dependencies (Linux)
if: runner.os == 'Linux'
shell: bash
run: scripts/ci/install-system-deps/install-linux.sh
- name: Cache Tesseract (Windows)
if: runner.os == 'Windows'
id: cache-tesseract-windows
uses: actions/cache@v5
with:
path: |
C:\Program Files\Tesseract-OCR
C:\ProgramData\chocolatey\lib\tesseract
key: tesseract-windows-${{ runner.arch }}-v5-data
restore-keys: |
tesseract-windows-${{ runner.arch }}-
- name: Cache LLVM (Windows)
if: runner.os == 'Windows'
id: cache-llvm-windows
uses: actions/cache@v5
with:
path: |
C:\Program Files\LLVM
C:\ProgramData\chocolatey\lib\llvm
key: llvm-windows-${{ runner.arch }}-v1
- name: Cache CMake (Windows)
if: runner.os == 'Windows'
id: cache-cmake-windows
uses: actions/cache@v5
with:
path: |
C:\Program Files\CMake
C:\ProgramData\chocolatey\lib\cmake
key: cmake-windows-${{ runner.arch }}-v1
- name: Install dependencies (Windows)
if: runner.os == 'Windows'
shell: pwsh
env:
TESSERACT_CACHE_HIT: ${{ steps.cache-tesseract-windows.outputs.cache-hit }}
LLVM_CACHE_HIT: ${{ steps.cache-llvm-windows.outputs.cache-hit }}
CMAKE_CACHE_HIT: ${{ steps.cache-cmake-windows.outputs.cache-hit }}
run: pwsh -File scripts/ci/install-system-deps/install-windows.ps1

197
.github/actions/setup-layout-models/action.yml vendored Normal file
View File

@@ -0,0 +1,197 @@
name: Setup Layout Detection Models Cache
description: Download and cache layout detection ONNX models (RT-DETR + TATR) for CI testing
inputs:
cache-enabled:
description: Enable model caching (set to false for cross-arch builds)
required: false
default: "true"
models:
description: Comma-separated list of models to setup (rtdetr,tatr)
required: false
default: "rtdetr,tatr"
cache-key-suffix:
description: Suffix for cache key to differentiate model sets
required: false
default: "layout-models-v2"
outputs:
cache-hit:
description: Whether models were restored from cache (true/false)
value: ${{ steps.cache-models.outputs.cache-hit }}
cache-dir:
description: Path to the layout model cache directory
value: ${{ steps.set-outputs.outputs.cache-dir }}
models-available:
description: Comma-separated list of available models
value: ${{ steps.verify-models.outputs.available-models }}
runs:
using: composite
steps:
- name: Setup cache directory
shell: bash
run: |
mkdir -p ~/.cache/kreuzberg/layout
echo "Cache directory: $HOME/.cache/kreuzberg/layout"
- name: Restore layout models from cache
if: inputs.cache-enabled == 'true'
uses: actions/cache@v5
id: cache-models
with:
path: ~/.cache/kreuzberg/layout
key: ${{ inputs.cache-key-suffix }}-${{ runner.os }}-${{ runner.arch }}-rtdetr_3bf2fb0e+tatr_c11f4033
restore-keys: |
${{ inputs.cache-key-suffix }}-${{ runner.os }}-${{ runner.arch }}-
${{ inputs.cache-key-suffix }}-${{ runner.os }}-
${{ inputs.cache-key-suffix }}-
- name: Download RT-DETR model (rtdetr)
if: contains(inputs.models, 'rtdetr') && steps.cache-models.outputs.cache-hit != 'true'
shell: bash
run: |
MODEL_URL="https://huggingface.co/Kreuzberg/layout-models/resolve/main/rtdetr/model.onnx"
CACHE_DIR="$HOME/.cache/kreuzberg/layout"
MODEL_DIR="$CACHE_DIR/rtdetr"
MODEL_FILE="$MODEL_DIR/model.onnx"
echo "Downloading RT-DETR layout detection model from $MODEL_URL"
mkdir -p "$MODEL_DIR"
for attempt in 1 2 3; do
if [ $attempt -gt 1 ]; then
backoff=$((5 * 3 ** (attempt - 2)))
echo "Retry attempt $attempt/3 after ${backoff}s backoff..."
sleep $backoff
fi
if curl -f -L --progress-bar --connect-timeout 30 --max-time 600 \
-o "$MODEL_FILE" "$MODEL_URL"; then
echo "RT-DETR model downloaded successfully ($(du -h "$MODEL_FILE" | cut -f1))"
break
fi
done
if [ ! -f "$MODEL_FILE" ]; then
echo "ERROR: Failed to download RT-DETR model after 3 attempts"
exit 1
fi
- name: Verify RT-DETR SHA256
if: contains(inputs.models, 'rtdetr') && steps.cache-models.outputs.cache-hit != 'true'
shell: bash
run: |
MODEL_FILE="$HOME/.cache/kreuzberg/layout/rtdetr/model.onnx"
EXPECTED="3bf2fb0ee6df87435b7ae47f0f3930ec3dc97ec56fd824acc6d57bc7a6b89ef2"
if command -v sha256sum &>/dev/null; then
ACTUAL=$(sha256sum "$MODEL_FILE" | awk '{print $1}')
else
ACTUAL=$(shasum -a 256 "$MODEL_FILE" | awk '{print $1}')
fi
if [ "$ACTUAL" != "$EXPECTED" ]; then
echo "ERROR: RT-DETR SHA256 mismatch"
echo " Expected: $EXPECTED"
echo " Actual: $ACTUAL"
rm -f "$MODEL_FILE"
exit 1
fi
echo "RT-DETR SHA256 verified"
- name: Download TATR model (tatr)
if: contains(inputs.models, 'tatr') && steps.cache-models.outputs.cache-hit != 'true'
shell: bash
run: |
MODEL_URL="https://huggingface.co/Kreuzberg/layout-models/resolve/main/tatr/model.onnx"
CACHE_DIR="$HOME/.cache/kreuzberg/layout"
MODEL_DIR="$CACHE_DIR/tatr"
MODEL_FILE="$MODEL_DIR/tatr.onnx"
echo "Downloading TATR table recognition model from $MODEL_URL"
mkdir -p "$MODEL_DIR"
for attempt in 1 2 3; do
if [ $attempt -gt 1 ]; then
backoff=$((5 * 3 ** (attempt - 2)))
echo "Retry attempt $attempt/3 after ${backoff}s backoff..."
sleep $backoff
fi
if curl -f -L --progress-bar --connect-timeout 30 --max-time 600 \
-o "$MODEL_FILE" "$MODEL_URL"; then
echo "TATR model downloaded successfully ($(du -h "$MODEL_FILE" | cut -f1))"
break
fi
done
if [ ! -f "$MODEL_FILE" ]; then
echo "ERROR: Failed to download TATR model after 3 attempts"
exit 1
fi
- name: Verify TATR SHA256
if: contains(inputs.models, 'tatr') && steps.cache-models.outputs.cache-hit != 'true'
shell: bash
run: |
MODEL_FILE="$HOME/.cache/kreuzberg/layout/tatr/tatr.onnx"
EXPECTED="c11f4033da75e9c4d41c403ef356e89caa0a37a7d111b55461e7d5ba856bb6b6"
if command -v sha256sum &>/dev/null; then
ACTUAL=$(sha256sum "$MODEL_FILE" | awk '{print $1}')
else
ACTUAL=$(shasum -a 256 "$MODEL_FILE" | awk '{print $1}')
fi
if [ "$ACTUAL" != "$EXPECTED" ]; then
echo "ERROR: TATR SHA256 mismatch"
echo " Expected: $EXPECTED"
echo " Actual: $ACTUAL"
rm -f "$MODEL_FILE"
exit 1
fi
echo "TATR SHA256 verified"
- name: Verify downloaded models
id: verify-models
shell: bash
run: |
CACHE_DIR="$HOME/.cache/kreuzberg/layout"
AVAILABLE_MODELS=()
TOTAL_SIZE=0
echo "Checking for layout models in $CACHE_DIR"
if [ -f "$CACHE_DIR/rtdetr/model.onnx" ]; then
SIZE=$(wc -c < "$CACHE_DIR/rtdetr/model.onnx" | tr -d ' ')
AVAILABLE_MODELS+=("rtdetr")
TOTAL_SIZE=$((TOTAL_SIZE + SIZE))
echo " ✓ RT-DETR model: $(numfmt --to=iec-i --suffix=B $SIZE 2>/dev/null || echo $SIZE bytes)"
fi
if [ -f "$CACHE_DIR/tatr/tatr.onnx" ]; then
SIZE=$(wc -c < "$CACHE_DIR/tatr/tatr.onnx" | tr -d ' ')
AVAILABLE_MODELS+=("tatr")
TOTAL_SIZE=$((TOTAL_SIZE + SIZE))
echo " ✓ TATR model: $(numfmt --to=iec-i --suffix=B $SIZE 2>/dev/null || echo $SIZE bytes)"
fi
if [ ${#AVAILABLE_MODELS[@]} -eq 0 ]; then
echo "ERROR: No layout models found in cache directory after download"
echo "available-models=" >> $GITHUB_OUTPUT
exit 1
fi
AVAILABLE_MODELS_STR=$(IFS=, ; echo "${AVAILABLE_MODELS[*]}")
echo "✓ Total cached layout models: ${#AVAILABLE_MODELS[@]} ($(numfmt --to=iec-i --suffix=B $TOTAL_SIZE 2>/dev/null || echo $TOTAL_SIZE bytes))"
echo "available-models=$AVAILABLE_MODELS_STR" >> $GITHUB_OUTPUT
- name: Set cache directory output
id: set-outputs
shell: bash
run: |
CACHE_DIR="$HOME/.cache/kreuzberg/layout"
echo "cache-dir=$CACHE_DIR" >> $GITHUB_OUTPUT
echo "KREUZBERG_CACHE_DIR=$HOME/.cache/kreuzberg" >> $GITHUB_ENV
echo "Layout model cache configured at: $CACHE_DIR"

46
.github/actions/setup-onnx-runtime/action.yml vendored Normal file
View File

@@ -0,0 +1,46 @@
name: Setup ONNX Runtime
description: Download and stage ONNX Runtime libraries for bindings
inputs:
ort-version:
description: ONNX Runtime version to download
required: true
dest-dir:
description: Directory (relative to workspace) where libraries should be copied
required: false
default: crates/kreuzberg-node
arch-id:
description: Override architecture (x64|arm64). Defaults to runner architecture.
required: false
default: ""
strategy:
description: "ORT linking strategy: 'system' (dynamic link, default) or 'bundled' (static link via ort-bundled cargo feature)"
required: false
default: system
runs:
using: composite
steps:
- name: Cache ONNX Runtime
id: cache-onnx
uses: actions/cache@v5
with:
path: |
${{ runner.temp }}/onnxruntime
key: onnx-v2-${{ runner.os }}-${{ inputs.arch-id != '' && inputs.arch-id || runner.arch }}-${{ inputs.ort-version }}
restore-keys: |
onnx-v2-${{ runner.os }}-${{ inputs.arch-id != '' && inputs.arch-id || runner.arch }}-
onnx-v2-${{ runner.os }}-
- name: Prepare ONNX Runtime (Linux)
if: runner.os == 'Linux'
shell: bash
run: scripts/ci/actions/setup-onnx-runtime/linux.sh "${{ inputs.ort-version }}" "${{ inputs.dest-dir }}" "${{ inputs.arch-id }}" "${{ inputs.strategy }}"
- name: Prepare ONNX Runtime (macOS)
if: runner.os == 'macOS'
shell: bash
run: scripts/ci/actions/setup-onnx-runtime/macos.sh "${{ inputs.ort-version }}" "${{ inputs.dest-dir }}" "${{ inputs.arch-id }}" "${{ inputs.strategy }}"
- name: Prepare ONNX Runtime (Windows)
if: runner.os == 'Windows'
shell: pwsh
run: scripts/ci/actions/setup-onnx-runtime/windows.ps1 "${{ inputs.ort-version }}" "${{ inputs.dest-dir }}" "${{ inputs.arch-id }}" "${{ inputs.strategy }}"

202
.github/actions/setup-paddle-ocr-models/README.md vendored Normal file
View File

@@ -0,0 +1,202 @@
# Setup PaddleOCR Models Cache
GitHub Action to download and cache PaddleOCR ONNX models for CI testing and development.
## Overview
This action manages the setup of PaddleOCR PP-OCRv5 ONNX models used by the `kreuzberg-paddle-ocr` crate for optical character recognition testing. It:
- Downloads three model types (detection, classification, recognition) from Hugging Face
- Caches models per OS and CPU architecture (Linux x86_64, Linux ARM64, macOS, Windows)
- Provides environment variables for downstream use
- Outputs cache hit status and available model information
- Gracefully handles download failures (continues with available models)
## Models
The action downloads pre-converted ONNX format models from the `Kreuzberg/paddleocr-onnx-models` Hugging Face repository:
| Model Type | File | Size | Purpose |
| -------------------- | ------------------------------------- | ------- | ----------------------------------------- |
| Detection (det) | `PP-OCRv5_server_det_infer.onnx` | ~84 MB | Text location detection (PP-OCRv5 server) |
| Classification (cls) | `ch_ppocr_mobile_v2.0_cls_infer.onnx` | ~0.6 MB | Text orientation classification |
| Recognition (rec) | `rec/english/model.onnx` | ~8 MB | Text character recognition (PP-OCRv5) |
**Total cache size: ~93 MB per OS/architecture combination**
## Usage
### Basic Usage
```yaml
- uses: ./.github/actions/setup-paddle-ocr-models
```
### With Custom Cache Suffix
```yaml
- uses: ./.github/actions/setup-paddle-ocr-models
with:
cache-key-suffix: my-paddle-ocr-v5
```
### Disable Caching
For cross-architecture builds where caching doesn't help:
```yaml
- uses: ./.github/actions/setup-paddle-ocr-models
with:
cache-enabled: false
```
### Download Specific Models Only
```yaml
- uses: ./.github/actions/setup-paddle-ocr-models
with:
models: "det,rec" # Skip classification model
```
## Inputs
| Name | Description | Required | Default |
| ------------------ | --------------------------------------------------------------- | -------- | -------------------- |
| `cache-enabled` | Enable model caching (set false for cross-arch builds) | No | `true` |
| `models` | Comma-separated list of models to setup (det,cls,rec or subset) | No | `det,cls,rec` |
| `cache-key-suffix` | Suffix for cache key to differentiate model sets | No | `paddle-ocr-v5-onnx` |
## Outputs
| Name | Description |
| ------------------ | ---------------------------------------------------- |
| `cache-hit` | Whether models were restored from cache (true/false) |
| `cache-dir` | Path to the PaddleOCR model cache directory |
| `models-available` | Comma-separated list of available models after setup |
## Outputs as Environment Variables
The action automatically exports:
- `PADDLE_OCR_MODEL_CACHE`: Absolute path to model cache directory
## Cache Strategy
Models are cached using GitHub Actions cache with the following key structure:
```text
paddle-ocr-v5-onnx-{OS}-{ARCHITECTURE}-v4
```
Cache restoration order (restore-keys):
1. Exact match: `paddle-ocr-v5-onnx-{OS}-{ARCHITECTURE}-v4`
2. OS-Architecture: `paddle-ocr-v5-onnx-{OS}-{ARCHITECTURE}-`
3. OS only: `paddle-ocr-v5-onnx-{OS}-`
4. Any: `paddle-ocr-v5-onnx-`
## Example: CI Rust Workflow Integration
```yaml
jobs:
paddle-ocr-tests:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/setup-paddle-ocr-models
id: paddle-models
- name: Run PaddleOCR tests
run: cargo test --package kreuzberg-paddle-ocr
env:
PADDLE_OCR_MODEL_CACHE: ${{ steps.paddle-models.outputs.cache-dir }}
- name: Report cache status
if: always()
run: |
echo "Cache hit: ${{ steps.paddle-models.outputs.cache-hit }}"
echo "Available models: ${{ steps.paddle-models.outputs.models-available }}"
```
## Error Handling
The action downloads models sequentially and will fail if a required model download fails. After downloading:
- The verify step reports which models are actually available in the output
- Downstream tests can check `models-available` to know what's available
- If all models fail, tests can fall back to alternative behavior
## Download Sources
Models are downloaded from:
```text
https://huggingface.co/Kreuzberg/paddleocr-onnx-models/resolve/main/
```
If this repository becomes unavailable, the action will fail gracefully. Alternative sources can be configured by modifying the `MODEL_URL` environment variables in the action.
## Troubleshooting
### Models not being cached
1. Check that `cache-enabled` is not set to `false`
2. Verify GitHub Actions cache is not full (max 10 GB per repository)
3. Check runner OS and architecture match cache keys
4. View cache in repository settings (Settings → Actions → Caches)
### Download timeouts
If downloads timeout:
- Increase the 300-second timeout in the action steps
- Check Hugging Face API availability
- Try reducing the number of models (`models: "det,rec"`)
### Verifying models are present
Check that all expected models exist in the correct directory structure:
```bash
ls -lh ~/.cache/kreuzberg/paddle-ocr/
```
Expected output:
```text
drwxr-xr-x det/
drwxr-xr-x cls/
drwxr-xr-x rec/
ls -lh ~/.cache/kreuzberg/paddle-ocr/det/
-rw-r--r-- model.onnx (~84 MB)
ls -lh ~/.cache/kreuzberg/paddle-ocr/cls/
-rw-r--r-- model.onnx (~0.6 MB)
ls -lh ~/.cache/kreuzberg/paddle-ocr/rec/english/
-rw-r--r-- model.onnx (~8 MB)
-rw-r--r-- dict.txt
```
The directory structure must match what `ModelManager` expects in `model_manager.rs`.
## Performance Impact
- **First run (no cache)**: ~30-60 seconds (download time depends on network)
- **Cached run**: <1 second (cache restore)
- **Cache size**: ~93 MB per OS/architecture
- **Network bandwidth**: ~93 MB download on cache miss
## Related Actions
- `.github/actions/setup-tesseract-cache` - Similar caching for Tesseract models
- `.github/actions/cache-hf-fastembed` - Hugging Face model caching for fastembed
- `.github/actions/setup-onnx-runtime` - ONNX Runtime setup for inference
## See Also
- [PaddleOCR Documentation](https://github.com/PaddlePaddle/PaddleOCR)
- [kreuzberg-paddle-ocr crate](../../../crates/kreuzberg-paddle-ocr)
- [ModelManager source](../../../crates/kreuzberg/src/paddle_ocr/model_manager.rs)

231
.github/actions/setup-paddle-ocr-models/action.yml vendored Normal file
View File

@@ -0,0 +1,231 @@
name: Setup PaddleOCR Models Cache
description: Download and cache PaddleOCR ONNX models for CI testing
inputs:
cache-enabled:
description: Enable model caching (set to false for cross-arch builds)
required: false
default: "true"
models:
description: Comma-separated list of models to setup (det,cls,rec or specific subset)
required: false
default: "det,cls,rec"
cache-key-suffix:
description: Suffix for cache key to differentiate model sets
required: false
default: "paddle-ocr-v5-onnx"
outputs:
cache-hit:
description: Whether models were restored from cache (true/false)
value: ${{ steps.cache-models.outputs.cache-hit }}
cache-dir:
description: Path to the PaddleOCR model cache directory
value: ${{ steps.set-outputs.outputs.cache-dir }}
models-available:
description: Comma-separated list of available models
value: ${{ steps.verify-models.outputs.available-models }}
runs:
using: composite
steps:
- name: Setup cache directory
shell: bash
run: |
mkdir -p ~/.cache/kreuzberg/paddle-ocr
echo "Cache directory: $HOME/.cache/kreuzberg/paddle-ocr"
- name: Restore PaddleOCR models from cache
if: inputs.cache-enabled == 'true'
uses: actions/cache@v5
id: cache-models
with:
path: ~/.cache/kreuzberg/paddle-ocr
key: ${{ inputs.cache-key-suffix }}-${{ runner.os }}-${{ runner.arch }}-v4
restore-keys: |
${{ inputs.cache-key-suffix }}-${{ runner.os }}-${{ runner.arch }}-
${{ inputs.cache-key-suffix }}-${{ runner.os }}-
${{ inputs.cache-key-suffix }}-
- name: Download detection model (det)
if: contains(inputs.models, 'det') && steps.cache-models.outputs.cache-hit != 'true'
shell: bash
run: |
MODEL_URL="https://huggingface.co/Kreuzberg/paddleocr-onnx-models/resolve/main/PP-OCRv5_server_det_infer.onnx"
CACHE_DIR="$HOME/.cache/kreuzberg/paddle-ocr"
MODEL_DIR="$CACHE_DIR/det"
MODEL_FILE="$MODEL_DIR/model.onnx"
echo "Downloading detection model from $MODEL_URL"
mkdir -p "$MODEL_DIR"
for attempt in 1 2 3; do
if [ $attempt -gt 1 ]; then
backoff=$((5 * 3 ** (attempt - 2)))
echo "Retry attempt $attempt/3 after ${backoff}s backoff..."
sleep $backoff
fi
if curl -f -L --progress-bar --connect-timeout 30 --max-time 600 \
-o "$MODEL_FILE" "$MODEL_URL"; then
echo "Detection model downloaded successfully ($(du -h "$MODEL_FILE" | cut -f1))"
exit 0
fi
done
echo "ERROR: Failed to download detection model after 3 attempts"
rm -f "$MODEL_FILE"
exit 1
- name: Download classification model (cls)
if: contains(inputs.models, 'cls') && steps.cache-models.outputs.cache-hit != 'true'
shell: bash
run: |
MODEL_URL="https://huggingface.co/Kreuzberg/paddleocr-onnx-models/resolve/main/ch_ppocr_mobile_v2.0_cls_infer.onnx"
CACHE_DIR="$HOME/.cache/kreuzberg/paddle-ocr"
MODEL_DIR="$CACHE_DIR/cls"
MODEL_FILE="$MODEL_DIR/model.onnx"
echo "Downloading classification model from $MODEL_URL"
mkdir -p "$MODEL_DIR"
for attempt in 1 2 3; do
if [ $attempt -gt 1 ]; then
backoff=$((5 * 3 ** (attempt - 2)))
echo "Retry attempt $attempt/3 after ${backoff}s backoff..."
sleep $backoff
fi
if curl -f -L --progress-bar --connect-timeout 30 --max-time 600 \
-o "$MODEL_FILE" "$MODEL_URL"; then
echo "Classification model downloaded successfully ($(du -h "$MODEL_FILE" | cut -f1))"
exit 0
fi
done
echo "ERROR: Failed to download classification model after 3 attempts"
rm -f "$MODEL_FILE"
exit 1
- name: Download recognition model (rec/english)
if: contains(inputs.models, 'rec') && steps.cache-models.outputs.cache-hit != 'true'
shell: bash
run: |
MODEL_URL="https://huggingface.co/Kreuzberg/paddleocr-onnx-models/resolve/main/rec/english/model.onnx"
CACHE_DIR="$HOME/.cache/kreuzberg/paddle-ocr"
MODEL_DIR="$CACHE_DIR/rec/english"
MODEL_FILE="$MODEL_DIR/model.onnx"
echo "Downloading English recognition model from $MODEL_URL"
mkdir -p "$MODEL_DIR"
for attempt in 1 2 3; do
if [ $attempt -gt 1 ]; then
backoff=$((5 * 3 ** (attempt - 2)))
echo "Retry attempt $attempt/3 after ${backoff}s backoff..."
sleep $backoff
fi
if curl -f -L --progress-bar --connect-timeout 30 --max-time 600 \
-o "$MODEL_FILE" "$MODEL_URL"; then
echo "Recognition model downloaded successfully ($(du -h "$MODEL_FILE" | cut -f1))"
exit 0
fi
done
echo "ERROR: Failed to download recognition model after 3 attempts"
rm -f "$MODEL_FILE"
exit 1
- name: Download recognition dictionary (rec/english/dict.txt)
if: contains(inputs.models, 'rec') && steps.cache-models.outputs.cache-hit != 'true'
shell: bash
run: |
DICT_URL="https://huggingface.co/Kreuzberg/paddleocr-onnx-models/resolve/main/rec/english/dict.txt"
CACHE_DIR="$HOME/.cache/kreuzberg/paddle-ocr"
MODEL_DIR="$CACHE_DIR/rec/english"
DICT_FILE="$MODEL_DIR/dict.txt"
echo "Downloading English recognition dictionary from $DICT_URL"
mkdir -p "$MODEL_DIR"
for attempt in 1 2 3; do
if [ $attempt -gt 1 ]; then
backoff=$((5 * 3 ** (attempt - 2)))
echo "Retry attempt $attempt/3 after ${backoff}s backoff..."
sleep $backoff
fi
if curl -f -L --progress-bar --connect-timeout 30 --max-time 600 \
-o "$DICT_FILE" "$DICT_URL"; then
echo "Dictionary downloaded successfully ($(du -h "$DICT_FILE" | cut -f1))"
exit 0
fi
done
echo "ERROR: Failed to download dictionary after 3 attempts"
rm -f "$DICT_FILE"
exit 1
- name: Verify downloaded models
id: verify-models
shell: bash
run: |
CACHE_DIR="$HOME/.cache/kreuzberg/paddle-ocr"
AVAILABLE_MODELS=()
TOTAL_SIZE=0
echo "Checking for PaddleOCR models in $CACHE_DIR"
if [ -f "$CACHE_DIR/det/model.onnx" ]; then
SIZE=$(wc -c < "$CACHE_DIR/det/model.onnx" | tr -d ' ')
AVAILABLE_MODELS+=("det")
TOTAL_SIZE=$((TOTAL_SIZE + SIZE))
echo " ✓ Detection model: $(numfmt --to=iec-i --suffix=B $SIZE 2>/dev/null || echo $SIZE bytes)"
fi
if [ -f "$CACHE_DIR/cls/model.onnx" ]; then
SIZE=$(wc -c < "$CACHE_DIR/cls/model.onnx" | tr -d ' ')
AVAILABLE_MODELS+=("cls")
TOTAL_SIZE=$((TOTAL_SIZE + SIZE))
echo " ✓ Classification model: $(numfmt --to=iec-i --suffix=B $SIZE 2>/dev/null || echo $SIZE bytes)"
fi
if [ -f "$CACHE_DIR/rec/english/model.onnx" ]; then
SIZE=$(wc -c < "$CACHE_DIR/rec/english/model.onnx" | tr -d ' ')
AVAILABLE_MODELS+=("rec")
TOTAL_SIZE=$((TOTAL_SIZE + SIZE))
echo " ✓ Recognition model (English): $(numfmt --to=iec-i --suffix=B $SIZE 2>/dev/null || echo $SIZE bytes)"
fi
if [ -f "$CACHE_DIR/rec/english/dict.txt" ]; then
SIZE=$(wc -c < "$CACHE_DIR/rec/english/dict.txt" | tr -d ' ')
TOTAL_SIZE=$((TOTAL_SIZE + SIZE))
echo " ✓ Recognition dictionary (English): $(numfmt --to=iec-i --suffix=B $SIZE 2>/dev/null || echo $SIZE bytes)"
fi
if [ ${#AVAILABLE_MODELS[@]} -eq 0 ]; then
echo "ERROR: No models found in cache directory after download"
echo "available-models=" >> $GITHUB_OUTPUT
exit 1
fi
AVAILABLE_MODELS_STR=$(IFS=, ; echo "${AVAILABLE_MODELS[*]}")
echo "✓ Total cached models: ${#AVAILABLE_MODELS[@]} ($(numfmt --to=iec-i --suffix=B $TOTAL_SIZE 2>/dev/null || echo $TOTAL_SIZE bytes))"
echo "available-models=$AVAILABLE_MODELS_STR" >> $GITHUB_OUTPUT
- name: Set cache directory output
id: set-outputs
shell: bash
run: |
CACHE_DIR="$HOME/.cache/kreuzberg/paddle-ocr"
echo "cache-dir=$CACHE_DIR" >> $GITHUB_OUTPUT
echo "PADDLE_OCR_MODEL_CACHE=$CACHE_DIR" >> $GITHUB_ENV
echo "KREUZBERG_CACHE_DIR=$HOME/.cache/kreuzberg" >> $GITHUB_ENV
- name: Export cache environment
shell: bash
run: |
echo "PADDLE_OCR_MODEL_CACHE=$HOME/.cache/kreuzberg/paddle-ocr" >> $GITHUB_ENV
echo "KREUZBERG_CACHE_DIR=$HOME/.cache/kreuzberg" >> $GITHUB_ENV
echo "PaddleOCR model cache configured at: $HOME/.cache/kreuzberg/paddle-ocr"

60
.github/actions/setup-tesseract-cache/action.yml vendored Normal file
View File

@@ -0,0 +1,60 @@
name: Setup Tesseract Cache
description: Manages kreuzberg-tesseract build cache per architecture
inputs:
label:
description: Platform label (e.g. linux-x86_64, linux-aarch64)
required: true
enable-cache:
description: Enable tesseract caching (disable for cross-arch builds)
required: false
default: "true"
rust-target:
description: Rust target triple for per-target cache cleanup
required: false
default: ""
outputs:
cache-dir:
description: Tesseract cache directory path
value: ${{ steps.set-outputs.outputs.cache-dir }}
cache-enabled:
description: Whether caching is enabled (true/false)
value: ${{ steps.set-outputs.outputs.cache-enabled }}
docker-options:
description: Docker options for passing cache env vars
value: ${{ steps.set-outputs.outputs.docker-options }}
runs:
using: composite
steps:
- name: Clean cache directories (cache disabled)
if: inputs.enable-cache != 'true'
shell: bash
run: scripts/ci/actions/setup-tesseract-cache/clean-dirs.sh "${{ inputs.label }}"
- name: Setup cache directories
if: inputs.enable-cache == 'true'
shell: bash
run: scripts/ci/actions/setup-tesseract-cache/setup-dirs.sh "${{ inputs.label }}"
- name: Cache kreuzberg-tesseract build cache
if: inputs.enable-cache == 'true'
uses: actions/cache@v5
with:
path: |
.tesseract-cache/${{ inputs.label }}
.xdg-cache/${{ inputs.label }}
key: kreuzberg-tesseract-cache-v2-${{ runner.os }}-${{ inputs.label }}-${{ hashFiles('crates/kreuzberg-tesseract/Cargo.toml', 'crates/kreuzberg-tesseract/build.rs') }}
restore-keys: |
kreuzberg-tesseract-cache-v2-${{ runner.os }}-${{ inputs.label }}-
- name: Clean per-target Tesseract cache
if: inputs.rust-target != ''
shell: bash
run: scripts/ci/actions/setup-tesseract-cache/clean-target-cache.sh "${{ inputs.rust-target }}"
- name: Set outputs and environment
id: set-outputs
shell: bash
run: scripts/ci/actions/setup-tesseract-cache/set-outputs.sh "${{ inputs.label }}" "${{ inputs.enable-cache }}"

67
.github/dependabot.yaml vendored Normal file
View File

@@ -0,0 +1,67 @@
version: 2
multi-ecosystem-groups:
dependencies:
schedule:
interval: "weekly"
updates:
- package-ecosystem: "cargo"
# Explicitly list root only — packages/ruby/ext and packages/r/src have
# standalone workspaces with path deps to vendored crates that only exist
# at build time. Dependabot cannot resolve these paths.
directories:
- "/"
ignore:
- dependency-name: "kreuzberg"
- dependency-name: "kreuzberg-ffi"
patterns: ["*"]
multi-ecosystem-group: "dependencies"
- package-ecosystem: "pip"
directories:
- "/"
- "/packages/python"
patterns: ["*"]
multi-ecosystem-group: "dependencies"
- package-ecosystem: "npm"
directories:
- "/"
- "/crates/kreuzberg-node"
- "/crates/kreuzberg-wasm"
- "/packages/typescript/core"
patterns: ["*"]
multi-ecosystem-group: "dependencies"
- package-ecosystem: "bundler"
directory: "/packages/ruby"
patterns: ["*"]
multi-ecosystem-group: "dependencies"
- package-ecosystem: "composer"
directories:
- "/"
- "/packages/php"
patterns: ["*"]
multi-ecosystem-group: "dependencies"
- package-ecosystem: "gomod"
directory: "/packages/go/v5"
patterns: ["*"]
multi-ecosystem-group: "dependencies"
- package-ecosystem: "maven"
directory: "/packages/java"
patterns: ["*"]
multi-ecosystem-group: "dependencies"
- package-ecosystem: "nuget"
directory: "/packages/csharp"
patterns: ["*"]
multi-ecosystem-group: "dependencies"
- package-ecosystem: "mix"
directory: "/packages/elixir"
patterns: ["*"]
multi-ecosystem-group: "dependencies"

39
.github/documentation/runners.md vendored Normal file
View File

@@ -0,0 +1,39 @@
# Custom GitHub Actions Runners
## Available Runners
| Runner Label | Architecture | Size | Ephemeral | Notes |
| -------------------------- | ------------ | ------ | --------- | ---------------------------------------------------------- |
| `runner-small` | x86_64 | Small | No | Light tasks: linting, formatting, validation |
| `runner-medium` | x86_64 | Medium | No | Standard CI: tests, builds |
| `runner-medium-arm64` | arm64 | Medium | No | ARM64 builds and tests |
| `runner-large` | x86_64 | Large | No | Heavy workloads: benchmarks, coverage, release builds |
| `runner-large-spot` | x86_64 | Large | Yes | Cost-optimized large jobs where interruption is acceptable |
| `runner-medium-arm64-spot` | arm64 | Medium | Yes | Cost-optimized ARM64 jobs where interruption is acceptable |
## Spot Runners
Spot runners (`*-spot`) use ephemeral cloud instances provisioned on a best-effort basis. They are significantly cheaper but can be preempted at any time if the cloud provider reclaims capacity.
**Use spot runners for:**
- Jobs that can be retried without consequence (test suites, linting)
- Non-time-critical workloads
- PR validation where re-runs are acceptable
**Do not use spot runners for:**
- Benchmarks (preemption and noisy-neighbor effects skew results)
- Release builds and publishing
- Jobs requiring consistent, reproducible timing
## Choosing a Runner
| Workload | Recommended Runner |
| ------------------------------- | -------------------------- |
| Linting, formatting, validation | `runner-small` |
| Unit tests, standard builds | `runner-medium` |
| ARM64 cross-compilation / tests | `runner-medium-arm64` |
| Benchmarks, coverage reports | `runner-large` |
| Non-critical large builds | `runner-large-spot` |
| Non-critical ARM64 builds | `runner-medium-arm64-spot` |

1244
.github/workflows/benchmarks.yaml vendored Normal file
View File

File diff suppressed because it is too large Load Diff

74
.github/workflows/build-node-native.yml vendored Normal file
View File

@@ -0,0 +1,74 @@
name: Build Node Native
on:
push:
branches: [main]
paths:
- "crates/kreuzberg-node/**"
- "crates/kreuzberg/**"
- "Cargo.toml"
- "Cargo.lock"
- "rust-toolchain.toml"
- ".github/workflows/build-node-native.yml"
pull_request:
branches: [main]
paths:
- "crates/kreuzberg-node/**"
- "crates/kreuzberg/**"
- "Cargo.toml"
- "Cargo.lock"
- "rust-toolchain.toml"
- ".github/workflows/build-node-native.yml"
workflow_dispatch:
concurrency:
group: build-node-native-${{ github.ref }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: 0
MACOSX_DEPLOYMENT_TARGET: "14.0"
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
permissions:
contents: read
jobs:
build:
name: Build ${{ matrix.target }}
runs-on: ${{ matrix.os }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
target: x86_64-unknown-linux-gnu
- os: ubuntu-24.04-arm
target: aarch64-unknown-linux-gnu
- os: macos-13
target: x86_64-apple-darwin
- os: macos-latest
target: aarch64-apple-darwin
- os: windows-latest
target: x86_64-pc-windows-msvc
steps:
- uses: actions/checkout@v6.0.2
with:
submodules: recursive
- uses: kreuzberg-dev/actions/setup-rust@v1
with:
target: ${{ matrix.target }}
- uses: kreuzberg-dev/actions/setup-node-workspace@v1
with:
node-version: "24"
- name: Build NAPI binding
uses: kreuzberg-dev/actions/build-node-napi@v1
with:
crate-dir: crates/kreuzberg-node
build-command: pnpm exec napi build --release --target ${{ matrix.target }} --platform

79
.github/workflows/ci-docker.yaml vendored Normal file
View File

@@ -0,0 +1,79 @@
name: CI Docker
on:
workflow_dispatch:
concurrency:
group: ci-docker-${{ github.ref }}
cancel-in-progress: true
env:
ORT_VERSION: "1.24.2"
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
permissions:
contents: read
jobs:
docker:
name: Docker (${{ matrix.variant }})
if: github.repository == 'kreuzberg-dev/kreuzberg' && github.actor != 'dependabot[bot]'
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
variant: [core, full, cli]
steps:
- uses: actions/checkout@v6
- name: Free disk space
uses: kreuzberg-dev/actions/free-disk-space-linux@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Build Docker image
uses: docker/build-push-action@v7
with:
context: .
file: docker/Dockerfile.${{ matrix.variant }}
push: false
load: true
tags: kreuzberg:${{ matrix.variant }}
build-args: ONNXRUNTIME_VERSION=${{ env.ORT_VERSION }}
cache-from: type=gha,scope=ci-docker-${{ matrix.variant }}
cache-to: type=gha,mode=max,scope=ci-docker-${{ matrix.variant }}
- name: Save Docker image
shell: bash
run: |
mkdir -p /tmp
docker save kreuzberg:${{ matrix.variant }} | gzip > /tmp/kreuzberg-${{ matrix.variant }}.tar.gz
ls -lh /tmp/kreuzberg-${{ matrix.variant }}.tar.gz
- name: Check image size
uses: kreuzberg-dev/actions/check-docker-image-size@v1
with:
image: kreuzberg:${{ matrix.variant }}
warn-mb: ${{ matrix.variant == 'cli' && '200' || '' }}
label: "${{ matrix.variant }} image"
- name: Run feature tests
if: matrix.variant != 'cli'
run: scripts/ci/docker/run-feature-tests.sh "${{ matrix.variant }}"
- name: Run configuration tests
if: matrix.variant != 'cli'
run: scripts/ci/docker/run-config-tests.sh "${{ matrix.variant }}"
- name: Run API contract tests with schemathesis
if: matrix.variant != 'cli'
uses: kreuzberg-dev/actions/run-api-contract-tests@v1
with:
image: kreuzberg:${{ matrix.variant }}
port: "8000"
- name: Run CLI tests
if: matrix.variant == 'cli'
run: scripts/ci/docker/run-cli-tests.sh

102
.github/workflows/ci-docs.yaml vendored Normal file
View File

@@ -0,0 +1,102 @@
name: CI Docs
on:
pull_request:
paths:
- "docs/**"
- "packages/**/README.md"
- "crates/*/README.md"
- "packages/python/pyproject.toml"
- "packages/typescript/package.json"
- "packages/ruby/kreuzberg.gemspec"
- "packages/php/composer.json"
- "packages/go/v5/go.mod"
- "packages/java/pom.xml"
- "packages/csharp/**/Kreuzberg.csproj"
- "packages/elixir/mix.exs"
- "packages/r/DESCRIPTION"
- "packages/dart/pubspec.yaml"
- "zensical.toml"
- "mkdocs.yml"
- "alef.toml"
- ".github/workflows/ci-docs.yaml"
push:
branches: [main]
paths:
- "docs/**"
- "packages/**/README.md"
- "crates/*/README.md"
- "zensical.toml"
- "pyproject.toml"
- "alef.toml"
- "CHANGELOG.md"
- ".github/workflows/ci-docs.yaml"
workflow_dispatch:
permissions:
contents: read
pages: write
id-token: write
concurrency:
group: ci-docs-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: false
jobs:
lint:
name: Lint
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
- name: Lint documentation + validate snippets
uses: kreuzberg-dev/actions/lint-docs@v1
with:
working-directory: .
strict: "true"
validate-snippets: "true"
alef-ref: v0.19.5
build:
name: Build
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Build documentation
uses: kreuzberg-dev/actions/build-docs@v1
with:
working-directory: .
strict: "true"
- name: Upload site artifact
uses: actions/upload-artifact@v7
with:
name: docs-site
path: site/
retention-days: 1
deploy:
name: Deploy
needs: [build, lint]
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
runs-on: ubuntu-latest
steps:
- name: Download site artifact
uses: actions/download-artifact@v8
with:
name: docs-site
path: site/
- name: Upload Pages artifact
uses: actions/upload-pages-artifact@v5
with:
path: site
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v5

345
.github/workflows/ci-e2e.yaml vendored Normal file
View File

@@ -0,0 +1,345 @@
name: CI E2E
on:
push:
branches: [main]
paths:
- "crates/**"
- "packages/**"
- "e2e/**"
- "fixtures/**"
- "alef.toml"
- "Cargo.toml"
- "Cargo.lock"
- "Taskfile.yml"
- ".github/workflows/ci-e2e.yaml"
pull_request:
branches: [main]
paths:
- "crates/**"
- "packages/**"
- "e2e/**"
- "fixtures/**"
- "alef.toml"
- "Cargo.toml"
- "Cargo.lock"
- "Taskfile.yml"
- ".github/workflows/ci-e2e.yaml"
workflow_dispatch:
concurrency:
group: ci-e2e-${{ github.ref }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: 0
CARGO_PROFILE_DEV_DEBUG: 0
RUST_BACKTRACE: short
RUST_MIN_STACK: 16777216
ORT_VERSION: "1.24.2"
MACOSX_DEPLOYMENT_TARGET: "14.0"
BUILD_PROFILE: "ci"
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
permissions:
contents: read
jobs:
build-ffi:
name: Build FFI (${{ matrix.target }})
if: github.repository == 'kreuzberg-dev/kreuzberg' && github.actor != 'dependabot[bot]'
runs-on: ${{ matrix.os }}
timeout-minutes: ${{ matrix.os == 'windows-latest' && 120 || 60 }}
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-24.04-arm
target: aarch64-unknown-linux-gnu
- os: ubuntu-latest
target: x86_64-unknown-linux-gnu
- os: macos-latest
target: aarch64-apple-darwin
- os: windows-latest
target: x86_64-pc-windows-msvc
steps:
- uses: actions/checkout@v6
with:
submodules: recursive
- name: Setup Rust
uses: kreuzberg-dev/actions/setup-rust@v1
with:
cache-key-prefix: build-ffi-${{ matrix.target }}
target: ${{ matrix.target }}
- name: Install system dependencies
uses: ./.github/actions/install-system-deps
- name: Setup OpenSSL
uses: kreuzberg-dev/actions/setup-openssl@v1
- name: Build FFI library
uses: kreuzberg-dev/actions/build-rust-ffi@v1
with:
crate-name: kreuzberg-ffi
- name: Build CLI
uses: kreuzberg-dev/actions/build-rust-cli@v1
with:
package-name: kreuzberg-cli
binary-name: kreuzberg
extra-cargo-args: --features all
- name: Upload FFI artifacts
uses: actions/upload-artifact@v7
with:
name: ffi-${{ matrix.target }}
path: |
target/release/libkreuzberg_ffi.*
target/release/kreuzberg_ffi.*
crates/kreuzberg-ffi/include/kreuzberg.h
crates/kreuzberg-ffi/kreuzberg-ffi.pc
crates/kreuzberg-ffi/cmake/
target/release/kreuzberg
target/release/kreuzberg.exe
retention-days: 7
if-no-files-found: error
e2e-tests:
name: E2E (${{ matrix.lang }})
if: github.repository == 'kreuzberg-dev/kreuzberg' && github.actor != 'dependabot[bot]'
needs: [build-ffi]
runs-on: ubuntu-24.04-arm
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
include:
- lang: python
python-version: "3.13"
test-cmd: "pip install maturin && cd packages/python && maturin develop --release && cd ../../e2e/python && python3 -m pytest tests/ -q"
- lang: node
node-version: "24"
test-cmd: "cd crates/kreuzberg-node && npm run build && cd ../../e2e/node && npx vitest run"
- lang: go
go-version: "1.26"
test-cmd: "cd e2e/go && go test ./... -count=1 -v"
- lang: ruby
ruby-version: "3.4"
test-cmd: "cd e2e/ruby && bundle exec rspec"
- lang: java
java-version: "25"
test-cmd: "cd packages/java && mvn -q package -DskipTests && cd ../../e2e/java && mvn test -q"
- lang: csharp
dotnet-version: "10.0.x"
test-cmd: "cd e2e/csharp && dotnet test"
- lang: php
php-version: "8.4"
test-cmd: 'cd crates/kreuzberg-php && cargo build --release && echo "extension=$(pwd)/../../target/release/libkreuzberg_php.so" | sudo tee -a "$(php -r ''echo php_ini_loaded_file();'')" >/dev/null && cd ../../e2e/php && composer install -q && vendor/bin/phpunit'
- lang: elixir
elixir-version: "1.19"
otp-version: "28"
test-cmd: "cd e2e/elixir && KREUZBERG_BUILD=true mix deps.get && KREUZBERG_BUILD=true mix test"
- lang: wasm
node-version: "24"
test-cmd: 'curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && export PATH="$HOME/.cargo/bin:$PATH" && export RUSTFLAGS=''--cfg getrandom_backend="wasm_js"'' && cd crates/kreuzberg-wasm && wasm-pack build --release --target web --out-dir ../../packages/wasm/pkg && cd ../../e2e/wasm && npm install && npm test'
- lang: rust
test-cmd: "cd e2e/rust && cargo test"
- lang: r
r-version: "4.3"
test-cmd: "cd e2e/r && Rscript run_tests.R"
- lang: dart
dart-version: "3.11"
test-cmd: "cargo build --release -p kreuzberg-dart && mkdir -p packages/dart/rust/target/release && cp target/release/libkreuzberg_dart.* packages/dart/rust/target/release/ 2>/dev/null || true && cd packages/dart && dart pub get && cd ../../e2e/dart && dart pub get && dart test"
- lang: kotlin_android
java-version: "25"
test-cmd: "cd e2e/kotlin_android && gradle test --no-daemon"
- lang: swift
swift-version: "6.0"
test-cmd: "cd e2e/swift_e2e && swift test"
- lang: zig
zig-version: "0.16.0"
test-cmd: 'FFI_ABS="$PWD/target/release" && cd e2e/zig && zig build test -Dffi_path="$FFI_ABS"'
steps:
- uses: actions/checkout@v6
with:
submodules: recursive
- name: Setup Rust
uses: kreuzberg-dev/actions/setup-rust@v1
with:
cache-key-prefix: e2e-${{ matrix.lang }}
- name: Download FFI artifacts
uses: actions/download-artifact@v8
with:
name: ffi-aarch64-unknown-linux-gnu
path: ffi-artifacts
- name: Stage FFI artifacts
shell: bash
run: |
mkdir -p target/release crates/kreuzberg-ffi/include crates/kreuzberg-ffi/cmake
if [ -d ffi-artifacts/target/release ]; then
cp -r ffi-artifacts/target/release/. target/release/
fi
if [ -d ffi-artifacts/crates/kreuzberg-ffi/include ]; then
cp -r ffi-artifacts/crates/kreuzberg-ffi/include/. crates/kreuzberg-ffi/include/
fi
if [ -d ffi-artifacts/crates/kreuzberg-ffi/cmake ]; then
cp -r ffi-artifacts/crates/kreuzberg-ffi/cmake/. crates/kreuzberg-ffi/cmake/
fi
if [ -f ffi-artifacts/crates/kreuzberg-ffi/kreuzberg-ffi.pc ]; then
cp ffi-artifacts/crates/kreuzberg-ffi/kreuzberg-ffi.pc crates/kreuzberg-ffi/
fi
chmod +x target/release/libkreuzberg_ffi.so 2>/dev/null || true
ls -la target/release/
if [ -f target/release/libkreuzberg_ffi.so ]; then
sudo cp target/release/libkreuzberg_ffi.so /usr/local/lib/
sudo ldconfig
fi
- name: Install system dependencies
uses: ./.github/actions/install-system-deps
- name: Setup OpenSSL
uses: kreuzberg-dev/actions/setup-openssl@v1
- name: Setup ONNX Runtime
uses: ./.github/actions/setup-onnx-runtime
with:
ort-version: ${{ env.ORT_VERSION }}
- name: Setup Tesseract cache
uses: ./.github/actions/setup-tesseract-cache
with:
label: e2e-${{ matrix.lang }}
- name: Install WASI SDK
if: matrix.lang == 'wasm'
uses: kreuzberg-dev/actions/install-wasi-sdk@v1
- name: Setup Python
if: matrix.python-version
uses: kreuzberg-dev/actions/setup-python-env@v1
with:
python-version: ${{ matrix.python-version }}
cache-prefix: e2e-py-${{ matrix.python-version }}
- name: Setup Node
if: matrix.node-version
uses: kreuzberg-dev/actions/setup-node-workspace@v1
- name: Setup Go
if: matrix.go-version
uses: actions/setup-go@v6
with:
go-version: ${{ matrix.go-version }}
- name: Setup Ruby
if: matrix.ruby-version
uses: ruby/setup-ruby@v1
with:
ruby-version: ${{ matrix.ruby-version }}
bundler-cache: true
working-directory: e2e/ruby
- name: Setup Java
if: matrix.java-version
uses: actions/setup-java@v5
with:
distribution: temurin
java-version: ${{ matrix.java-version }}
- name: Setup Android SDK
if: matrix.lang == 'kotlin_android'
uses: android-actions/setup-android@v3
with:
api-level: 35
build-tools-version: "35.0.0"
- name: Setup Gradle
if: matrix.lang == 'kotlin_android'
uses: kreuzberg-dev/actions/setup-gradle@v1
with:
gradle-version: "9.1.0"
- name: Setup .NET
if: matrix.dotnet-version
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ matrix.dotnet-version }}
- name: Setup PHP
if: matrix.php-version
uses: kreuzberg-dev/actions/setup-php@v1
with:
php-version: ${{ matrix.php-version }}
tools: composer
- name: Setup Elixir
if: matrix.elixir-version
uses: kreuzberg-dev/actions/setup-elixir@v1
with:
elixir-version: ${{ matrix.elixir-version }}
otp-version: ${{ matrix.otp-version }}
- name: Setup R
if: matrix.r-version
uses: kreuzberg-dev/actions/setup-r@v1
with:
r-version: ${{ matrix.r-version }}
- name: Install R test packages
if: matrix.lang == 'r'
run: R -e 'install.packages(c("testthat","jsonlite","devtools"), repos="https://cloud.r-project.org")'
- name: Setup Dart
if: matrix.dart-version
uses: dart-lang/setup-dart@v1
with:
sdk: ${{ matrix.dart-version }}
- name: Setup Swift
if: matrix.swift-version
uses: kreuzberg-dev/actions/setup-swift@v1
with:
swift-version: ${{ matrix.swift-version }}
- name: Setup Zig
if: matrix.zig-version
uses: kreuzberg-dev/actions/setup-zig@v1
with:
version: ${{ matrix.zig-version }}
- name: Setup library paths for FFI bindings
if: |
matrix.lang == 'go' || matrix.lang == 'java' ||
matrix.lang == 'csharp' || matrix.lang == 'elixir' ||
matrix.lang == 'r' || matrix.lang == 'kotlin_android' ||
matrix.lang == 'swift' || matrix.lang == 'zig'
shell: bash
run: |
export PKG_CONFIG_PATH="${PWD}/crates/kreuzberg-ffi:${PKG_CONFIG_PATH}"
export LD_LIBRARY_PATH="${PWD}/target/release:${LD_LIBRARY_PATH}"
echo "PKG_CONFIG_PATH=${PKG_CONFIG_PATH}" >> "$GITHUB_ENV"
echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> "$GITHUB_ENV"
- name: Install Task
uses: kreuzberg-dev/actions/install-task@v1
- name: Compile Ruby native extension
if: matrix.lang == 'ruby'
working-directory: packages/ruby
run: bundle install && bundle exec rake compile
- name: Run tests
run: ${{ matrix.test-cmd }}
shell: bash
env:
PKG_CONFIG_PATH: ${{ env.PKG_CONFIG_PATH }}
LD_LIBRARY_PATH: ${{ env.LD_LIBRARY_PATH }}
DYLD_LIBRARY_PATH: ${{ env.DYLD_LIBRARY_PATH || '' }}
TESSDATA_PREFIX: "/usr/share/tesseract-ocr/5/tessdata"

112
.github/workflows/ci-gpu.yaml vendored Normal file
View File

@@ -0,0 +1,112 @@
name: CI GPU
on:
workflow_dispatch:
concurrency:
group: ci-gpu-${{ github.ref }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: 0
CARGO_PROFILE_DEV_DEBUG: 0
RUST_BACKTRACE: short
RUST_MIN_STACK: 16777216
ORT_VERSION: "1.24.2"
permissions:
contents: read
jobs:
build:
name: "Build test binary"
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Checkout
uses: actions/checkout@v6
with:
submodules: recursive
- name: Install system dependencies
uses: ./.github/actions/install-system-deps
- name: Setup Rust
uses: dtolnay/rust-toolchain@stable
with:
toolchain: "1.95"
- name: Cache Cargo
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: gpu-build-${{ runner.os }}-${{ hashFiles('Cargo.lock') }}
restore-keys: |
gpu-build-${{ runner.os }}-
- name: Build GPU test binary
uses: kreuzberg-dev/actions/build-gpu-test-binary@v1
with:
package: kreuzberg
test-name: gpu_acceleration
features: "paddle-ocr,layout-detection,embeddings,pdf,ocr,ort-dynamic"
output-name: gpu-acceleration-test
- name: Upload test binary
uses: actions/upload-artifact@v7
with:
name: gpu-test-binary
path: gpu-acceleration-test
retention-days: 1
gpu-tests:
name: "GPU Tests (CUDA)"
needs: build
runs-on: runner-gpu-l4
timeout-minutes: 15
steps:
- name: Checkout
uses: actions/checkout@v6
with:
submodules: recursive
- name: Verify GPU
run: |
nvidia-smi || {
echo "ERROR: nvidia-smi failed — no GPU detected"
exit 1
}
echo "GPU detected:"
nvidia-smi --query-gpu=name,driver_version,memory.total --format=csv,noheader
- name: Download test binary
uses: actions/download-artifact@v8.0.1
with:
name: gpu-test-binary
- name: Download ONNX Runtime (GPU/CUDA)
uses: kreuzberg-dev/actions/setup-onnx-runtime-gpu@v1
with:
version: ${{ env.ORT_VERSION }}
- name: Setup PaddleOCR models
uses: ./.github/actions/setup-paddle-ocr-models
- name: Clear stale layout model cache (self-hosted runner persistence)
run: |
rm -rf "$HOME/.cache/kreuzberg/layout"
echo "Cleared layout model cache"
- name: Run GPU tests
run: |
chmod +x gpu-acceleration-test
./gpu-acceleration-test --ignored --nocapture
env:
RUST_LOG: "kreuzberg=debug"
TEST_DOCUMENTS_DIR: ${{ github.workspace }}/test_documents

107
.github/workflows/ci-lint.yaml vendored Normal file
View File

@@ -0,0 +1,107 @@
name: CI Lint
on:
push:
branches: [main]
pull_request:
branches: [main]
workflow_dispatch:
concurrency:
group: ci-lint-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
lint:
name: Lint
if: github.repository == 'kreuzberg-dev/kreuzberg' && github.actor != 'dependabot[bot]'
runs-on: ubuntu-24.04-arm
timeout-minutes: 60
env:
CARGO_TERM_COLOR: always
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
- uses: actions/checkout@v6
with:
submodules: recursive
- name: Setup Rust
uses: kreuzberg-dev/actions/setup-rust@v1
with:
cache-key-prefix: lint
- name: Setup Python
uses: kreuzberg-dev/actions/setup-python-env@v1
with:
python-version: "3.13"
cache-prefix: lint-py
install-command: "uv sync --group dev --no-install-project --no-install-workspace --frozen"
- name: Setup Node
uses: kreuzberg-dev/actions/setup-node-workspace@v1
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: "1.26"
- name: Setup Java
uses: actions/setup-java@v5
with:
distribution: temurin
java-version: "25"
- name: Setup Elixir
uses: kreuzberg-dev/actions/setup-elixir@v1
- name: Setup Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: "3.4"
bundler-cache: true
working-directory: packages/ruby
- name: Setup PHP
uses: kreuzberg-dev/actions/setup-php@v1
- name: Setup .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: "10.0.x"
- name: Setup R
uses: kreuzberg-dev/actions/setup-r@v1
with:
r-version: "release"
install-deps: "false"
- name: Install Task
uses: kreuzberg-dev/actions/install-task@v1
- name: Setup Helm
uses: azure/setup-helm@v5
- name: Setup kubeconform
uses: bmuschko/setup-kubeconform@v1
- name: Install alef CLI
uses: kreuzberg-dev/actions/install-alef@v1
- name: Run all prek hooks
uses: j178/prek-action@v2
with:
cache: false
extra-args: --all-files
- name: Validate C header
shell: bash
run: |
HEADER="crates/kreuzberg-ffi/include/kreuzberg.h"
if [ ! -f "$HEADER" ]; then
echo "::error::C header not found at $HEADER — run 'task alef:generate'"
exit 1
fi
echo "C header verified at $HEADER"

79
.github/workflows/ci-mobile.yaml vendored Normal file
View File

@@ -0,0 +1,79 @@
name: CI Mobile
on:
push:
branches: [main]
paths:
- "crates/**"
- "packages/dart/**"
- "packages/swift/**"
- "packages/kotlin-android/**"
- ".github/workflows/ci-mobile.yaml"
pull_request:
branches: [main]
paths:
- "crates/**"
- "packages/dart/**"
- "packages/swift/**"
- "packages/kotlin-android/**"
- ".github/workflows/ci-mobile.yaml"
workflow_dispatch:
concurrency:
group: ci-mobile-${{ github.ref }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: 0
CARGO_PROFILE_DEV_DEBUG: 0
RUST_BACKTRACE: short
# Mobile feature subsets (Android drops ORT-requiring features) leave some
# functions only used in the full-feature graph; -A dead_code keeps the
# cross-compile check honest about other classes of warnings without choking
# on these.
RUSTFLAGS: "-D warnings -A dead_code -A unpredictable-function-pointer-comparisons -A mismatched-lifetime-syntaxes"
permissions:
contents: read
jobs:
android-check:
name: Android cargo check (${{ matrix.abi }})
if: github.repository == 'kreuzberg-dev/kreuzberg' && github.actor != 'dependabot[bot]'
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
abi: [arm64-v8a, x86_64]
steps:
- uses: actions/checkout@v6
- uses: kreuzberg-dev/actions/setup-rust@v1
with:
cache-key-prefix: ci-mobile-android-${{ matrix.abi }}
- uses: kreuzberg-dev/actions/setup-android-ndk@v1
- name: cargo ndk check kreuzberg-dart
run: cargo ndk --target ${{ matrix.abi }} --platform 21 -- check -p kreuzberg-dart
- name: cargo ndk check kreuzberg-ffi
run: cargo ndk --target ${{ matrix.abi }} --platform 21 -- check -p kreuzberg-ffi
ios-check:
name: iOS cargo check (${{ matrix.target }})
if: github.repository == 'kreuzberg-dev/kreuzberg' && github.actor != 'dependabot[bot]'
runs-on: macos-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
target: [aarch64-apple-ios, aarch64-apple-ios-sim]
steps:
- uses: actions/checkout@v6
- uses: kreuzberg-dev/actions/setup-rust@v1
with:
target: ${{ matrix.target }}
cache-key-prefix: ci-mobile-ios-${{ matrix.target }}
- name: cargo check kreuzberg-dart
run: cargo check -p kreuzberg-dart --target ${{ matrix.target }}
- name: cargo check kreuzberg-swift
run: cargo check -p kreuzberg-swift --target ${{ matrix.target }}

103
.github/workflows/ci-rust.yaml vendored Normal file
View File

@@ -0,0 +1,103 @@
name: CI Rust
on:
push:
branches: [main]
paths:
- "crates/**"
- "Cargo.toml"
- "Cargo.lock"
- "rust-toolchain.toml"
- ".github/workflows/ci-rust.yaml"
pull_request:
branches: [main]
paths:
- "crates/**"
- "Cargo.toml"
- "Cargo.lock"
- "rust-toolchain.toml"
- ".github/workflows/ci-rust.yaml"
workflow_dispatch:
concurrency:
group: ci-rust-${{ github.ref }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: 0
CARGO_PROFILE_DEV_DEBUG: 0
RUST_BACKTRACE: short
RUST_MIN_STACK: 16777216
ORT_VERSION: "1.24.2"
MACOSX_DEPLOYMENT_TARGET: "14.0"
BUILD_PROFILE: "ci"
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
permissions:
contents: read
jobs:
rust:
name: Rust (${{ matrix.os }})
if: github.repository == 'kreuzberg-dev/kreuzberg' && github.actor != 'dependabot[bot]'
runs-on: ${{ matrix.os }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-24.04-arm
- os: macos-latest
steps:
- uses: actions/checkout@v6
with:
submodules: recursive
- name: Free disk space
if: runner.os == 'Linux'
uses: kreuzberg-dev/actions/free-disk-space-linux@v1
with:
show-initial: "false"
show-final: "true"
- name: Setup Rust
uses: kreuzberg-dev/actions/setup-rust@v1
with:
cache-key-prefix: rust-${{ matrix.os }}
use-sccache: "true"
- name: Install system dependencies
uses: ./.github/actions/install-system-deps
- name: Setup OpenSSL
uses: kreuzberg-dev/actions/setup-openssl@v1
- name: Setup ONNX Runtime
uses: ./.github/actions/setup-onnx-runtime
with:
ort-version: ${{ env.ORT_VERSION }}
- name: Setup Tesseract cache
uses: ./.github/actions/setup-tesseract-cache
with:
label: ${{ matrix.os }}
- name: Install Task
uses: kreuzberg-dev/actions/install-task@v1
- name: Run clippy
run: cargo clippy --workspace --exclude kreuzberg-ffi --exclude kreuzberg-py --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm --exclude kreuzberg-dart --exclude kreuzberg-swift --exclude kreuzberg_nif -- -D warnings
shell: bash
- name: Run tests
run: task rust:test:ci
shell: bash
env:
LD_LIBRARY_PATH: ${{ env.LD_LIBRARY_PATH || '' }}
DYLD_LIBRARY_PATH: ${{ env.DYLD_LIBRARY_PATH || '' }}
DYLD_FALLBACK_LIBRARY_PATH: ${{ env.DYLD_FALLBACK_LIBRARY_PATH || '' }}
- name: Check no-default-features
run: cargo check -p kreuzberg --no-default-features
shell: bash

1303
.github/workflows/profiling.yaml vendored Normal file
View File

File diff suppressed because it is too large Load Diff

262
.github/workflows/publish-docker.yaml vendored Normal file
View File

@@ -0,0 +1,262 @@
name: Publish Docker Images
on:
workflow_dispatch:
inputs:
tag:
description: "Release tag to build (e.g., v4.3.6)"
required: true
type: string
dry_run:
description: "Prepare artifacts without publishing"
required: false
type: boolean
default: false
ref:
description: "Git ref (branch, tag, or commit) to build; defaults to the tag"
required: false
type: string
force_republish:
description: "Force re-publish even if artifacts already exist"
required: false
type: boolean
default: false
release:
types: [published]
repository_dispatch:
types: [publish-docker]
concurrency:
group: ${{ github.workflow }}-${{ (github.event_name == 'workflow_dispatch' && (github.event.inputs.ref || github.event.inputs.tag)) || github.ref || github.run_id }}
cancel-in-progress: false
env:
CARGO_TERM_COLOR: always
ORT_VERSION: "1.24.2"
MACOSX_DEPLOYMENT_TARGET: "14.0"
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
permissions:
contents: read
jobs:
prepare:
name: Prepare metadata
if: ${{ github.event_name != 'release' || !github.event.release.prerelease }}
runs-on: ubuntu-latest
permissions:
contents: read
outputs:
tag: ${{ steps.meta.outputs.tag }}
version: ${{ steps.meta.outputs.version }}
ref: ${{ steps.meta.outputs.ref }}
dry_run: ${{ steps.meta.outputs.dry_run }}
force_republish: ${{ steps.meta.outputs.force_republish }}
checkout_ref: ${{ steps.meta.outputs.checkout_ref }}
target_sha: ${{ steps.meta.outputs.target_sha }}
is_tag: ${{ steps.meta.outputs.is_tag }}
release_docker: ${{ steps.meta.outputs.release_docker }}
steps:
- name: Checkout code (default)
uses: actions/checkout@v6
- name: Resolve release metadata
id: meta
uses: kreuzberg-dev/actions/prepare-release-metadata@v1
with:
tag: ${{ inputs.tag }}
ref: ${{ inputs.ref }}
targets: docker
dry-run: ${{ inputs.dry_run }}
force-republish: ${{ inputs.force_republish }}
- name: Re-checkout at target ref
if: ${{ steps.meta.outputs.checkout_ref != '' }}
uses: actions/checkout@v6
with:
ref: ${{ steps.meta.outputs.checkout_ref }}
fetch-depth: 0
submodules: recursive
- name: Show metadata
env:
META_TAG: ${{ steps.meta.outputs.tag }}
META_VERSION: ${{ steps.meta.outputs.version }}
META_REF: ${{ steps.meta.outputs.ref }}
META_DRY_RUN: ${{ steps.meta.outputs.dry_run }}
META_FORCE_REPUBLISH: ${{ steps.meta.outputs.force_republish }}
META_CHECKOUT_REF: ${{ steps.meta.outputs.checkout_ref }}
META_TARGET_SHA: ${{ steps.meta.outputs.target_sha }}
META_IS_TAG: ${{ steps.meta.outputs.is_tag }}
META_RELEASE_DOCKER: ${{ steps.meta.outputs.release_docker }}
run: |
{
echo "## Release Metadata"
echo "- **Tag**: \`$META_TAG\`"
echo "- **Version**: \`$META_VERSION\`"
echo "- **Ref**: \`$META_REF\`"
echo "- **Dry Run**: \`$META_DRY_RUN\`"
echo "- **Force Republish**: \`$META_FORCE_REPUBLISH\`"
echo "- **Checkout Ref**: \`$META_CHECKOUT_REF\`"
echo "- **Target SHA**: \`$META_TARGET_SHA\`"
echo "- **Is Tag**: \`$META_IS_TAG\`"
echo "- **Release Docker**: \`$META_RELEASE_DOCKER\`"
} >> "$GITHUB_STEP_SUMMARY"
check-docker:
name: Check if Docker image tag exists
needs: prepare
if: ${{ needs.prepare.outputs.release_docker == 'true' }}
runs-on: ubuntu-latest
permissions:
contents: read
packages: read
outputs:
core_exists: ${{ steps.core.outputs.exists }}
full_exists: ${{ steps.full.outputs.exists }}
cli_exists: ${{ steps.cli.outputs.exists }}
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
ref: ${{ needs.prepare.outputs.tag }}
- name: Log in to GitHub Container Registry
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Check core image tag
id: core
env:
DOCKER_TAG: ghcr.io/kreuzberg-dev/kreuzberg:${{ needs.prepare.outputs.version }}-core
SUMMARY_LABEL: core
run: scripts/publish/check-docker-tag.sh
- name: Check full image tag
id: full
env:
DOCKER_TAG: ghcr.io/kreuzberg-dev/kreuzberg:${{ needs.prepare.outputs.version }}
SUMMARY_LABEL: full
run: scripts/publish/check-docker-tag.sh
- name: Check CLI image tag
id: cli
env:
DOCKER_TAG: ghcr.io/kreuzberg-dev/kreuzberg-cli:${{ needs.prepare.outputs.version }}
SUMMARY_LABEL: cli
run: scripts/publish/check-docker-tag.sh
publish-docker:
name: Publish Docker image (${{ matrix.variant }})
needs:
- prepare
- check-docker
runs-on: ubuntu-latest
timeout-minutes: 360
permissions:
contents: read
packages: write
strategy:
fail-fast: false
matrix:
include:
- variant: core
dockerfile: docker/Dockerfile.core
image: ghcr.io/kreuzberg-dev/kreuzberg
tag_suffix: "-core"
extra_tag: "core"
- variant: full
dockerfile: docker/Dockerfile.full
image: ghcr.io/kreuzberg-dev/kreuzberg
tag_suffix: ""
extra_tag: "latest"
- variant: cli
dockerfile: docker/Dockerfile.cli
image: ghcr.io/kreuzberg-dev/kreuzberg-cli
tag_suffix: ""
extra_tag: "latest"
if: ${{ needs.prepare.outputs.release_docker == 'true' }}
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: ${{ needs.prepare.outputs.checkout_ref }}
fetch-depth: 0
submodules: recursive
- name: Free up disk space
uses: kreuzberg-dev/actions/free-disk-space-linux@v1
- name: Ensure target commit
if: ${{ needs.prepare.outputs.target_sha != '' }}
run: git checkout --progress --force ${{ needs.prepare.outputs.target_sha }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Skip because tag already exists
if: ${{ needs.prepare.outputs.force_republish != 'true' && ((matrix.variant == 'core' && needs.check-docker.outputs.core_exists == 'true') || (matrix.variant == 'full' && needs.check-docker.outputs.full_exists == 'true') || (matrix.variant == 'cli' && needs.check-docker.outputs.cli_exists == 'true')) }}
run: echo "Docker tag already exists for variant ${{ matrix.variant }}; skipping publish." >> "$GITHUB_STEP_SUMMARY"
- name: Build AMD64 test image
if: ${{ needs.prepare.outputs.force_republish == 'true' || (matrix.variant == 'core' && needs.check-docker.outputs.core_exists != 'true') || (matrix.variant == 'full' && needs.check-docker.outputs.full_exists != 'true') || (matrix.variant == 'cli' && needs.check-docker.outputs.cli_exists != 'true') }}
run: docker build -f ${{ matrix.dockerfile }} --build-arg ONNXRUNTIME_VERSION=${{ env.ORT_VERSION }} -t kreuzberg-publish:${{ matrix.variant }}-test .
- name: Run Docker tests
if: ${{ needs.prepare.outputs.force_republish == 'true' || (matrix.variant == 'core' && needs.check-docker.outputs.core_exists != 'true') || (matrix.variant == 'full' && needs.check-docker.outputs.full_exists != 'true') || (matrix.variant == 'cli' && needs.check-docker.outputs.cli_exists != 'true') }}
run: python3 scripts/ci/docker/test_docker.py --image kreuzberg-publish:${{ matrix.variant }}-test --variant ${{ matrix.variant }} --verbose
- name: Log in to GitHub Container Registry
if: ${{ needs.prepare.outputs.dry_run != 'true' && (needs.prepare.outputs.force_republish == 'true' || (matrix.variant == 'core' && needs.check-docker.outputs.core_exists != 'true') || (matrix.variant == 'full' && needs.check-docker.outputs.full_exists != 'true') || (matrix.variant == 'cli' && needs.check-docker.outputs.cli_exists != 'true')) }}
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract Docker metadata
if: ${{ needs.prepare.outputs.dry_run != 'true' && (needs.prepare.outputs.force_republish == 'true' || (matrix.variant == 'core' && needs.check-docker.outputs.core_exists != 'true') || (matrix.variant == 'full' && needs.check-docker.outputs.full_exists != 'true') || (matrix.variant == 'cli' && needs.check-docker.outputs.cli_exists != 'true')) }}
id: docker_meta
uses: docker/metadata-action@v6
with:
images: ${{ matrix.image }}
tags: |
type=raw,value=${{ needs.prepare.outputs.version }}${{ matrix.tag_suffix }}
type=raw,value=${{ matrix.extra_tag }}
- name: Build and push image
if: ${{ needs.prepare.outputs.dry_run != 'true' && (needs.prepare.outputs.force_republish == 'true' || (matrix.variant == 'core' && needs.check-docker.outputs.core_exists != 'true') || (matrix.variant == 'full' && needs.check-docker.outputs.full_exists != 'true') || (matrix.variant == 'cli' && needs.check-docker.outputs.cli_exists != 'true')) }}
uses: docker/build-push-action@v7
with:
context: .
file: ${{ matrix.dockerfile }}
push: true
build-args: |
ONNXRUNTIME_VERSION=${{ env.ORT_VERSION }}
tags: ${{ steps.docker_meta.outputs.tags }}
labels: |
${{ steps.docker_meta.outputs.labels }}
org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
org.opencontainers.image.description=Kreuzberg document intelligence - ${{ matrix.variant }} variant
org.opencontainers.image.licenses=MIT
platforms: linux/amd64,linux/arm64
cache-from: type=gha
cache-to: type=gha,mode=max,scope=publish-docker-${{ matrix.variant }}
- name: Docker dry-run summary
if: ${{ needs.prepare.outputs.dry_run == 'true' }}
env:
IMAGE: ${{ matrix.image }}
VERSION: ${{ needs.prepare.outputs.version }}
TAG_SUFFIX: ${{ matrix.tag_suffix }}
run: scripts/publish/docker/dry-run-summary.sh
- name: Clean up local Docker images
if: ${{ always() }}
run: docker rmi kreuzberg-publish:${{ matrix.variant }}-test || true

108
.github/workflows/publish-helm.yaml vendored Normal file
View File

@@ -0,0 +1,108 @@
name: Publish Helm Chart
on:
workflow_dispatch:
inputs:
tag:
description: "Release tag to build (e.g., v4.3.6)"
required: true
type: string
dry_run:
description: "Prepare artifacts without publishing"
required: false
type: boolean
default: false
release:
types: [published]
repository_dispatch:
types: [publish-helm]
concurrency:
group: ${{ github.workflow }}-${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.tag) || github.ref || github.run_id }}
cancel-in-progress: false
permissions:
contents: read
jobs:
publish-helm:
name: Publish Helm chart to GHCR
if: ${{ github.event_name != 'release' || !github.event.release.prerelease }}
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Resolve version
id: meta
run: |
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
TAG="${{ inputs.tag }}"
elif [[ "${{ github.event_name }}" == "release" ]]; then
TAG="${{ github.event.release.tag_name }}"
elif [[ "${{ github.event_name }}" == "repository_dispatch" ]]; then
TAG="${{ github.event.client_payload.tag }}"
fi
VERSION="${TAG#v}"
DRY_RUN="${{ inputs.dry_run || 'false' }}"
{
echo "tag=${TAG}"
echo "version=${VERSION}"
echo "dry_run=${DRY_RUN}"
} >> "$GITHUB_OUTPUT"
{
echo "## Helm Publish Metadata"
echo "- **Tag**: \`${TAG}\`"
echo "- **Version**: \`${VERSION}\`"
echo "- **Dry Run**: \`${DRY_RUN}\`"
} >> "$GITHUB_STEP_SUMMARY"
- name: Setup Helm
uses: azure/setup-helm@v5
- name: Lint chart
run: helm lint --strict charts/kreuzberg/
- name: Update Chart.yaml version
run: |
sed -i "s/^version:.*/version: ${{ steps.meta.outputs.version }}/" charts/kreuzberg/Chart.yaml
sed -i "s/^appVersion:.*/appVersion: \"${{ steps.meta.outputs.version }}\"/" charts/kreuzberg/Chart.yaml
{
echo "### Chart.yaml"
echo '```yaml'
cat charts/kreuzberg/Chart.yaml
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Package chart
run: |
helm package charts/kreuzberg/ --destination .helm-packages/
echo "### Packaged" >> "$GITHUB_STEP_SUMMARY"
ls -lh .helm-packages/ >> "$GITHUB_STEP_SUMMARY"
- name: Log in to GitHub Container Registry
if: ${{ steps.meta.outputs.dry_run != 'true' }}
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Push chart to GHCR
if: ${{ steps.meta.outputs.dry_run != 'true' }}
run: |
helm push .helm-packages/kreuzberg-${{ steps.meta.outputs.version }}.tgz oci://ghcr.io/kreuzberg-dev/charts
echo "### Published" >> "$GITHUB_STEP_SUMMARY"
echo "Chart pushed to \`oci://ghcr.io/kreuzberg-dev/charts/kreuzberg:${{ steps.meta.outputs.version }}\`" >> "$GITHUB_STEP_SUMMARY"
- name: Dry-run summary
if: ${{ steps.meta.outputs.dry_run == 'true' }}
run: |
echo "### Dry Run" >> "$GITHUB_STEP_SUMMARY"
echo "Would have pushed \`kreuzberg-${{ steps.meta.outputs.version }}.tgz\` to \`oci://ghcr.io/kreuzberg-dev/charts\`" >> "$GITHUB_STEP_SUMMARY"

46
.github/workflows/publish-pubdev.yaml vendored Normal file
View File

@@ -0,0 +1,46 @@
name: Publish pub.dev
# pub.dev OIDC trusted publishing rejects tokens originating from `release`
# events; only `push` and `workflow_dispatch` are accepted.
#
# Because the kreuzberg Dart package embeds platform-specific native binaries
# (Android JNI, iOS XCFramework, server libs for linux/macos/windows), we
# cannot just rebuild here — those artifacts are produced by the main
# `publish.yaml` workflow. Instead, the main workflow's `trigger-pubdev` job
# dispatches this workflow with the run_id of the main workflow, and this
# workflow downloads the `dart-package-assembled` artifact from that run.
#
# One-time setup: on pub.dev → kreuzberg package → Admin → Automated publishing,
# set the workflow path to `.github/workflows/publish-pubdev.yaml`.
on:
workflow_dispatch:
inputs:
run_id:
description: "GitHub Actions run ID of publish.yaml that produced the dart-package-assembled artifact"
required: true
type: string
permissions:
contents: read
id-token: write
actions: read
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
jobs:
publish-pub:
name: Publish pub.dev
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v8.0.1
with:
name: dart-package-assembled
path: packages/dart
run-id: ${{ inputs.run_id }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- uses: kreuzberg-dev/actions/publish-pub@v1
with:
package-dir: packages/dart

2345
.github/workflows/publish.yaml vendored Normal file
View File

File diff suppressed because it is too large Load Diff

10
.github/workflows/validate-issues.yml vendored Normal file
View File

@@ -0,0 +1,10 @@
name: Validate Issues
on:
issues:
types: [opened, edited]
jobs:
validate:
uses: kreuzberg-dev/actions/.github/workflows/reusable-validate-issues.yml@v1
secrets: inherit

10
.github/workflows/validate-pr.yml vendored Normal file
View File

@@ -0,0 +1,10 @@
name: Validate PR
on:
pull_request:
types: [opened, edited, synchronize]
jobs:
validate:
uses: kreuzberg-dev/actions/.github/workflows/reusable-validate-pr.yml@v1
secrets: inherit

296
.gitignore vendored Normal file
View File

@@ -0,0 +1,296 @@
# Build artifacts
target/
build/
dist/
# Ad-hoc PDF/image debug binaries
/check_pdf_oxide
/check_*
# Hand-authored WASM JS distribution files (intentionally committed)
!crates/kreuzberg-wasm/dist/index.js
!crates/kreuzberg-wasm/dist/extraction/files.js
!crates/kreuzberg-wasm/dist/ocr/enabler.js
.lycheecache
dist-musl/
*.so
*.dylib
*.dll
*.pdb
*.node
*.whl
*.tar.gz
*.gem
*.jar
wheels/
**/bin/
!**/src/bin/
**/obj/
# Rust
*.rs.bk
*.profraw
rust-coverage.lcov
rust_out
# Python
*.py[cod]
*$py.class
*.egg-info/
__pycache__/
.venv/
.mypy_cache/
.ruff_cache/
.pytest_cache/
.hypothesis/
.coverage
.coverage*
htmlcov/
share/python-wheels/
prompt_template.egg-info/
packages/python/kreuzberg/kreuzberg-cli
# Node.js / WASM
node_modules/
*.tsbuildinfo
npm-debug.log*
pnpm-debug.log*
yarn-debug.log*
yarn-error.log*
# NAPI-RS generated files (index.d.ts is committed for TypeScript type resolution)
crates/kreuzberg-node/index.js
crates/kreuzberg-node/*.tgz
.wrangler/
.pnpm-store/
# Ruby
packages/ruby/.bundle/
packages/ruby/vendor/
!packages/ruby/vendor/rb-sys/
!vendor/rb-sys/build/
!vendor/rb-sys/build/**
*.Cache
# PHP
vendor/
php-kreuzberg.ini
packages/php/php-kreuzberg.ini
packages/php/tests.xml
packages/php/var/
.php-cs-fixer.cache
# Java / C# / .NET
*.suo
*.user
*.cscfg
*.cobertura.xml
TestResults/
# Generated test apps (e2e-generator --mode published)
test_apps/
# R
packages/r/src/.cargo/
.Rhistory
.RData
# Elixir
.nox/
# Go
packages/go/v*/install
# C FFI
crates/kreuzberg-ffi/tests/c/test_*
!crates/kreuzberg-ffi/tests/c/test_*.c
crates/kreuzberg-ffi/tests/c/*.o
crates/kreuzberg-ffi/tests/c/*.dSYM/
e2e/c/test_*
!e2e/c/test_*.c
e2e/c/*.o
e2e/c/*.dSYM/
# Generated pkg-config
*.pc
# Coverage
coverage/
coverage.info
coverage.json
coverage.lcov
coverage.out
coverage.xml
packages/*/.coverage
packages/*/coverage.json
packages/*/coverage.out
packages/*/htmlcov/
packages/*/test-results/
packages/*/TestResults/
# IDE / Editor
.idea/
.vscode/
.run/
.DS_store
# CI / Tools
.cache/
.tmp/
*temp/
*.temp
*.tmp
*.log
.mvn/
.tox/
.artifacts/
.remote-cache/
.rumdl_cache/
docker-logs/
# Benchmarks
benchmark-results/
benchmark_results.json
benchmarks/results/
!benchmarks/baselines/
aggregated-results/
results/
reports/
tests/benchmarks/results/
profiling-results/
profiling-results-*/
# Local perf-iteration artifacts (flamegraphs/ is committed; intermediate outputs are not)
profile.json.gz
profile.pb.gz
*.heap
bench/
# Docs
docs/_build/
docs/build/
docs/benchmarks/charts/
site/
# Elixir build artifacts
packages/elixir/_build/
packages/elixir/deps/
packages/elixir/native/kreuzberg_nif/Cargo.lock
e2e/elixir/_build/
e2e/elixir/deps/
e2e/elixir/mix.lock
e2e/elixir/config/
test_documents/_build/
test_documents/deps/
# Maven plugin artifacts
*.versionsBackup
# Gradle (Kotlin e2e)
e2e/kotlin/.gradle/
e2e/kotlin/build/
e2e/kotlin_android/.gradle/
e2e/kotlin_android/build/
# Ruby native build artifacts
packages/ruby/lib/*.bundle
packages/ruby/tmp/
# Tests
test_report.json
tests/e2e/logs/
tests/e2e/test_report.json
temp-test-*
packages/csharp/test_html_debug.cs
# AI / MCP / Agent config
.claude/
.codex/
.gemini/
.kreuzberg/
.mcp.json
.playwright-mcp/
.fastembed_cache/
*/.fastembed_cache/
AGENTS.md
CLAUDE.md
GEMINI.md
.cursorrules
.windsurfrules
[Tt][Oo][Dd][Oo]*
# Git worktrees
.worktrees/
# Misc
.env
output.txt
examples.txt
requirements.txt
docker-compose.yaml
/tmp/kreuzberg-docker-test-results.json
tools/benchmark-harness/datasets/
tools/benchmark-harness/vendored/docling/md/iso_21111_10.md
visualizations/
# Additional generated artifacts
.alef/
*.pyd
.gems/
vendor/bundle/
*.h.bak
*.test
*.class
bin/
obj/
*.nupkg
pkg/
# Local dev artifacts
docs/demo-dev.html
docs/serve.json
demo-loaded.png
scratch/
.php-cs-fixer.*
e2e/zig/.zig-cache/
# Swift e2e build artifacts (Swift Package Manager)
e2e/swift/.build/
e2e/swift/.swiftpm/
e2e/swift/Package.resolved
e2e/swift_e2e/.build/
e2e/swift_e2e/.swiftpm/
e2e/swift_e2e/Package.resolved
# Dart e2e build artifacts
e2e/dart/.dart_tool/
e2e/dart/build/
e2e/dart/pubspec.lock
# Dart FRB codegen cache (incremental build optimization)
packages/dart/rust/.frb_codegen_hash
# Go e2e build artifacts
e2e/go/vendor/
erl_crash.dump
.gradle/
packages/go/v5/.lib/
.build/
packages/r/src/*.o
# BEGIN ai-rulez (DO NOT EDIT - managed by ai-rulez)
.agents/
.cursor/
.github/agents/
.github/commands/
.github/copilot-instructions.md
.github/skills/
# END ai-rulez

3
.gitmodules vendored Normal file
View File

@@ -0,0 +1,3 @@
[submodule "test_documents"]
path = test_documents
url = https://github.com/kreuzberg-dev/test_documents.git

157
.golangci.yml Normal file
View File

@@ -0,0 +1,157 @@
version: "2"
run:
timeout: 5m
issues-exit-code: 1
tests: true
concurrency: 4
modules-download-mode: readonly
allow-serial-runners: false
allow-parallel-runners: true
linters:
default: none
enable:
- errcheck
- govet
- ineffassign
- staticcheck
- unused
- revive
- gocyclo
- goconst
- gocritic
- gosec
- misspell
- nakedret
settings:
errcheck:
check-type-assertions: true
check-blank: true
exclude-functions:
- (net/http.ResponseWriter).Write
- (io.Closer).Close
- fmt.Fprintf
- fmt.Printf
- fmt.Println
- os.Setenv
- os.Unsetenv
goconst:
min-len: 3
min-occurrences: 3
gocyclo:
min-complexity: 50
gocritic:
disabled-checks:
- dupSubExpr
gosec:
excludes:
- G101 # ~keep hardcoded credentials check (too many false positives)
govet:
enable-all: true
disable:
- shadow
misspell:
locale: US
nakedret:
max-func-lines: 30
revive:
confidence: 0.8
severity: warning
enable-all-rules: false
rules:
- name: blank-imports
- name: context-keys-type
- name: time-naming
- name: var-declaration
- name: unexported-return
- name: errorf
- name: context-as-argument
- name: dot-imports
- name: error-return
- name: error-strings
- name: error-naming
- name: if-return
- name: increment-decrement
- name: var-naming
- name: range
- name: receiver-naming
- name: indent-error-flow
- name: exported
disabled: true
- name: package-comments
disabled: true
exclusions:
generated: lax
rules:
- linters:
- goconst
path: _test\.go
- linters:
- gocyclo
path: _test\.go
- linters:
- gosec
path: _test\.go
- linters:
- revive
path: _test\.go
text: "context-as-argument"
- linters:
- goconst
- revive
- errcheck
- govet
path: _test\.go
text: "unusedwrite:"
- linters:
- govet
text: "unsafeptr:"
- linters:
- govet
text: "fieldalignment:"
- linters:
- errcheck
path: _test\.go
paths:
- vendor
- build
- deployments
- third_party$
- builtin$
- examples$
- tools/benchmark-harness/scripts
issues:
max-issues-per-linter: 0
max-same-issues: 0
uniq-by-line: true
new: false
exclude:
- 'Error return value of `\(\*github\.com/goccy/go-json\.Encoder\)\.Encode` is not checked'
- 'Error return value of `w\.Write` is not checked'
- 'Error return value of `resp\.Body\.Close` is not checked'
- 'Error return value of `res\.Body\.Close` is not checked'
- 'Error return value of `r\.Body\.Read` is not checked'
- 'Error return value of `os\.Setenv` is not checked'
- 'Error return value of `os\.Unsetenv` is not checked'
- 'shadow: declaration of "err" shadows declaration'
- "unusedwrite: unused write to field"
- 'Error return value of `c\.provider\.Delete` is not checked'
- 'Error return value of `provider\.Close` is not checked'
- 'Error return value of `natsClient\.Close` is not checked'
- 'Error return value of `cacheProvider\.Close` is not checked'
- 'Error return value of `processor\.Close` is not checked'
- 'Error return value of `sub\.Unsubscribe` is not checked'
- 'Error return value of `json\.Marshal` is not checked'
- 'Error return value of `strconv\.'
- 'Error return value of `fmt\.Sscanf` is not checked'
- "Error return value is not checked"
formatters:
exclusions:
generated: lax
paths:
- third_party$
- builtin$
- examples$

61
.lychee.toml Normal file
View File

@@ -0,0 +1,61 @@
# Configuration for the lychee link checker.
# https://github.com/lycheeverse/lychee
# root_dir for resolving root-relative links is set dynamically in
# `.task/tools/docs.yml` via `--root-dir "$PWD/docs"` — must be absolute,
# so we do not hardcode it here.
# Exclude common patterns that produce false positives
exclude = [
# Localhost / loopback addresses
"localhost",
"127\\.0\\.0\\.1",
"0\\.0\\.0\\.0",
# Placeholder / example domains
"example\\.com",
"example\\.org",
# Template variables (e.g. ${var}, {{ var }})
"\\$\\{",
"\\{\\{",
# GitHub edit links (require auth)
"github\\.com/.*/edit/",
# GitHub raw links for new/renamed files (404 until pushed)
"github\\.com/.*/raw/",
# Docs site self-references (404 until deployed, or unreachable from CI)
"kreuzberg\\.dev",
# PyPI project pages (rate-limited)
"pypi\\.org/project/",
# crates.io (rate-limited)
"crates\\.io/crates/",
# npm (rate-limited)
"npmjs\\.com/package/",
]
# Accept these HTTP status codes as valid
accept = [200, 204, 301, 302, 429]
# Connection timeout in seconds
timeout = 30
# Max retries per link
max_retries = 3
# Max concurrent requests
max_concurrency = 10
# Do not check email addresses
include_mail = false
# Do not require HTTPS
require_https = false
# Cache results to speed up repeated runs
cache = true

28
.oxfmtrc.json Normal file
View File

@@ -0,0 +1,28 @@
{
"printWidth": 120,
"useTabs": true,
"tabWidth": 4,
"semi": true,
"singleQuote": false,
"trailingComma": "all",
"ignorePatterns": [
"**/*.md",
"**/*.mdx",
"**/dist/**",
"**/coverage/**",
"**/node_modules/**",
"**/pkg/**",
"**/.next/**",
"**/*.min.js"
],
"overrides": [
{
"files": ["**/*.json", "**/*.jsonc"],
"options": {
"useTabs": true,
"tabWidth": 4,
"trailingComma": "none"
}
}
]
}

56
.oxlintrc.json Normal file
View File

@@ -0,0 +1,56 @@
{
"plugins": ["typescript"],
"env": {
"browser": true,
"esnext": true,
"node": true
},
"globals": {
"describe": "readonly",
"it": "readonly",
"test": "readonly",
"expect": "readonly",
"beforeEach": "readonly",
"afterEach": "readonly",
"beforeAll": "readonly",
"afterAll": "readonly",
"vi": "readonly"
},
"categories": {
"correctness": "error",
"suspicious": "error",
"perf": "warn"
},
"ignorePatterns": [
"**/_generated/**",
"**/dist/**",
"**/coverage/**",
"**/node_modules/**",
"**/pkg/**",
"**/.next/**",
"**/*.min.js",
"docs/snippets/**/*.ts"
],
"rules": {
"no-console": "off",
"no-debugger": "error",
"prefer-const": "error",
"eqeqeq": ["error", "always", { "null": "ignore" }],
"no-unused-vars": "off",
"no-underscore-dangle": "off",
"@typescript-eslint/no-unused-vars": "warn",
"@typescript-eslint/no-explicit-any": "error",
"@typescript-eslint/no-non-null-assertion": "error",
"@typescript-eslint/consistent-type-imports": "error"
},
"overrides": [
{
"files": ["**/*.spec.ts", "**/*.spec.tsx", "**/*.test.ts", "**/*.test.tsx", "**/tests/**"],
"rules": {
"@typescript-eslint/no-explicit-any": "off",
"@typescript-eslint/no-non-null-assertion": "off",
"no-unused-vars": "off"
}
}
]
}

201
.pre-commit-config.yaml Normal file
View File

@@ -0,0 +1,201 @@
default_install_hook_types:
- pre-commit
- commit-msg
default_stages:
- pre-commit
exclude: ^docs/snippets/|vendor/|node_modules/|target/|invalid*|dist/|artifacts/|test_documents/|scripts/ci/|^e2e/|^packages/|^crates/kreuzberg-(py|node|ffi|php|wasm)/|^docs/reference/api-|\.cache/|\.venv/|rust-vendor/
repos:
- repo: https://github.com/Goldziher/gitfluff
rev: v0.8.0
hooks:
- id: gitfluff-lint
args:
- --write
stages:
- commit-msg
- repo: https://github.com/kreuzberg-dev/pre-commit-hooks
rev: v1.2.3
hooks:
- id: ai-rulez-generate
- id: gh-actions-updater
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-merge-conflict
- id: check-added-large-files
exclude: uv.lock|docs/benchmarks/app/
- id: detect-private-key
- id: check-json
- id: check-yaml
args:
- --allow-multiple-documents
- --unsafe
exclude: ^charts/
- id: check-toml
- id: check-case-conflict
- id: pyproject-fmt
- id: ruff
- id: ruff-format
- id: cargo-sort
exclude: ^(crates/kreuzberg-(wasm|node|py|php|ffi)|packages/(dart|swift)/rust)/Cargo\.toml$
- id: cargo-fmt
args:
- --all
- id: cargo-check
args:
- --workspace
- --exclude=kreuzberg-ffi
- --exclude=kreuzberg-py
- --exclude=kreuzberg-node
- --exclude=kreuzberg-php
- --exclude=kreuzberg-wasm
- --exclude=kreuzberg-dart
- --exclude=kreuzberg-swift
- --all-features
- --all-targets
- id: cargo-clippy
args:
- --fix
- --allow-dirty
- --allow-staged
- --workspace
- --exclude=kreuzberg-ffi
- --exclude=kreuzberg-py
- --exclude=kreuzberg-node
- --exclude=kreuzberg-php
- --exclude=kreuzberg-wasm
- --exclude=kreuzberg-dart
- --exclude=kreuzberg-swift
- --all-features
- --all-targets
- --
- -D
- warnings
- id: cargo-machete
args:
- crates/
- packages/
- tools/
exclude: ^e2e/
# - id: alef-docs-fresh # TEMP DISABLED — Step 0 lockdown active
- id: cargo-deny
args:
- check
- id: shfmt
args:
- -w
- -i
- '2'
- id: shellcheck
args:
- -x
- id: clang-format
args:
- --style=file
files: ^crates/kreuzberg-ffi/tests/c/
- id: clang-tidy
files: ^crates/kreuzberg-ffi/tests/c/
- id: checkstyle
args:
- -c
- packages/java/checkstyle.xml
- -p
- packages/java/checkstyle.properties
exclude: ^(\.mvn/wrapper/|e2e/|tools/e2e-generator/e2e/|crates/kreuzberg-wasm/e2e/|packages/java/src/)
- id: javadoc-lint
files: ^packages/java/src/main/.*\.java$
- id: java-verify
files: ^packages/java/
- id: mypy
files: ^(lib|src|kreuzberg|tools)/.*\.py$
exclude: e2e/|tests/|scripts/
- id: pydocstyle
files: ^packages/python/.*\.py$
exclude: ^packages/python/(tests/|.*_native\.py$)
- id: go-fmt
exclude: ^(e2e/|test_apps/|tools/)
- id: golangci-lint
exclude: ^(e2e/|test_apps/|tools/)
- id: govulncheck
exclude: ^(e2e/|test_apps/|tools/)
- id: rubocop
files: ^packages/ruby/.*\.rb$
- id: rubocop-lint
files: ^packages/ruby/.*\.(rb|rbs)$
- id: steep
files: ^packages/ruby/.*\.(rb|rbs)$
- id: dotnet-format
files: ^packages/csharp/.*\.cs$
- id: dotnet-format-check
files: ^packages/csharp/.*\.cs$
- id: cs-xmldoc-lint
files: ^packages/csharp/Kreuzberg/.*\.cs$
- id: php-cs-fixer
files: ^packages/php/.*\.php$
- id: phpstan
files: ^packages/php/.*\.php$
- id: phpdoc-lint
files: ^packages/php/src/.*\.php$
- id: mix-format
files: ^packages/elixir/
- id: mix-credo
files: ^packages/elixir/
- id: roxygen2-check
files: ^packages/r/R/.*\.R$
- id: air-format
files: ^packages/r/.*\.[Rr]$
- id: air-check
files: ^packages/r/.*\.[Rr]$
- id: lintr
files: ^packages/r/.*\.[Rr]$
- id: ktfmt
args:
- --kotlinlang-style
files: ^packages/kotlin-android/.*\.kts?$
- id: detekt
args:
- --build-upon-default-config
files: ^packages/kotlin-android/.*\.kts?$
- id: ktlint
files: ^packages/kotlin-android/.*\.kts?$
- id: swift-format
files: ^packages/swift/.*\.swift$
- id: swiftlint
files: ^packages/swift/.*\.swift$
- id: dart-format
files: ^packages/dart/.*\.dart$
- id: dart-analyze
files: ^packages/dart/.*\.dart$
- id: dart-doc-lint
files: ^packages/dart/lib/.*\.dart$
exclude: ^packages/dart/lib/src/rust/
- id: zig-fmt
files: ^packages/zig/.*\.zig$
- id: zig-build-check
files: ^packages/zig/.*\.zig$
- id: helm-lint
files: ^charts/.*/(Chart\.yaml|templates/.+\.ya?ml)$
- id: kubeconform
args:
- -strict
- -summary
files: ^charts/.*/templates/.+\.ya?ml$
exclude: ^charts/.*/(values|Chart)\.ya?ml$
- id: textlint
files: ^docs/.*\.md$
exclude: ^docs/(reference/|api-|snippets/|migration/)
args:
- --fix
- --config
- .textlintrc.json
- id: oxlint
- id: oxfmt
- id: cppcheck
- id: rumdl-fmt
exclude: ^templates/
- id: cpd
# - id: alef-verify # TEMP DISABLED — Step 0 lockdown active
# - id: alef-sync-versions # TEMP DISABLED — Step 0 lockdown active
- id: actionlint
- id: typos
args:
- --force-exclude

47
.rumdl.toml Normal file
View File

@@ -0,0 +1,47 @@
# rumdl — Rust-based markdown linter
# https://github.com/rvben/rumdl
respect-gitignore = true
exclude = [
"node_modules",
"target",
"dist",
"vendor",
# Snippets carry alef snippet-runner directives (e.g. <!-- snippet:skip -->)
# that must sit directly above the fence. Reflowing them with rumdl breaks
# the directive contract.
"docs/snippets",
]
# MD013: Disable line-length enforcement (tables and code blocks can be long)
# MD041: Don't require first line to be an H1
# MD046: Disable code block style — Zensical tabs/admonitions indent fenced
# blocks, which rumdl misidentifies as indented code blocks
# MD051: Disable cross-file link fragment checking (incompatible with Zensical
# HTML processing — Zensical strips <span> tags from heading IDs)
# MD013: Line length (tables/code can be long)
# MD033: Inline HTML (Zensical uses HTML extensively)
# MD036: Emphasis as heading (intentional style in docs/READMEs)
# MD041: First line H1 not required
# MD046: Code block style (Zensical tabs indent fenced blocks)
# MD051: Link fragment checking (incompatible with Zensical anchor generation)
# MD076: Blank lines between list items (intentional formatting in READMEs)
# MD030: Spaces after list markers (MkDocs Material grid cards require 3 spaces)
# MD035: Horizontal rule style (grid cards use indented --- as card separators)
disable = [
"MD012",
"MD013",
"MD024",
"MD030",
"MD033",
"MD035",
"MD036",
"MD041",
"MD046",
"MD051",
"MD076",
]
# MD024: Allow duplicate heading names if they are not siblings
[MD024]
siblings_only = true

2
.sdkmanrc Normal file
View File

@@ -0,0 +1,2 @@
java=25.0.2-tem
maven=3.9.11

3
.shellcheckrc Normal file
View File

@@ -0,0 +1,3 @@
# Disable SC1091: Not following sourced files
# This is expected for shared utility libraries in scripts/lib/
disable=SC1091

89
.task/config/platforms.yml Normal file
View File

@@ -0,0 +1,89 @@
version: "3"
internal: true
includes:
vars: ./vars.yml
vars:
EXE_EXT:
sh: |
if [[ "$OSTYPE" == "msys" ]] || [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then
echo ".exe"
else
echo ""
fi
LIB_EXT:
sh: |
if [[ "$OSTYPE" == "darwin"* ]]; then
echo "dylib"
elif [[ "$OSTYPE" == "msys" ]] || [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then
echo "dll"
else
echo "so"
fi
LIB_PREFIX:
sh: |
if [[ "$OSTYPE" == "msys" ]] || [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then
echo ""
else
echo "lib"
fi
RUST_TARGET:
sh: |
ARCH=$(uname -m)
OS_TYPE="$OSTYPE"
case "$ARCH" in
x86_64|x64)
ARCH_STR="x86_64"
;;
aarch64|arm64)
ARCH_STR="aarch64"
;;
armv7l|armv7)
ARCH_STR="armv7"
;;
*)
ARCH_STR="$ARCH"
;;
esac
if [[ "$OS_TYPE" == "darwin"* ]]; then
echo "${ARCH_STR}-apple-darwin"
elif [[ "$OS_TYPE" == "linux-gnu"* ]] || [[ "$OS_TYPE" == "linux"* ]]; then
echo "${ARCH_STR}-unknown-linux-gnu"
elif [[ "$OS_TYPE" == "msys" ]] || [[ "$OS_TYPE" == "cygwin" ]] || [[ "$OS_TYPE" == "win32" ]]; then
echo "${ARCH_STR}-pc-windows-msvc"
else
echo "${ARCH_STR}-unknown-unknown"
fi
IS_WINDOWS: "{{.IS_WINDOWS}}"
IS_MACOS: "{{.IS_MACOS}}"
IS_LINUX: "{{.IS_LINUX}}"
RUBY_FULL_PATH:
sh: |
if command -v ruby >/dev/null 2>&1; then
command -v ruby
elif [[ "$OSTYPE" == "darwin"* ]] && [[ -f "/opt/homebrew/opt/ruby/bin/ruby" ]]; then
echo "/opt/homebrew/opt/ruby/bin/ruby"
else
echo "ruby"
fi
CARGO_BIN:
sh: command -v cargo 2>/dev/null || echo "cargo"
RUSTC_BIN:
sh: command -v rustc 2>/dev/null || echo "rustc"
SHELL_EXT:
sh: |
if [[ "$OSTYPE" == "msys" ]] || [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then
echo ".ps1"
else
echo ".sh"
fi

118
.task/config/vars.yml Normal file
View File

@@ -0,0 +1,118 @@
version: "3"
internal: true
vars:
# Version extraction from Cargo.toml (workspace.package.version)
VERSION:
sh: grep -m 1 'version = ' Cargo.toml | sed 's/version = "\(.*\)"/\1/'
# Build profile (dev/release/ci) - default to release
BUILD_PROFILE: '{{.BUILD_PROFILE | default "release"}}'
# Kreuzberg-specific versions
ORT_VERSION: "1.24.1"
# Toolchain versions
GOLANGCI_LINT_VERSION: "latest"
# Logging
RUST_LOG: "info"
# Root project directories (absolute paths)
ROOT: "{{.ROOT_DIR}}"
CRATES_DIR: "{{.ROOT_DIR}}/crates"
PACKAGES_DIR: "{{.ROOT_DIR}}/packages"
SCRIPTS_DIR: "{{.ROOT_DIR}}/scripts"
TOOLS_DIR: "{{.ROOT_DIR}}/tools"
TARGET_DIR: "{{.ROOT_DIR}}/target"
# OS Detection
OS:
sh: |
case "$(uname -s 2>/dev/null || echo 'unknown')" in
Darwin*)
echo "darwin"
;;
Linux*)
echo "linux"
;;
MINGW*|MSYS*|CYGWIN*)
echo "windows"
;;
*)
if [[ "$OSTYPE" == "darwin"* ]]; then
echo "darwin"
elif [[ "$OSTYPE" == "linux-gnu"* ]] || [[ "$OSTYPE" == "linux"* ]]; then
echo "linux"
elif [[ "$OSTYPE" == "msys" ]] || [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then
echo "windows"
else
echo "unknown"
fi
;;
esac
# OS Boolean helpers
IS_WINDOWS:
sh: |
if [[ "$OSTYPE" == "msys" ]] || [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then
echo "true"
else
echo "false"
fi
IS_MACOS:
sh: |
if [[ "$OSTYPE" == "darwin"* ]]; then
echo "true"
else
echo "false"
fi
IS_LINUX:
sh: |
if [[ "$OSTYPE" == "linux-gnu"* ]] || [[ "$OSTYPE" == "linux"* ]]; then
echo "true"
else
echo "false"
fi
# Architecture detection
ARCH:
sh: |
ARCH=$(uname -m)
case "$ARCH" in
x86_64|x64)
echo "x86_64"
;;
aarch64|arm64)
echo "arm64"
;;
armv7l|armv7)
echo "armv7"
;;
armv6l|armv6)
echo "armv6"
;;
i686|i386)
echo "i386"
;;
*)
echo "$ARCH"
;;
esac
# Number of CPUs available
NUM_CPUS:
sh: |
if command -v nproc >/dev/null 2>&1; then
nproc
elif [[ "$OSTYPE" == "darwin"* ]]; then
sysctl -n hw.ncpu
elif [[ "$OSTYPE" == "msys" ]] || [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then
echo "${NUMBER_OF_PROCESSORS:-4}"
else
echo "4"
fi
MAKE_JOBS: "{{.NUM_CPUS}}"

20
.task/languages/csharp.yml Normal file
View File

@@ -0,0 +1,20 @@
version: "3"
internal: true
tasks:
update:
desc: "Update C# dependencies within major versions (dotnet outdated -u --version-lock Minor)"
silent: false
cmds:
- cmd: |
cd packages/csharp && dotnet outdated -u --version-lock Minor
ignore_error: false
upgrade:
desc: "Upgrade C# dependencies to latest including breaking changes (dotnet outdated -u)"
silent: false
cmds:
- cmd: |
command -v dotnet >/dev/null 2>&1 || { echo "Dotnet not found, skipping C# upgrade"; exit 0; }
cd packages/csharp/Kreuzberg && dotnet outdated -u
ignore_error: false

20
.task/languages/dart.yml Normal file
View File

@@ -0,0 +1,20 @@
version: "3"
internal: true
tasks:
update:
desc: "Update Dart dependencies within major versions (dart pub upgrade)"
silent: false
cmds:
- cmd: |
cd packages/dart && dart pub upgrade
ignore_error: false
upgrade:
desc: "Upgrade Dart dependencies to latest including breaking changes (dart pub upgrade --major-versions)"
silent: false
cmds:
- cmd: |
command -v dart >/dev/null 2>&1 || { echo "Dart not found, skipping Dart upgrade"; exit 0; }
cd packages/dart && dart pub upgrade --major-versions
ignore_error: false

20
.task/languages/elixir.yml Normal file
View File

@@ -0,0 +1,20 @@
version: "3"
internal: true
tasks:
update:
desc: "Update Elixir dependencies within major versions (mix deps.update --all)"
silent: false
cmds:
- cmd: |
cd packages/elixir && mix deps.update --all
ignore_error: false
upgrade:
desc: "Upgrade Elixir dependencies to latest including breaking changes (mix hex.outdated --all + mix deps.update)"
silent: false
cmds:
- cmd: |
command -v mix >/dev/null 2>&1 || { echo "Elixir not found, skipping Elixir upgrade"; exit 0; }
cd packages/elixir && mix hex.outdated --all || true && mix deps.update --all
ignore_error: false

20
.task/languages/go.yml Normal file
View File

@@ -0,0 +1,20 @@
version: "3"
internal: true
tasks:
update:
desc: "Update Go dependencies within major versions (go get -u=patch + go mod tidy)"
silent: false
cmds:
- cmd: |
cd packages/go/v5 && go get -u=patch ./... && go mod tidy
ignore_error: false
upgrade:
desc: "Upgrade Go dependencies to latest including breaking changes (go get -u + go mod tidy)"
silent: false
cmds:
- cmd: |
command -v go >/dev/null 2>&1 || { echo "Go not found, skipping Go upgrade"; exit 0; }
cd packages/go/v5 && go get -u ./... && go mod tidy
ignore_error: false

19
.task/languages/java.yml Normal file
View File

@@ -0,0 +1,19 @@
version: "3"
internal: true
tasks:
update:
desc: "Update Java dependencies within major versions (mvn versions:use-next-versions)"
silent: false
cmds:
- cmd: |
cd packages/java && command -v mvn >/dev/null 2>&1 && mvn versions:use-next-versions -DgenerateBackupPoms=false || echo "Maven not found, skipping Java upgrade"
ignore_error: false
upgrade:
desc: "Upgrade Java dependencies to latest including breaking changes (mvn versions:use-latest-releases)"
silent: false
cmds:
- cmd: |
cd packages/java && command -v mvn >/dev/null 2>&1 && mvn versions:use-latest-releases -DgenerateBackupPoms=false || echo "Maven not found, skipping Java upgrade"
ignore_error: false

20
.task/languages/kotlin_android.yml Normal file
View File

@@ -0,0 +1,20 @@
version: "3"
internal: true
tasks:
update:
desc: "Update Kotlin/Android dependencies within major versions (gradlew dependencyUpdates)"
silent: false
cmds:
- cmd: |
cd packages/kotlin-android && ./gradlew dependencyUpdates
ignore_error: false
upgrade:
desc: "Upgrade Kotlin/Android dependencies to latest including breaking changes (gradlew useLatestVersions)"
silent: false
cmds:
- cmd: |
test -f packages/kotlin-android/gradlew || { echo "Kotlin-Android gradlew not found, skipping upgrade"; exit 0; }
cd packages/kotlin-android && ./gradlew useLatestVersions
ignore_error: false

19
.task/languages/node.yml Normal file
View File

@@ -0,0 +1,19 @@
version: "3"
internal: true
tasks:
update:
desc: "Update Node.js dependencies within major versions (pnpm up)"
silent: false
cmds:
- cmd: |
pnpm up
ignore_error: false
upgrade:
desc: "Upgrade Node.js dependencies to latest including breaking changes (pnpm up --latest)"
silent: false
cmds:
- cmd: |
pnpm up --latest
ignore_error: false

20
.task/languages/php.yml Normal file
View File

@@ -0,0 +1,20 @@
version: "3"
internal: true
tasks:
update:
desc: "Update PHP dependencies within major versions (composer update)"
silent: false
cmds:
- cmd: |
cd packages/php && composer update
ignore_error: false
upgrade:
desc: "Upgrade PHP dependencies to latest including breaking changes (composer update --with-all-dependencies)"
silent: false
cmds:
- cmd: |
command -v composer >/dev/null 2>&1 || { echo "Composer not found, skipping PHP upgrade"; exit 0; }
cd packages/php && composer update --with-all-dependencies
ignore_error: false

22
.task/languages/python.yml Normal file
View File

@@ -0,0 +1,22 @@
version: "3"
internal: true
vars:
PYTHON_PKG: "packages/python"
tasks:
update:
desc: "Update Python dependencies within major versions (uv sync --upgrade-package)"
silent: false
cmds:
- cmd: |
cd {{.PYTHON_PKG}} && uv sync --no-install-project --no-install-workspace --all-extras --upgrade-package "*" --no-prerelease
ignore_error: false
upgrade:
desc: "Upgrade Python dependencies to latest including breaking changes (uv sync --upgrade)"
silent: false
cmds:
- cmd: |
cd {{.PYTHON_PKG}} && uv sync --no-install-project --no-install-workspace --all-extras --upgrade
ignore_error: false

20
.task/languages/r.yml Normal file
View File

@@ -0,0 +1,20 @@
version: "3"
internal: true
tasks:
update:
desc: "Update R dependencies within major versions (devtools::update_packages(check.built = TRUE))"
silent: false
cmds:
- cmd: |
cd packages/r && Rscript -e "devtools::update_packages(check.built = TRUE)"
ignore_error: false
upgrade:
desc: "Upgrade R dependencies to latest including breaking changes (update.packages())"
silent: false
cmds:
- cmd: |
command -v Rscript >/dev/null 2>&1 || { echo "R not found, skipping R upgrade"; exit 0; }
cd packages/r && Rscript -e 'options(repos = c(CRAN = "https://cloud.r-project.org")); update.packages(ask = FALSE)'
ignore_error: false

20
.task/languages/ruby.yml Normal file
View File

@@ -0,0 +1,20 @@
version: "3"
internal: true
tasks:
update:
desc: "Update Ruby dependencies within major versions (bundle update --conservative)"
silent: false
cmds:
- cmd: |
cd packages/ruby && bundle update --conservative
ignore_error: false
upgrade:
desc: "Upgrade Ruby dependencies to latest including breaking changes (bundle update)"
silent: false
cmds:
- cmd: |
command -v bundle >/dev/null 2>&1 || { echo "Bundle not found, skipping Ruby upgrade"; exit 0; }
cd packages/ruby && bundle update
ignore_error: false

247
.task/languages/rust.yml Normal file
View File

@@ -0,0 +1,247 @@
version: "3"
internal: true
includes:
platforms: ../config/platforms.yml
vars:
RUST_LOG: '{{.RUST_LOG | default "info"}}'
BUILD_PROFILE: '{{.BUILD_PROFILE | default "release"}}'
RUST_BACKTRACE: '{{.RUST_BACKTRACE | default "1"}}'
CARGO_TERM_COLOR: "always"
tasks:
install:
desc: "Install Rust toolchain and components"
cmds:
- rustup update stable
- rustup component add rustfmt clippy
# Windows: install lld-link to avoid PATH conflicts with Git Bash's link.exe
- cmd: rustup component add llvm-tools
platforms: [windows]
- cargo install cargo-llvm-cov --locked
- cargo install cargo-edit --locked
- cargo --version
- rustc --version
build:
desc: "Build all Rust crates with {{.BUILD_PROFILE}} profile"
cmds:
- task: build:{{.BUILD_PROFILE | default "release"}}
build:dev:
desc: "Build all Rust crates in debug mode"
cmds:
- cmd: cargo build --workspace --all-features --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm
platforms: [linux, darwin]
# Note: exclude benchmark-harness on Windows as jemalloc doesn't build with MSVC
- cmd: cargo build --workspace --all-features --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm --exclude benchmark-harness
platforms: [windows]
build:release:
desc: "Build all Rust crates in release mode"
cmds:
- cmd: cargo build --release --workspace --all-features --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm
platforms: [linux, darwin]
- cmd: cargo build --release --workspace --all-features --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm --exclude benchmark-harness
platforms: [windows]
build:ci:
desc: "Build for CI with debug info enabled"
env:
RUSTFLAGS: "-C debuginfo=2"
cmds:
- cmd: cargo build --release --workspace --all-features --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm
platforms: [linux, darwin]
- cmd: cargo build --release --workspace --all-features --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm --exclude benchmark-harness
platforms: [windows]
build:profiling:
desc: "Build all Rust crates with profiling features (for flamegraph generation)"
env:
ENABLE_PROFILING: "true"
RUSTFLAGS: "-g"
cmds:
- cargo build --workspace --release --features full,profiling,api,mcp,otel --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm
- cargo build --manifest-path tools/benchmark-harness/Cargo.toml --release --features profiling
cli:build:
desc: "Build CLI binary"
cmds:
- cargo build --release --package kreuzberg-cli {{if .TARGET}}--target {{.TARGET}}{{end}}
cli:build:dev:
desc: "Build CLI binary in debug mode"
cmds:
- cargo build --package kreuzberg-cli
cli:install:
desc: "Install CLI binary"
cmds:
- cargo install --path crates/kreuzberg-cli
ffi:build:
desc: "Build FFI library for language bindings"
cmds:
- cargo build --release --package kreuzberg-ffi {{.CARGO_ARGS | default ""}}
ffi:build:dev:
desc: "Build FFI library in debug mode"
cmds:
- cargo build --package kreuzberg-ffi {{.CARGO_ARGS | default ""}}
ffi:build:ci:
desc: "Build FFI library in CI mode (with debug info)"
env:
RUSTFLAGS: "-C debuginfo=2"
cmds:
- cargo build --release --package kreuzberg-ffi {{.CARGO_ARGS | default ""}}
wasm:build:all:
desc: "Build all WASM targets (web/bundler/nodejs/deno) plus the TypeScript wrapper"
dir: crates/kreuzberg-wasm
cmds:
- pnpm run build:all
test:
desc: "Run all Rust tests"
cmds:
- cmd: bash scripts/ci/rust/run-unit-tests.sh
platforms: [linux, darwin]
test:ci:
desc: "Run Rust tests in CI mode with tessdata setup"
env:
RUST_BACKTRACE: "{{.RUST_BACKTRACE}}"
CARGO_TERM_COLOR: "{{.CARGO_TERM_COLOR}}"
cmds:
- cmd: bash scripts/ci/rust/run-unit-tests.sh
platforms: [linux, darwin]
test:quick:
desc: "Run fast Rust tests (unit tests only)"
cmds:
- cargo test --lib --workspace --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm
lint:
desc: "Lint Rust code WITH auto-fix (cargo fmt + cargo clippy --fix)"
cmds:
- cargo fmt --all
- cargo clippy --workspace --fix --allow-dirty --allow-staged --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm
lint:check:
desc: "Lint Rust code WITHOUT auto-fix (check-only)"
cmds:
- cargo fmt --all -- --check
- cargo clippy --workspace --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm -- -D warnings
format:
desc: "Format Rust code (with modifications)"
cmds:
- cargo fmt --all
format:check:
desc: "Check Rust formatting without modifications"
cmds:
- cargo fmt --all -- --check
licenses:
desc: "Check Rust dependency licenses with cargo-deny"
cmds:
- cargo deny check licenses
doc:
desc: "Generate Rust documentation and open in browser"
cmds:
- cargo doc --no-deps --open
doc:build:
desc: "Build Rust documentation (without opening)"
cmds:
- cargo doc --no-deps
clean:
desc: "Clean Rust build artifacts"
cmds:
- cargo clean
cache:cleanup:
desc: "Clean up large build artifacts to reduce cache size (for CI)"
cmds:
- echo "Cleaning up large build artifacts to reduce cache size..."
- cmd: find target -type f -name "*.rlib" -size +10M -exec rm -f {} \; 2>/dev/null || true
ignore_error: true
- cmd: find target -type f -name "*.so" -size +10M -exec rm -f {} \; 2>/dev/null || true
ignore_error: true
- cmd: find target -type f -name "*.dylib" -size +10M -exec rm -f {} \; 2>/dev/null || true
ignore_error: true
- cmd: find target -type f -name "*.dll" -size +10M -exec rm -f {} \; 2>/dev/null || true
ignore_error: true
- cmd: rm -rf target/*/incremental 2>/dev/null || true
ignore_error: true
- echo "Cleanup completed successfully"
- cmd: du -sh target 2>/dev/null || echo "No target directory found"
ignore_error: true
update:
desc: "Update Rust dependencies (compatible only)"
cmds:
- cargo update
- cmd: cargo update --manifest-path packages/ruby/ext/kreuzberg_rb/native/Cargo.toml
ignore_error: true
upgrade:
desc: "Upgrade Rust dependencies aggressively (cargo upgrade --incompatible)"
cmds:
- cargo upgrade --incompatible
- cargo update
- cmd: cargo upgrade --incompatible --manifest-path packages/ruby/ext/kreuzberg_rb/native/Cargo.toml
ignore_error: true
- cmd: cargo update --manifest-path packages/ruby/ext/kreuzberg_rb/native/Cargo.toml
ignore_error: true
e2e:format:
desc: "Format generated Rust E2E sources"
cmds:
- cargo fmt --manifest-path e2e/rust/Cargo.toml
e2e:lint:
desc: "Lint Rust E2E crate with clippy"
cmds:
- cargo clippy --manifest-path e2e/rust/Cargo.toml -- -D warnings
e2e:test:
desc: "Run Rust E2E tests"
env:
RUST_TEST_THREADS: "1"
cmds:
- cargo test --manifest-path e2e/rust/Cargo.toml --release
check:android:
desc: "cargo check kreuzberg-dart + kreuzberg-ffi for Android ABIs (requires ANDROID_NDK_HOME and cargo-ndk)"
preconditions:
- sh: command -v cargo-ndk
msg: "cargo-ndk not installed. Run: cargo install cargo-ndk --locked"
- sh: 'test -n "${ANDROID_NDK_HOME:-}"'
msg: "ANDROID_NDK_HOME is not set. Install Android NDK and export ANDROID_NDK_HOME=<path-to-ndk>"
cmds:
- cargo ndk --target arm64-v8a --platform 21 -- check -p kreuzberg-dart
- cargo ndk --target x86_64 --platform 21 -- check -p kreuzberg-dart
- cargo ndk --target arm64-v8a --platform 21 -- check -p kreuzberg-ffi
- cargo ndk --target x86_64 --platform 21 -- check -p kreuzberg-ffi
check:ios:
desc: "cargo check kreuzberg-dart + kreuzberg-swift for iOS targets (macOS only)"
platforms: [darwin]
cmds:
- rustup target add aarch64-apple-ios aarch64-apple-ios-sim
- cargo check -p kreuzberg-dart --target aarch64-apple-ios
- cargo check -p kreuzberg-dart --target aarch64-apple-ios-sim
- cargo check -p kreuzberg-swift --target aarch64-apple-ios
- cargo check -p kreuzberg-swift --target aarch64-apple-ios-sim
check:mobile:
desc: "cargo check Android + iOS mobile targets"
cmds:
- task: check:android
- task: check:ios

20
.task/languages/swift.yml Normal file
View File

@@ -0,0 +1,20 @@
version: "3"
internal: true
tasks:
update:
desc: "Update Swift dependencies (swift package update)"
silent: false
cmds:
- cmd: |
cd packages/swift && swift package update
ignore_error: false
upgrade:
desc: "Upgrade Swift dependencies (swift package update)"
silent: false
cmds:
- cmd: |
command -v swift >/dev/null 2>&1 || { echo "Swift not found, skipping Swift upgrade"; exit 0; }
cd packages/swift && swift package update
ignore_error: false

Some files were not shown because too many files have changed in this diff Show More