Nomad changes

2026-06-01 23:40:55 +02:00
parent 72b1a0a6ed
commit b4c07d3693
5723 changed files with 1130655 additions and 0 deletions
--- a/crates/kreuzberg-ffi/Cargo.toml
+++ b/crates/kreuzberg-ffi/Cargo.toml
@@ -0,0 +1,46 @@
+[package]
+name = "kreuzberg-ffi"
+version = "5.0.0-rc.3"
+edition = "2021"
+license = "Elastic-2.0"
+description = "High-performance document intelligence library"
+readme = false
+keywords = ["document", "extraction", "ocr", "pdf", "text"]
+categories = ["text-processing"]
+repository = "https://github.com/kreuzberg-dev/kreuzberg"
+
+# `serde_json`, `ahash`, and `tokio` are emitted unconditionally above so the
+# manifest is stable across regens (and so the C FFI codegen can pull them in
+# when an async / Result-typed function appears in the API surface), but for
+# umbrella crates with no async fns and no JSON-marshalled return types they
+# are genuinely unused. The conditional `async-trait` / `futures-util` deps
+# are similarly flagged when the umbrella has trait-bridge / streaming adapters
+# configured but no actual async-trait / async-stream callsite in the generated
+# FFI shim.
+[package.metadata.cargo-machete]
+ignored = ["ahash", "serde_json", "tokio", "async-trait"]
+
+[lib]
+crate-type = ["cdylib", "staticlib", "rlib"]
+
+[features]
+default = []
+
+[dependencies]
+ahash = "0.8"
+async-trait = "0.1"
+serde_json = "1"
+tokio = { version = "1", features = ["full"] }
+
+[target.'cfg(not(all(target_os = "android", target_arch = "x86_64")))'.dependencies]
+kreuzberg = { path = "../kreuzberg", version = "5.0.0-rc.3", features = ["full", "pdf", "ocr", "paddle-ocr", "paddle-ocr-types", "layout-detection", "layout-types", "embeddings", "embedding-presets", "chunking", "keywords-yake", "keywords-rake", "language-detection", "html", "tree-sitter", "office", "email", "archives", "stopwords", "auto-rotate", "auto-rotate-types", "tokio-runtime", "api", "mcp", "liter-llm", "quality"] }
+
+[target.'cfg(all(target_os = "android", target_arch = "x86_64"))'.dependencies]
+kreuzberg = { path = "../kreuzberg", version = "5.0.0-rc.3", features = ["android-target"] }
+
+
+[build-dependencies]
+cbindgen = "0.29"
+
+[dev-dependencies]
+tempfile = "3"
--- a/crates/kreuzberg-ffi/README.md
+++ b/crates/kreuzberg-ffi/README.md
@@ -0,0 +1,295 @@
+# FFI (C/C++)
+
+<div align="center" style="display: flex; flex-wrap: wrap; gap: 8px; justify-content: center; margin: 20px 0;">
+  <a href="https://github.com/kreuzberg-dev/alef">
+    <img src="https://img.shields.io/badge/Bindings-alef%20%D7%90-007ec6" alt="Bindings">
+  </a>
+  <!-- Language Bindings -->
+  <a href="https://crates.io/crates/kreuzberg">
+    <img src="https://img.shields.io/crates/v/kreuzberg?label=Rust&color=007ec6" alt="Rust">
+  </a>
+  <a href="https://pypi.org/project/kreuzberg/">
+    <img src="https://img.shields.io/pypi/v/kreuzberg?label=Python&color=007ec6" alt="Python">
+  </a>
+  <a href="https://www.npmjs.com/package/@kreuzberg/node">
+    <img src="https://img.shields.io/npm/v/@kreuzberg/node?label=Node.js&color=007ec6" alt="Node.js">
+  </a>
+  <a href="https://www.npmjs.com/package/@kreuzberg/wasm">
+    <img src="https://img.shields.io/npm/v/@kreuzberg/wasm?label=WASM&color=007ec6" alt="WASM">
+  </a>
+  <a href="https://central.sonatype.com/artifact/dev.kreuzberg/kreuzberg">
+    <img src="https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg?label=Java&color=007ec6" alt="Java">
+  </a>
+  <a href="https://github.com/kreuzberg-dev/kreuzberg/tree/main/packages/go/v5">
+    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v5*" alt="Go">
+  </a>
+  <a href="https://www.nuget.org/packages/Kreuzberg/">
+    <img src="https://img.shields.io/nuget/v/Kreuzberg?label=C%23&color=007ec6" alt="C#">
+  </a>
+  <a href="https://packagist.org/packages/kreuzberg/kreuzberg">
+    <img src="https://img.shields.io/packagist/v/kreuzberg/kreuzberg?label=PHP&color=007ec6" alt="PHP">
+  </a>
+  <a href="https://rubygems.org/gems/kreuzberg">
+    <img src="https://img.shields.io/gem/v/kreuzberg?label=Ruby&color=007ec6" alt="Ruby">
+  </a>
+  <a href="https://hex.pm/packages/kreuzberg">
+    <img src="https://img.shields.io/hexpm/v/kreuzberg?label=Elixir&color=007ec6" alt="Elixir">
+  </a>
+  <a href="https://kreuzberg-dev.r-universe.dev/kreuzberg">
+    <img src="https://img.shields.io/badge/R-kreuzberg-007ec6" alt="R">
+  </a>
+  <a href="https://pub.dev/packages/kreuzberg">
+    <img src="https://img.shields.io/pub/v/kreuzberg?label=Dart&color=007ec6" alt="Dart">
+  </a>
+  <a href="https://central.sonatype.com/artifact/dev.kreuzberg/kreuzberg-android">
+    <img src="https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg-android?label=Kotlin&color=007ec6" alt="Kotlin">
+  </a>
+  <a href="https://github.com/kreuzberg-dev/kreuzberg/tree/main/packages/swift">
+    <img src="https://img.shields.io/badge/Swift-SPM-007ec6" alt="Swift">
+  </a>
+  <a href="https://github.com/kreuzberg-dev/kreuzberg/tree/main/packages/zig">
+    <img src="https://img.shields.io/badge/Zig-package-007ec6" alt="Zig">
+  </a>
+  <a href="https://github.com/kreuzberg-dev/kreuzberg/releases">
+    <img src="https://img.shields.io/badge/C-FFI-007ec6" alt="C FFI">
+  </a>
+  <a href="https://github.com/kreuzberg-dev/kreuzberg/pkgs/container/kreuzberg">
+    <img src="https://img.shields.io/badge/Docker-ghcr.io-007ec6?logo=docker&logoColor=white" alt="Docker">
+  </a>
+  <a href="https://github.com/kreuzberg-dev/kreuzberg/pkgs/container/charts%2Fkreuzberg">
+    <img src="https://img.shields.io/badge/Helm-ghcr.io-007ec6?logo=helm&logoColor=white" alt="Helm">
+  </a>
+
+  <!-- Project Info -->
+  <a href="https://github.com/kreuzberg-dev/kreuzberg/blob/main/LICENSE">
+    <img src="https://img.shields.io/badge/License-Elastic--2.0-007ec6" alt="License">
+  </a>
+  <a href="https://docs.kreuzberg.dev">
+    <img src="https://img.shields.io/badge/Docs-kreuzberg-007ec6" alt="Documentation">
+  </a>
+  <a href="https://huggingface.co/Kreuzberg">
+    <img src="https://img.shields.io/badge/Hugging%20Face-Kreuzberg-007ec6" alt="Hugging Face">
+  </a>
+</div>
+
+<div align="center" style="margin: 24px 0 0;">
+  <a href="https://kreuzberg.dev">
+    <img alt="Kreuzberg" src="https://github.com/user-attachments/assets/419fc06c-8313-4324-b159-4b4d3cfce5c0" />
+  </a>
+</div>
+
+<div align="center" style="display: flex; flex-wrap: wrap; gap: 12px; justify-content: center; margin: 28px 0 24px;">
+  <a href="https://discord.gg/xt9WY3GnKR">
+    <img height="22" src="https://img.shields.io/badge/Discord-Chat-007ec6?logo=discord&logoColor=white" alt="Join Discord">
+  </a>
+  <a href="https://docs.kreuzberg.dev/demo.html">
+    <img height="22" src="https://img.shields.io/badge/Live%20Demo-Open-007ec6?logo=webassembly&logoColor=white" alt="Live Demo">
+  </a>
+</div>
+
+Extract text, tables, images, and metadata from 90+ file formats and 300+ programming languages including PDF, Office documents, and images. C/C++ FFI bindings providing a stable ABI for native integration, shared library distribution, and cross-language interop.
+
+## What This Package Provides
+
+- **Document intelligence core** — extract text, tables, images, metadata, entities, keywords, and code intelligence from one API.
+- **Format coverage** — PDF, Office, images, HTML/XML, email, archives, notebooks, citations, scientific formats, and plain text.
+- **OCR choices** — Tesseract, PaddleOCR, EasyOCR where supported, VLM OCR through liter-llm, and plugin hooks for custom backends.
+- **Same engine as every binding** — Rust, Python, Node.js, Go, Java, PHP, Ruby, .NET, Elixir, R, WASM, Kotlin Android, Swift, Dart, Zig, and C FFI share the same Rust implementation.
+- **C ABI** — stable shared library surface for custom hosts and secondary bindings.
+
+## Installation
+
+### Package Installation
+
+Build the shared library from the workspace:
+
+```bash
+cargo build --release -p kreuzberg-ffi
+```
+
+The built artifacts are emitted under `target/release/` (`libkreuzberg_ffi.{so,dylib,a}`) along with the C header at `crates/kreuzberg-ffi/include/kreuzberg.h`.
+
+### System Requirements
+- A C/C++ toolchain (clang, gcc, or MSVC) and a Rust toolchain (`rustup`) for building from source
+- A `pkg-config` or CMake-aware build system that can locate `libkreuzberg_ffi` and `kreuzberg.h`
+- Optional: [ONNX Runtime](https://github.com/microsoft/onnxruntime/releases) version 1.22.x for embeddings support
+- Optional: [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) for OCR functionality
+
+## Quick Start
+
+### Basic Extraction
+
+Extract text, metadata, and structure from any supported document format:
+
+<!-- snippet not found:  -->
+
+### Common Use Cases
+
+#### Extract with Custom Configuration
+
+Most use cases benefit from configuration to control extraction behavior:
+
+#### Table Extraction
+
+See [Configuration Guide](https://docs.kreuzberg.dev/guides/configuration/) for table extraction options.
+
+#### Processing Multiple Files
+
+### Next Steps
+
+- **[Installation Guide](https://docs.kreuzberg.dev/getting-started/installation/)** - Platform-specific setup
+- **[API Documentation](https://docs.kreuzberg.dev/reference/api-python/)** - Complete API reference
+- **[Examples & Guides](https://docs.kreuzberg.dev/)** - Full code examples and usage guides
+- **[Configuration Guide](https://docs.kreuzberg.dev/guides/configuration/)** - Advanced configuration options
+
+## Features
+
+### Supported File Formats (90+)
+
+90+ file formats across 8 major categories with intelligent format detection and comprehensive metadata extraction.
+
+#### Office Documents
+
+| Category | Formats | Capabilities |
+|----------|---------|--------------|
+| **Word Processing** | `.docx`, `.docm`, `.dotx`, `.dotm`, `.dot`, `.odt` | Full text, tables, images, metadata, styles |
+| **Spreadsheets** | `.xlsx`, `.xlsm`, `.xlsb`, `.xls`, `.xla`, `.xlam`, `.xltm`, `.xltx`, `.xlt`, `.ods` | Sheet data, formulas, cell metadata, charts |
+| **Presentations** | `.pptx`, `.pptm`, `.ppsx`, `.potx`, `.potm`, `.pot`, `.ppt` | Slides, speaker notes, images, metadata |
+| **PDF** | `.pdf` | Text, tables, images, metadata, OCR support |
+| **eBooks** | `.epub`, `.fb2` | Chapters, metadata, embedded resources |
+| **Database** | `.dbf` | Table data extraction, field type support |
+| **Hangul** | `.hwp`, `.hwpx` | Korean document format, text extraction |
+
+#### Images (OCR-Enabled)
+
+| Category | Formats | Features |
+|----------|---------|----------|
+| **Raster** | `.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, `.bmp`, `.tiff`, `.tif` | OCR, table detection, EXIF metadata, dimensions, color space |
+| **Advanced** | `.jp2`, `.jpx`, `.jpm`, `.mj2`, `.jbig2`, `.jb2`, `.pnm`, `.pbm`, `.pgm`, `.ppm` | OCR via hayro-jpeg2000 (pure Rust decoder), JBIG2 support, table detection, format-specific metadata |
+| **Vector** | `.svg` | DOM parsing, embedded text, graphics metadata |
+
+#### Web & Data
+
+| Category | Formats | Features |
+|----------|---------|----------|
+| **Markup** | `.html`, `.htm`, `.xhtml`, `.xml`, `.svg` | DOM parsing, metadata (Open Graph, Twitter Card), link extraction |
+| **Structured Data** | `.json`, `.yaml`, `.yml`, `.toml`, `.csv`, `.tsv` | Schema detection, nested structures, validation |
+| **Text & Markdown** | `.txt`, `.md`, `.markdown`, `.djot`, `.rst`, `.org`, `.rtf` | CommonMark, GFM, Djot, reStructuredText, Org Mode |
+
+#### Email & Archives
+
+| Category | Formats | Features |
+|----------|---------|----------|
+| **Email** | `.eml`, `.msg` | Headers, body (HTML/plain), attachments, threading |
+| **Archives** | `.zip`, `.tar`, `.tgz`, `.gz`, `.7z` | File listing, nested archives, metadata |
+
+#### Academic & Scientific
+
+| Category | Formats | Features |
+|----------|---------|----------|
+| **Citations** | `.bib`, `.biblatex`, `.ris`, `.nbib`, `.enw`, `.csl` | Structured parsing: RIS (structured), PubMed/MEDLINE, EndNote XML (structured), BibTeX, CSL JSON |
+| **Scientific** | `.tex`, `.latex`, `.typst`, `.jats`, `.ipynb`, `.docbook` | LaTeX, Jupyter notebooks, PubMed JATS |
+| **Documentation** | `.opml`, `.pod`, `.mdoc`, `.troff` | Technical documentation formats |
+
+#### Code Intelligence (300+ Languages)
+
+| Feature | Description |
+|---------|-------------|
+| **Structure Extraction** | Functions, classes, methods, structs, interfaces, enums |
+| **Import/Export Analysis** | Module dependencies, re-exports, wildcard imports |
+| **Symbol Extraction** | Variables, constants, type aliases, properties |
+| **Docstring Parsing** | Google, NumPy, Sphinx, JSDoc, RustDoc, and 10+ formats |
+| **Diagnostics** | Parse errors with line/column positions |
+| **Syntax-Aware Chunking** | Split code by semantic boundaries, not arbitrary byte offsets |
+
+Powered by [tree-sitter-language-pack](https://github.com/kreuzberg-dev/tree-sitter-language-pack) — [documentation](https://docs.tree-sitter-language-pack.kreuzberg.dev).
+
+**[Complete Format Reference](https://docs.kreuzberg.dev/reference/formats/)**
+
+### Key Capabilities
+
+- **Text Extraction** - Extract all text content with position and formatting information
+- **Metadata Extraction** - Retrieve document properties, creation date, author, etc.
+- **Table Extraction** - Parse tables with structure and cell content preservation
+- **Image Extraction** - Extract embedded images and render page previews
+- **OCR Support** - Integrate multiple OCR backends for scanned documents
+- **Async/Await** - Non-blocking document processing with concurrent operations
+- **Plugin System** - Extensible post-processing for custom text transformation
+- **Embeddings** - Generate vector embeddings using ONNX Runtime models
+- **Batch Processing** - Efficiently process multiple documents in parallel
+- **Memory Efficient** - Stream large files without loading entirely into memory
+- **Language Detection** - Detect and support multiple languages in documents
+- **Code Intelligence** - Extract structure, imports, exports, symbols, and docstrings from [300+ programming languages](https://docs.tree-sitter-language-pack.kreuzberg.dev) via tree-sitter
+- **Configuration** - Fine-grained control over extraction behavior
+
+### Performance Characteristics
+
+| Format | Speed | Memory | Notes |
+|--------|-------|--------|-------|
+| **PDF (text)** | 10-100 MB/s | ~50MB per doc | Fastest extraction |
+| **Office docs** | 20-200 MB/s | ~100MB per doc | DOCX, XLSX, PPTX |
+| **Images (OCR)** | 1-5 MB/s | Variable | Depends on OCR backend |
+| **Archives** | 5-50 MB/s | ~200MB per doc | ZIP, TAR, etc. |
+| **Web formats** | 50-200 MB/s | Streaming | HTML, XML, JSON |
+
+## OCR Support
+
+Kreuzberg supports multiple OCR backends for extracting text from scanned documents and images:
+
+### OCR Configuration Example
+
+<!-- snippet not found:  -->
+
+## Async Support
+
+This binding provides full async/await support for non-blocking document processing:
+
+<!-- snippet not found:  -->
+
+## Plugin System
+
+Kreuzberg supports extensible post-processing plugins for custom text transformation and filtering.
+
+For detailed plugin documentation, visit [Plugin System Guide](https://docs.kreuzberg.dev/guides/plugins/).
+
+## Embeddings Support
+
+Generate vector embeddings for extracted text using the built-in ONNX Runtime support. Requires ONNX Runtime installation.
+
+**[Embeddings Guide](https://docs.kreuzberg.dev/features/#embeddings)**
+
+## Configuration
+
+For advanced configuration options including language detection, table extraction, OCR settings, and more:
+
+**[Configuration Guide](https://docs.kreuzberg.dev/guides/configuration/)**
+
+## Documentation
+
+- **[Official Documentation](https://docs.kreuzberg.dev/)**
+- **[API Reference](https://docs.kreuzberg.dev/reference/api-python/)**
+- **[Examples & Guides](https://docs.kreuzberg.dev/)**
+
+## Contributing
+
+Contributions are welcome! See [Contributing Guide](https://github.com/kreuzberg-dev/kreuzberg/blob/main/CONTRIBUTING.md).
+
+## Part of Kreuzberg.dev
+
+- [Kreuzberg Cloud](https://github.com/kreuzberg-dev/kreuzberg-cloud) — managed extraction API with SDKs, dashboards, and observability.
+- [kreuzcrawl](https://github.com/kreuzberg-dev/kreuzcrawl) — web crawling and scraping with HTML→Markdown and headless-Chrome fallback.
+- [html-to-markdown](https://github.com/kreuzberg-dev/html-to-markdown) — fast, lossless HTML→Markdown engine.
+- [liter-llm](https://github.com/kreuzberg-dev/liter-llm) — universal LLM API client with native bindings for 14 languages and 143 providers.
+- [tree-sitter-language-pack](https://github.com/kreuzberg-dev/tree-sitter-language-pack) — tree-sitter grammars and code-intelligence primitives.
+- [alef](https://github.com/kreuzberg-dev/alef) — the polyglot binding generator that produces this README and all per-language bindings.
+- [Discord](https://discord.gg/xt9WY3GnKR) — community, roadmap, announcements.
+
+## License
+
+Elastic-2.0 License — see [LICENSE](../../LICENSE) for details.
+
+## Support
+
+- **Discord Community**: [Join our Discord](https://discord.gg/xt9WY3GnKR)
+- **GitHub Issues**: [Report bugs](https://github.com/kreuzberg-dev/kreuzberg/issues)
+- **Discussions**: [Ask questions](https://github.com/kreuzberg-dev/kreuzberg/discussions)
--- a/crates/kreuzberg-ffi/build.rs
+++ b/crates/kreuzberg-ffi/build.rs
@@ -0,0 +1,23 @@
+// This file is auto-generated by alef. DO NOT EDIT.
+// alef:hash:4e15143f4af1ae8bafbdb1506ef057da924484c66a19483966333558ad437e75
+fn main() {
+    let crate_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap();
+    cbindgen::generate(crate_dir)
+        .expect("Unable to generate C bindings")
+        .write_to_file("include/kreuzberg.h");
+
+    // Set @rpath-relative install_name on macOS so the cdylib can be relocated
+    // (bundled into language packages like packages/go/.lib/<rid>/, packages/
+    // java/src/main/resources/natives/<rid>/, etc.) and located via the consumer
+    // binary's rpath at runtime. Without this, the install_name embeds the CI
+    // runner build path (`/Users/runner/work/.../target/.../deps/lib<name>.dylib`)
+    // and dyld fails to load the bundled copy from its actual location.
+    if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") {
+        println!("cargo:rustc-link-arg-cdylib=-Wl,-install_name,@rpath/libkreuzberg_ffi.dylib");
+    }
+
+    let go_include_dir = std::path::Path::new("../../../packages/go/v5/include");
+    std::fs::create_dir_all(go_include_dir).expect("Unable to create Go include directory");
+    std::fs::copy("include/kreuzberg.h", go_include_dir.join("kreuzberg.h"))
+        .expect("Unable to copy header to Go include directory");
+}
--- a/crates/kreuzberg-ffi/cbindgen.toml
+++ b/crates/kreuzberg-ffi/cbindgen.toml
--- a/crates/kreuzberg-ffi/cmake/kreuzberg-ffi-config.cmake
+++ b/crates/kreuzberg-ffi/cmake/kreuzberg-ffi-config.cmake
@@ -0,0 +1,87 @@
+# kreuzberg-ffi CMake config-mode find module
+#
+# Defines the imported target:
+#   kreuzberg-ffi::kreuzberg-ffi
+#
+# Usage:
+#   find_package(kreuzberg-ffi REQUIRED)
+#   target_link_libraries(myapp PRIVATE kreuzberg-ffi::kreuzberg-ffi)
+
+if(TARGET kreuzberg-ffi::kreuzberg-ffi)
+  return()
+endif()
+
+get_filename_component(_FFI_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
+get_filename_component(_FFI_PREFIX "${_FFI_CMAKE_DIR}/.." ABSOLUTE)
+
+find_library(_FFI_LIBRARY
+  NAMES kreuzberg_ffi libkreuzberg_ffi
+  PATHS "${_FFI_PREFIX}/lib"
+  NO_DEFAULT_PATH
+)
+if(NOT _FFI_LIBRARY)
+  find_library(_FFI_LIBRARY NAMES kreuzberg_ffi libkreuzberg_ffi)
+endif()
+
+find_path(_FFI_INCLUDE_DIR
+  NAMES kreuzberg.h
+  PATHS "${_FFI_PREFIX}/include"
+  NO_DEFAULT_PATH
+)
+if(NOT _FFI_INCLUDE_DIR)
+  find_path(_FFI_INCLUDE_DIR NAMES kreuzberg.h)
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(kreuzberg-ffi
+  REQUIRED_VARS _FFI_LIBRARY _FFI_INCLUDE_DIR
+)
+
+if(kreuzberg_ffi_FOUND)
+  set(_FFI_LIB_TYPE UNKNOWN)
+  if(_FFI_LIBRARY MATCHES "\\.(dylib|so)$" OR _FFI_LIBRARY MATCHES "\\.so\\.")
+    set(_FFI_LIB_TYPE SHARED)
+  elseif(_FFI_LIBRARY MATCHES "\\.dll$")
+    set(_FFI_LIB_TYPE SHARED)
+  elseif(_FFI_LIBRARY MATCHES "\\.(a|lib)$")
+    set(_FFI_LIB_TYPE STATIC)
+  endif()
+
+  add_library(kreuzberg-ffi::kreuzberg-ffi ${_FFI_LIB_TYPE} IMPORTED)
+  set_target_properties(kreuzberg-ffi::kreuzberg-ffi PROPERTIES
+    IMPORTED_LOCATION "${_FFI_LIBRARY}"
+    INTERFACE_INCLUDE_DIRECTORIES "${_FFI_INCLUDE_DIR}"
+  )
+
+  if(WIN32 AND _FFI_LIB_TYPE STREQUAL "SHARED")
+    find_file(_FFI_DLL
+      NAMES kreuzberg_ffi.dll libkreuzberg_ffi.dll
+      PATHS "${_FFI_PREFIX}/bin" "${_FFI_PREFIX}/lib"
+      NO_DEFAULT_PATH
+    )
+    if(_FFI_DLL)
+      set_target_properties(kreuzberg-ffi::kreuzberg-ffi PROPERTIES
+        IMPORTED_LOCATION "${_FFI_DLL}"
+        IMPORTED_IMPLIB "${_FFI_LIBRARY}"
+      )
+    endif()
+    unset(_FFI_DLL CACHE)
+  endif()
+
+  if(APPLE)
+    set_property(TARGET kreuzberg-ffi::kreuzberg-ffi APPEND PROPERTY
+      INTERFACE_LINK_LIBRARIES "-framework CoreFoundation" "-framework Security" pthread)
+  elseif(UNIX)
+    set_property(TARGET kreuzberg-ffi::kreuzberg-ffi APPEND PROPERTY
+      INTERFACE_LINK_LIBRARIES pthread dl m)
+  elseif(WIN32)
+    set_property(TARGET kreuzberg-ffi::kreuzberg-ffi APPEND PROPERTY
+      INTERFACE_LINK_LIBRARIES ws2_32 userenv bcrypt)
+  endif()
+
+  unset(_FFI_LIB_TYPE)
+endif()
+
+mark_as_advanced(_FFI_LIBRARY _FFI_INCLUDE_DIR)
+unset(_FFI_CMAKE_DIR)
+unset(_FFI_PREFIX)
--- a/crates/kreuzberg-ffi/include/kreuzberg.h
+++ b/crates/kreuzberg-ffi/include/kreuzberg.h
--- a/crates/kreuzberg-ffi/src/lib.rs
+++ b/crates/kreuzberg-ffi/src/lib.rs
--- a/crates/kreuzberg-ffi/tests/email_attachment_data_len.rs
+++ b/crates/kreuzberg-ffi/tests/email_attachment_data_len.rs
@@ -0,0 +1,177 @@
+/// Regression test for GitHub #1059.
+///
+/// `kreuzberg_email_attachment_data` was the only byte-buffer accessor on a public
+/// FFI-exposed DTO that did not follow the established `*_data(ptr, out_len: *mut usize)`
+/// protocol used by `kreuzberg_extracted_image_data`, `kreuzberg_embedded_file_data`,
+/// and `kreuzberg_batch_bytes_item_content`.
+///
+/// Because `EmailAttachment.data` is `Option<Bytes>` (the only optional byte buffer among
+/// public types), alef's heuristic for emitting the two-parameter form did not trigger.
+/// Callers had no way to know the valid length of the returned pointer, making any read
+/// past the first byte undefined behaviour (especially for payloads containing 0x00).
+///
+/// The alef fix shipped with the 2-parameter form (`ptr`, `out_len`). These tests
+/// lock in the correct 2-param ABI and verify the full-length contract for payloads
+/// that contain embedded NUL bytes.
+///
+/// Per project rules: every unsafe block has a SAFETY comment.
+use std::ffi::{c_char, CString};
+use std::fs;
+use std::path::Path;
+
+use kreuzberg_ffi::{kreuzberg_email_attachment_free, kreuzberg_email_attachment_from_json, kreuzberg_last_error_code};
+
+/// Construct a minimal EmailAttachment JSON with a data payload that contains
+/// an embedded NUL and a trailing high byte (0xEF). This defeats any strlen-based
+/// or "read first byte only" implementations.
+fn attachment_json_with_nuls() -> CString {
+    // 8 bytes: JPEG-ish magic + NUL in the middle + high byte at the end.
+    // Length is authoritative and known.
+    let data: Vec<u8> = vec![0xFF, 0xD8, 0xFF, 0x00, 0xDE, 0xAD, 0xBE, 0xEF];
+    let json = format!(
+        r#"{{
+            "name": "test.bin",
+            "filename": "test.bin",
+            "mime_type": "application/octet-stream",
+            "size": {},
+            "is_image": false,
+            "data": {}
+        }}"#,
+        data.len(),
+        serde_json::to_string(&data).unwrap()
+    );
+    CString::new(json).expect("valid UTF-8 JSON for test attachment")
+}
+
+/// The committed C header must declare the 2-parameter form for
+/// `kreuzberg_email_attachment_data` (with `out_len`). This locks in the fix
+/// for GitHub #1059 so a future regeneration cannot silently revert to the
+/// 1-parameter form.
+#[test]
+fn email_attachment_data_accessor_must_provide_out_len_in_header() {
+    let header_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("include/kreuzberg.h");
+    let header = fs::read_to_string(&header_path).expect("committed kreuzberg.h must be readable by the test");
+
+    // Simple and robust: the declaration for this specific function must mention out_len.
+    let has_out_len = header.contains("kreuzberg_email_attachment_data") && header.contains("out_len");
+
+    assert!(
+        has_out_len,
+        "GitHub #1059 regression: the declaration of kreuzberg_email_attachment_data \
+         in crates/kreuzberg-ffi/include/kreuzberg.h does not contain the required \
+         `out_len` parameter.\n\n\
+         Expected something like:\n    uint8_t *kreuzberg_email_attachment_data(..., uintptr_t *out_len);\n\n\
+         Found the old 1-parameter form. Fix requires `task alef:generate` with an \
+         updated alef that handles Option<Bytes> fields for the FFI byte accessor heuristic.\n\n\
+         This is the lock-in test for #1059."
+    );
+}
+
+/// When an attachment has no data payload the accessor must return a null pointer
+/// and write 0 to out_len.
+#[test]
+fn email_attachment_data_none_returns_null_pointer() {
+    let json = CString::new(
+        r#"{"name":"empty","filename":"empty","mime_type":null,"size":null,"is_image":false,"data":null}"#,
+    )
+    .unwrap();
+
+    // SAFETY: json is valid null-terminated UTF-8.
+    let handle = unsafe { kreuzberg_email_attachment_from_json(json.as_ptr() as *const c_char) };
+    assert!(
+        !handle.is_null(),
+        "from_json should succeed (last_error_code={})",
+        // SAFETY: no precondition; reads a thread-local.
+        unsafe { kreuzberg_last_error_code() }
+    );
+
+    let mut out_len: usize = usize::MAX;
+    // SAFETY: handle is a valid non-null pointer returned by from_json;
+    // out_len is a valid stack-allocated usize.
+    let data_ptr = unsafe { kreuzberg_ffi::kreuzberg_email_attachment_data(handle, &mut out_len) };
+
+    assert!(
+        data_ptr.is_null(),
+        "data must be null when the attachment has no payload"
+    );
+    assert_eq!(out_len, 0, "out_len must be 0 when data is None");
+
+    // SAFETY: handle came from from_json; we are the sole owner.
+    unsafe { kreuzberg_email_attachment_free(handle) };
+}
+
+/// When an attachment carries a binary payload the accessor must return a non-null
+/// pointer and write the exact byte count — including bytes past any embedded NUL —
+/// to out_len. This is the core contract broken by the 1-parameter bug (#1059).
+#[test]
+fn email_attachment_data_with_out_len_returns_full_buffer_including_embedded_nuls() {
+    let json = attachment_json_with_nuls();
+    // SAFETY: json is a valid null-terminated CString we just created.
+    let handle = unsafe { kreuzberg_email_attachment_from_json(json.as_ptr() as *const c_char) };
+    assert!(
+        !handle.is_null(),
+        "from_json should succeed for our well-formed test attachment (last_error_code={})",
+        // SAFETY: no precondition; reads a thread-local.
+        unsafe { kreuzberg_last_error_code() }
+    );
+
+    let mut out_len: usize = 0;
+
+    // SAFETY: handle is non-null and freshly allocated by from_json;
+    // out_len is a valid stack-allocated usize. The returned pointer must not
+    // be freed by us — it borrows the internal Bytes of the handle.
+    let data_ptr = unsafe { kreuzberg_ffi::kreuzberg_email_attachment_data(handle, &mut out_len) };
+
+    assert!(
+        !data_ptr.is_null(),
+        "data pointer must be non-null for an attachment we created with a Some(data) payload"
+    );
+    assert_eq!(
+        out_len, 8,
+        "out_len must report the exact length of the Bytes payload (not 0, not guessed, not truncated at NUL)"
+    );
+
+    // SAFETY: data_ptr is valid for [0..out_len] because:
+    // - it came from the handle's internal Bytes (which we control),
+    // - out_len was written by the accessor,
+    // - the handle is still alive (we have not called free yet).
+    let slice = unsafe { std::slice::from_raw_parts(data_ptr, out_len) };
+
+    assert_eq!(slice.len(), 8);
+    assert_eq!(slice[0], 0xFF);
+    assert_eq!(slice[3], 0x00, "must be able to read the embedded NUL");
+    assert_eq!(
+        slice[7], 0xEF,
+        "must be able to read bytes after the NUL (no truncation)"
+    );
+
+    // Cleanup
+    // SAFETY: handle came from from_json; we are the owner.
+    unsafe { kreuzberg_email_attachment_free(handle) };
+}
+
+/// Verify that passing a null out_len pointer is safe: the accessor must not
+/// segfault, and the data pointer itself must still be returned.
+#[test]
+fn email_attachment_data_null_out_len_is_safe() {
+    let json = CString::new(
+        r#"{"name":"hasdata.bin","filename":"hasdata.bin","mime_type":"application/octet-stream","size":4,"is_image":false,"data":[65,0,66,67]}"#,
+    )
+    .unwrap();
+
+    // SAFETY: json is valid.
+    let handle = unsafe { kreuzberg_email_attachment_from_json(json.as_ptr() as *const c_char) };
+    assert!(!handle.is_null());
+
+    // SAFETY: handle is valid; passing null for out_len is a defined contract
+    // (the accessor null-checks before writing).
+    let data_ptr = unsafe { kreuzberg_ffi::kreuzberg_email_attachment_data(handle, std::ptr::null_mut()) };
+
+    assert!(
+        !data_ptr.is_null(),
+        "data pointer should be non-null when the attachment carries a payload"
+    );
+
+    // SAFETY: handle from from_json; we are the owner.
+    unsafe { kreuzberg_email_attachment_free(handle) };
+}
--- a/crates/kreuzberg-ffi/tests/vtable_bytes_len.rs
+++ b/crates/kreuzberg-ffi/tests/vtable_bytes_len.rs
@@ -0,0 +1,204 @@
+/// Regression tests: vtable Bytes params carry companion length
+///
+/// The alef vtable generator previously emitted only `*const u8` for `&[u8]`
+/// trait-method parameters without a companion `{name}_len: usize`. Binary
+/// payloads contain embedded NUL bytes; read-until-NUL semantics silently
+/// truncated every real image or document buffer at the first `0x00`.
+///
+/// Fix shipped in alef ≥ v0.19.21 and is present in the generated FFI shim.
+/// These tests construct a vtable bridge directly, pass a buffer with an
+/// embedded NUL at a known offset, and assert the full buffer is received.
+///
+/// Per-test state is passed via `user_data` — no global statics — so tests
+/// are independent and can run in parallel without interfering.
+use kreuzberg_ffi::{
+    KreuzbergDocumentExtractorBridge, KreuzbergDocumentExtractorVTable, KreuzbergOcrBackendBridge,
+    KreuzbergOcrBackendVTable,
+};
+use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering};
+
+// ── Per-test callback state ───────────────────────────────────────────────
+
+struct CallbackState {
+    received_len: AtomicUsize,
+    received_last_byte: AtomicU8,
+}
+
+impl CallbackState {
+    fn new() -> Self {
+        Self {
+            received_len: AtomicUsize::new(0),
+            received_last_byte: AtomicU8::new(0),
+        }
+    }
+}
+
+// ── C callback stubs ─────────────────────────────────────────────────────
+
+unsafe extern "C" fn ocr_process_image(
+    user_data: *const std::ffi::c_void,
+    image_bytes: *const u8,
+    image_bytes_len: usize,
+    _config: *const std::ffi::c_char,
+    out_result: *mut *mut std::ffi::c_char,
+    out_error: *mut *mut std::ffi::c_char,
+) -> i32 {
+    // SAFETY: user_data points to a CallbackState that the calling test keeps alive.
+    let state = unsafe { &*(user_data as *const CallbackState) };
+    state.received_len.store(image_bytes_len, Ordering::SeqCst);
+    if image_bytes_len > 0 {
+        // SAFETY: caller guarantees image_bytes[0..image_bytes_len] is valid.
+        let last = unsafe { *image_bytes.add(image_bytes_len - 1) };
+        state.received_last_byte.store(last, Ordering::SeqCst);
+    }
+    unsafe { *out_result = std::ptr::null_mut() };
+    let msg = std::ffi::CString::new("stub").unwrap();
+    // SAFETY: caller owns out_error and will free it via kreuzberg_free_string.
+    unsafe { *out_error = msg.into_raw() };
+    1
+}
+
+unsafe extern "C" fn extractor_extract_bytes(
+    user_data: *const std::ffi::c_void,
+    content: *const u8,
+    content_len: usize,
+    _mime_type: *const std::ffi::c_char,
+    _config: *const std::ffi::c_char,
+    out_result: *mut *mut std::ffi::c_char,
+    out_error: *mut *mut std::ffi::c_char,
+) -> i32 {
+    // SAFETY: user_data points to a CallbackState that the calling test keeps alive.
+    let state = unsafe { &*(user_data as *const CallbackState) };
+    state.received_len.store(content_len, Ordering::SeqCst);
+    if content_len > 0 {
+        // SAFETY: caller guarantees content[0..content_len] is valid.
+        let last = unsafe { *content.add(content_len - 1) };
+        state.received_last_byte.store(last, Ordering::SeqCst);
+    }
+    unsafe { *out_result = std::ptr::null_mut() };
+    let msg = std::ffi::CString::new("stub").unwrap();
+    unsafe { *out_error = msg.into_raw() };
+    1
+}
+
+// ── Tests ─────────────────────────────────────────────────────────────────
+
+/// OcrBackend.process_image must pass the full buffer length even when
+/// the payload contains embedded NUL bytes.
+#[tokio::test]
+async fn ocr_backend_vtable_process_image_passes_full_length_with_embedded_nuls() {
+    // 8-byte buffer; NUL at index 3. strlen-style reads would stop at 3.
+    let image_bytes: Vec<u8> = vec![0xFF, 0xD8, 0xFF, 0x00, 0xDE, 0xAD, 0xBE, 0xEF];
+
+    let state = Box::new(CallbackState::new());
+    let state_ptr = state.as_ref() as *const CallbackState as *const std::ffi::c_void;
+
+    let vtable = KreuzbergOcrBackendVTable {
+        process_image: Some(ocr_process_image),
+        process_image_file: None,
+        name_fn: None,
+        version_fn: None,
+        initialize_fn: None,
+        shutdown_fn: None,
+        supports_language: None,
+        backend_type: None,
+        supported_languages: None,
+        supports_table_detection: None,
+        supports_document_processing: None,
+        process_document: None,
+        free_user_data: None,
+    };
+
+    // SAFETY: state lives for the duration of this test and outlives the bridge.
+    let bridge = unsafe { KreuzbergOcrBackendBridge::new("test-ocr-stub".to_string(), vtable, state_ptr) };
+
+    use kreuzberg::OcrBackend;
+    let _ = bridge
+        .process_image(&image_bytes, &kreuzberg::OcrConfig::default())
+        .await;
+
+    assert_eq!(
+        state.received_len.load(Ordering::SeqCst),
+        8,
+        "process_image vtable received wrong length (truncated at embedded NUL?)"
+    );
+    assert_eq!(
+        state.received_last_byte.load(Ordering::SeqCst),
+        0xEF,
+        "process_image vtable could not read past the embedded NUL"
+    );
+}
+
+/// DocumentExtractor.extract_bytes must pass the full buffer length even when
+/// the document bytes contain embedded NUL bytes.
+#[tokio::test]
+async fn document_extractor_vtable_extract_bytes_passes_full_length_with_embedded_nuls() {
+    // 8-byte buffer; NUL at index 2.
+    let content: Vec<u8> = vec![0x50, 0x4B, 0x00, 0x03, 0x14, 0x00, 0x00, 0x02];
+
+    let state = Box::new(CallbackState::new());
+    let state_ptr = state.as_ref() as *const CallbackState as *const std::ffi::c_void;
+
+    let vtable = KreuzbergDocumentExtractorVTable {
+        extract_bytes: Some(extractor_extract_bytes),
+        extract_file: None,
+        name_fn: None,
+        version_fn: None,
+        initialize_fn: None,
+        shutdown_fn: None,
+        supported_mime_types: None,
+        priority: None,
+        can_handle: None,
+        free_user_data: None,
+    };
+
+    // SAFETY: state lives for the duration of this test and outlives the bridge.
+    let bridge = unsafe { KreuzbergDocumentExtractorBridge::new("test-extractor-stub".to_string(), vtable, state_ptr) };
+
+    use kreuzberg::DocumentExtractor;
+    let _ = bridge
+        .extract_bytes(
+            &content,
+            "application/octet-stream",
+            &kreuzberg::ExtractionConfig::default(),
+        )
+        .await;
+
+    assert_eq!(
+        state.received_len.load(Ordering::SeqCst),
+        8,
+        "extract_bytes vtable received wrong length (truncated at embedded NUL?)"
+    );
+    assert_eq!(
+        state.received_last_byte.load(Ordering::SeqCst),
+        0x02,
+        "extract_bytes vtable could not read past the embedded NUL"
+    );
+}
+
+/// ImageKind numeric values: PageRaster must be 10 and Unknown must be 11.
+///
+/// alef ≥ v0.19.21 added PageRaster between Mask (9) and Unknown, bumping
+/// Unknown from 10 → 11. Any C/Go/Java/C# code that hardcoded Unknown = 10
+/// must be updated; this test pins the new ordinals so the renumbering is
+/// visible to CI.
+#[test]
+fn image_kind_page_raster_is_10_and_unknown_is_11() {
+    // SAFETY: pure integer dispatch, no pointers.
+    assert_eq!(
+        unsafe { kreuzberg_ffi::kreuzberg_image_kind_from_i32(10) },
+        10,
+        "PageRaster == 10"
+    );
+    assert_eq!(
+        unsafe { kreuzberg_ffi::kreuzberg_image_kind_from_i32(11) },
+        11,
+        "Unknown == 11"
+    );
+    // Old Unknown value must now resolve to PageRaster, not Unknown.
+    assert_ne!(
+        unsafe { kreuzberg_ffi::kreuzberg_image_kind_from_i32(10) },
+        -1,
+        "10 must be valid"
+    );
+}