This commit is contained in:
Henrik Jess Nielsen
86
templates/readme/partials/badges.html.jinja
Normal file
86
templates/readme/partials/badges.html.jinja
Normal file
@@ -0,0 +1,86 @@
|
||||
<div align="center" style="display: flex; flex-wrap: wrap; gap: 8px; justify-content: center; margin: 20px 0;">
|
||||
<a href="https://github.com/kreuzberg-dev/alef">
|
||||
<img src="https://img.shields.io/badge/Bindings-alef%20%D7%90-007ec6" alt="Bindings">
|
||||
</a>
|
||||
<!-- Language Bindings -->
|
||||
<a href="https://crates.io/crates/kreuzberg">
|
||||
<img src="https://img.shields.io/crates/v/kreuzberg?label=Rust&color=007ec6" alt="Rust">
|
||||
</a>
|
||||
<a href="https://pypi.org/project/kreuzberg/">
|
||||
<img src="https://img.shields.io/pypi/v/kreuzberg?label=Python&color=007ec6" alt="Python">
|
||||
</a>
|
||||
<a href="https://www.npmjs.com/package/@kreuzberg/node">
|
||||
<img src="https://img.shields.io/npm/v/@kreuzberg/node?label=Node.js&color=007ec6" alt="Node.js">
|
||||
</a>
|
||||
<a href="https://www.npmjs.com/package/@kreuzberg/wasm">
|
||||
<img src="https://img.shields.io/npm/v/@kreuzberg/wasm?label=WASM&color=007ec6" alt="WASM">
|
||||
</a>
|
||||
<a href="https://central.sonatype.com/artifact/dev.kreuzberg/kreuzberg">
|
||||
<img src="https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg?label=Java&color=007ec6" alt="Java">
|
||||
</a>
|
||||
<a href="https://github.com/kreuzberg-dev/kreuzberg/tree/main/packages/go/v5">
|
||||
<img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v5*" alt="Go">
|
||||
</a>
|
||||
<a href="https://www.nuget.org/packages/Kreuzberg/">
|
||||
<img src="https://img.shields.io/nuget/v/Kreuzberg?label=C%23&color=007ec6" alt="C#">
|
||||
</a>
|
||||
<a href="https://packagist.org/packages/kreuzberg/kreuzberg">
|
||||
<img src="https://img.shields.io/packagist/v/kreuzberg/kreuzberg?label=PHP&color=007ec6" alt="PHP">
|
||||
</a>
|
||||
<a href="https://rubygems.org/gems/kreuzberg">
|
||||
<img src="https://img.shields.io/gem/v/kreuzberg?label=Ruby&color=007ec6" alt="Ruby">
|
||||
</a>
|
||||
<a href="https://hex.pm/packages/kreuzberg">
|
||||
<img src="https://img.shields.io/hexpm/v/kreuzberg?label=Elixir&color=007ec6" alt="Elixir">
|
||||
</a>
|
||||
<a href="https://kreuzberg-dev.r-universe.dev/kreuzberg">
|
||||
<img src="https://img.shields.io/badge/R-kreuzberg-007ec6" alt="R">
|
||||
</a>
|
||||
<a href="https://pub.dev/packages/kreuzberg">
|
||||
<img src="https://img.shields.io/pub/v/kreuzberg?label=Dart&color=007ec6" alt="Dart">
|
||||
</a>
|
||||
<a href="https://central.sonatype.com/artifact/dev.kreuzberg/kreuzberg-android">
|
||||
<img src="https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg-android?label=Kotlin&color=007ec6" alt="Kotlin">
|
||||
</a>
|
||||
<a href="https://github.com/kreuzberg-dev/kreuzberg/tree/main/packages/swift">
|
||||
<img src="https://img.shields.io/badge/Swift-SPM-007ec6" alt="Swift">
|
||||
</a>
|
||||
<a href="https://github.com/kreuzberg-dev/kreuzberg/tree/main/packages/zig">
|
||||
<img src="https://img.shields.io/badge/Zig-package-007ec6" alt="Zig">
|
||||
</a>
|
||||
<a href="https://github.com/kreuzberg-dev/kreuzberg/releases">
|
||||
<img src="https://img.shields.io/badge/C-FFI-007ec6" alt="C FFI">
|
||||
</a>
|
||||
<a href="https://github.com/kreuzberg-dev/kreuzberg/pkgs/container/kreuzberg">
|
||||
<img src="https://img.shields.io/badge/Docker-ghcr.io-007ec6?logo=docker&logoColor=white" alt="Docker">
|
||||
</a>
|
||||
<a href="https://github.com/kreuzberg-dev/kreuzberg/pkgs/container/charts%2Fkreuzberg">
|
||||
<img src="https://img.shields.io/badge/Helm-ghcr.io-007ec6?logo=helm&logoColor=white" alt="Helm">
|
||||
</a>
|
||||
|
||||
<!-- Project Info -->
|
||||
<a href="https://github.com/kreuzberg-dev/kreuzberg/blob/main/LICENSE">
|
||||
<img src="https://img.shields.io/badge/License-Elastic--2.0-007ec6" alt="License">
|
||||
</a>
|
||||
<a href="https://docs.kreuzberg.dev">
|
||||
<img src="https://img.shields.io/badge/Docs-kreuzberg-007ec6" alt="Documentation">
|
||||
</a>
|
||||
<a href="https://huggingface.co/Kreuzberg">
|
||||
<img src="https://img.shields.io/badge/Hugging%20Face-Kreuzberg-007ec6" alt="Hugging Face">
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div align="center" style="margin: 24px 0 0;">
|
||||
<a href="https://kreuzberg.dev">
|
||||
<img alt="Kreuzberg" src="https://github.com/user-attachments/assets/419fc06c-8313-4324-b159-4b4d3cfce5c0" />
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div align="center" style="display: flex; flex-wrap: wrap; gap: 12px; justify-content: center; margin: 28px 0 24px;">
|
||||
<a href="https://discord.gg/xt9WY3GnKR">
|
||||
<img height="22" src="https://img.shields.io/badge/Discord-Chat-007ec6?logo=discord&logoColor=white" alt="Join Discord">
|
||||
</a>
|
||||
<a href="https://docs.kreuzberg.dev/demo.html">
|
||||
<img height="22" src="https://img.shields.io/badge/Live%20Demo-Open-007ec6?logo=webassembly&logoColor=white" alt="Live Demo">
|
||||
</a>
|
||||
</div>
|
||||
95
templates/readme/partials/features.md.jinja
Normal file
95
templates/readme/partials/features.md.jinja
Normal file
@@ -0,0 +1,95 @@
|
||||
### Supported File Formats (90+)
|
||||
|
||||
90+ file formats across 8 major categories with intelligent format detection and comprehensive metadata extraction.
|
||||
|
||||
#### Office Documents
|
||||
|
||||
| Category | Formats | Capabilities |
|
||||
|----------|---------|--------------|
|
||||
| **Word Processing** | `.docx`, `.docm`, `.dotx`, `.dotm`, `.dot`, `.odt` | Full text, tables, images, metadata, styles |
|
||||
| **Spreadsheets** | `.xlsx`, `.xlsm`, `.xlsb`, `.xls`, `.xla`, `.xlam`, `.xltm`, `.xltx`, `.xlt`, `.ods` | Sheet data, formulas, cell metadata, charts |
|
||||
| **Presentations** | `.pptx`, `.pptm`, `.ppsx`, `.potx`, `.potm`, `.pot`, `.ppt` | Slides, speaker notes, images, metadata |
|
||||
| **PDF** | `.pdf` | Text, tables, images, metadata, OCR support |
|
||||
| **eBooks** | `.epub`, `.fb2` | Chapters, metadata, embedded resources |
|
||||
| **Database** | `.dbf` | Table data extraction, field type support |
|
||||
| **Hangul** | `.hwp`, `.hwpx` | Korean document format, text extraction |
|
||||
|
||||
#### Images (OCR-Enabled)
|
||||
|
||||
| Category | Formats | Features |
|
||||
|----------|---------|----------|
|
||||
| **Raster** | `.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, `.bmp`, `.tiff`, `.tif` | OCR, table detection, EXIF metadata, dimensions, color space |
|
||||
| **Advanced** | `.jp2`, `.jpx`, `.jpm`, `.mj2`, `.jbig2`, `.jb2`, `.pnm`, `.pbm`, `.pgm`, `.ppm` | OCR via hayro-jpeg2000 (pure Rust decoder), JBIG2 support, table detection, format-specific metadata |
|
||||
| **Vector** | `.svg` | DOM parsing, embedded text, graphics metadata |
|
||||
|
||||
#### Web & Data
|
||||
|
||||
| Category | Formats | Features |
|
||||
|----------|---------|----------|
|
||||
| **Markup** | `.html`, `.htm`, `.xhtml`, `.xml`, `.svg` | DOM parsing, metadata (Open Graph, Twitter Card), link extraction |
|
||||
| **Structured Data** | `.json`, `.yaml`, `.yml`, `.toml`, `.csv`, `.tsv` | Schema detection, nested structures, validation |
|
||||
| **Text & Markdown** | `.txt`, `.md`, `.markdown`, `.djot`, `.rst`, `.org`, `.rtf` | CommonMark, GFM, Djot, reStructuredText, Org Mode |
|
||||
|
||||
#### Email & Archives
|
||||
|
||||
| Category | Formats | Features |
|
||||
|----------|---------|----------|
|
||||
| **Email** | `.eml`, `.msg` | Headers, body (HTML/plain), attachments, threading |
|
||||
| **Archives** | `.zip`, `.tar`, `.tgz`, `.gz`, `.7z` | File listing, nested archives, metadata |
|
||||
|
||||
#### Academic & Scientific
|
||||
|
||||
| Category | Formats | Features |
|
||||
|----------|---------|----------|
|
||||
| **Citations** | `.bib`, `.biblatex`, `.ris`, `.nbib`, `.enw`, `.csl` | Structured parsing: RIS (structured), PubMed/MEDLINE, EndNote XML (structured), BibTeX, CSL JSON |
|
||||
| **Scientific** | `.tex`, `.latex`, `.typst`, `.jats`, `.ipynb`, `.docbook` | LaTeX, Jupyter notebooks, PubMed JATS |
|
||||
| **Documentation** | `.opml`, `.pod`, `.mdoc`, `.troff` | Technical documentation formats |
|
||||
|
||||
#### Code Intelligence (300+ Languages)
|
||||
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| **Structure Extraction** | Functions, classes, methods, structs, interfaces, enums |
|
||||
| **Import/Export Analysis** | Module dependencies, re-exports, wildcard imports |
|
||||
| **Symbol Extraction** | Variables, constants, type aliases, properties |
|
||||
| **Docstring Parsing** | Google, NumPy, Sphinx, JSDoc, RustDoc, and 10+ formats |
|
||||
| **Diagnostics** | Parse errors with line/column positions |
|
||||
| **Syntax-Aware Chunking** | Split code by semantic boundaries, not arbitrary byte offsets |
|
||||
|
||||
Powered by [tree-sitter-language-pack](https://github.com/kreuzberg-dev/tree-sitter-language-pack) — [documentation](https://docs.tree-sitter-language-pack.kreuzberg.dev).
|
||||
|
||||
**[Complete Format Reference](https://docs.kreuzberg.dev/reference/formats/)**
|
||||
|
||||
### Key Capabilities
|
||||
|
||||
- **Text Extraction** - Extract all text content with position and formatting information
|
||||
- **Metadata Extraction** - Retrieve document properties, creation date, author, etc.
|
||||
- **Table Extraction** - Parse tables with structure and cell content preservation
|
||||
- **Image Extraction** - Extract embedded images and render page previews
|
||||
- **OCR Support** - Integrate multiple OCR backends for scanned documents
|
||||
{% if features.async %}
|
||||
- **Async/Await** - Non-blocking document processing with concurrent operations
|
||||
{% endif %}
|
||||
{% if features.plugin_system %}
|
||||
- **Plugin System** - Extensible post-processing for custom text transformation
|
||||
{% endif %}
|
||||
{% if features.embeddings %}
|
||||
- **Embeddings** - Generate vector embeddings using ONNX Runtime models
|
||||
{% endif %}
|
||||
- **Batch Processing** - Efficiently process multiple documents in parallel
|
||||
- **Memory Efficient** - Stream large files without loading entirely into memory
|
||||
- **Language Detection** - Detect and support multiple languages in documents
|
||||
{% if features.code_intelligence %}
|
||||
- **Code Intelligence** - Extract structure, imports, exports, symbols, and docstrings from [300+ programming languages](https://docs.tree-sitter-language-pack.kreuzberg.dev) via tree-sitter
|
||||
{% endif %}
|
||||
- **Configuration** - Fine-grained control over extraction behavior
|
||||
|
||||
### Performance Characteristics
|
||||
|
||||
| Format | Speed | Memory | Notes |
|
||||
|--------|-------|--------|-------|
|
||||
| **PDF (text)** | 10-100 MB/s | ~50MB per doc | Fastest extraction |
|
||||
| **Office docs** | 20-200 MB/s | ~100MB per doc | DOCX, XLSX, PPTX |
|
||||
| **Images (OCR)** | 1-5 MB/s | Variable | Depends on OCR backend |
|
||||
| **Archives** | 5-50 MB/s | ~200MB per doc | ZIP, TAR, etc. |
|
||||
| **Web formats** | 50-200 MB/s | Streaming | HTML, XML, JSON |
|
||||
272
templates/readme/partials/installation.md.jinja
Normal file
272
templates/readme/partials/installation.md.jinja
Normal file
@@ -0,0 +1,272 @@
|
||||
### Package Installation
|
||||
|
||||
{% for pm in package_manager %}
|
||||
{% if pm == "pip" %}
|
||||
Install via pip:
|
||||
|
||||
```bash
|
||||
pip install {{ package_name }}
|
||||
```
|
||||
|
||||
For async support and additional features:
|
||||
|
||||
```bash
|
||||
pip install {{ package_name }}[async]
|
||||
```
|
||||
|
||||
{% elif pm == "npm" %}
|
||||
```bash
|
||||
npm install {{ package_name }}
|
||||
```
|
||||
|
||||
{% elif pm == "pnpm" %}
|
||||
```bash
|
||||
pnpm add {{ package_name }}
|
||||
```
|
||||
|
||||
{% elif pm == "yarn" %}
|
||||
```bash
|
||||
yarn add {{ package_name }}
|
||||
```
|
||||
|
||||
{% elif pm == "go get" %}
|
||||
```bash
|
||||
go get {{ package_name }}
|
||||
```
|
||||
|
||||
For more details on FFI setup and native library linking, see the [Go API Reference](https://docs.kreuzberg.dev/reference/api-go/).
|
||||
|
||||
{% elif pm == "maven" %}
|
||||
{% set maven_parts = package_name | split(":") %}
|
||||
Add to your `pom.xml`:
|
||||
|
||||
```xml
|
||||
<dependency>
|
||||
<groupId>{{ maven_parts[0] }}</groupId>
|
||||
<artifactId>{{ maven_parts[1] }}</artifactId>
|
||||
<version>{{ version }}</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
{% elif pm == "gradle" %}
|
||||
Kotlin DSL (`build.gradle.kts`):
|
||||
|
||||
```kotlin
|
||||
implementation("{{ package_name }}:{{ version }}")
|
||||
```
|
||||
|
||||
Groovy DSL (`build.gradle`):
|
||||
|
||||
```groovy
|
||||
implementation '{{ package_name }}:{{ version }}'
|
||||
```
|
||||
|
||||
{% elif pm == "rubygems" %}
|
||||
Install via gem:
|
||||
|
||||
```bash
|
||||
gem install {{ package_name }}
|
||||
```
|
||||
|
||||
{% elif pm == "bundler" %}
|
||||
Or add to your Gemfile:
|
||||
|
||||
```ruby
|
||||
gem '{{ package_name }}'
|
||||
```
|
||||
|
||||
{% elif pm == "composer" %}
|
||||
Install via Composer:
|
||||
|
||||
```bash
|
||||
composer require {{ package_name }}
|
||||
```
|
||||
|
||||
{% elif pm == "mix" %}
|
||||
Add to your `mix.exs` dependencies:
|
||||
|
||||
```elixir
|
||||
def deps do
|
||||
[
|
||||
{:{{ package_name }}, "~> {{ version }}"}
|
||||
]
|
||||
end
|
||||
```
|
||||
|
||||
Then run:
|
||||
|
||||
```bash
|
||||
mix deps.get
|
||||
```
|
||||
|
||||
{% elif pm == "nuget" %}
|
||||
Install via NuGet:
|
||||
|
||||
```bash
|
||||
dotnet add package {{ package_name }}
|
||||
```
|
||||
|
||||
Or via NuGet Package Manager:
|
||||
|
||||
```
|
||||
Install-Package {{ package_name }}
|
||||
```
|
||||
|
||||
{% elif pm == "pub" %}
|
||||
Install via pub:
|
||||
|
||||
```bash
|
||||
dart pub add {{ package_name }}
|
||||
```
|
||||
|
||||
For Flutter projects:
|
||||
|
||||
```bash
|
||||
flutter pub add {{ package_name }}
|
||||
```
|
||||
|
||||
{% elif pm == "spm" %}
|
||||
Add to your `Package.swift` dependencies:
|
||||
|
||||
```swift
|
||||
.package(url: "https://github.com/kreuzberg-dev/kreuzberg.git", from: "{{ version }}"),
|
||||
```
|
||||
|
||||
Then add the product to the relevant target:
|
||||
|
||||
```swift
|
||||
.target(
|
||||
name: "YourTarget",
|
||||
dependencies: [
|
||||
.product(name: "{{ package_name }}", package: "kreuzberg"),
|
||||
]
|
||||
),
|
||||
```
|
||||
|
||||
{% elif pm == "hex" %}
|
||||
Install via Hex:
|
||||
|
||||
```elixir
|
||||
def deps do
|
||||
[
|
||||
{:{{ package_name }}, "~> {{ version }}"}
|
||||
]
|
||||
end
|
||||
```
|
||||
|
||||
{% elif pm == "zig" %}
|
||||
Fetch the package and pin it in `build.zig.zon`:
|
||||
|
||||
```bash
|
||||
zig fetch --save https://github.com/kreuzberg-dev/kreuzberg/archive/refs/tags/v{{ version }}.tar.gz
|
||||
```
|
||||
|
||||
Then wire it into `build.zig`:
|
||||
|
||||
```zig
|
||||
const kreuzberg_dep = b.dependency("{{ package_name }}", .{
|
||||
.target = target,
|
||||
.optimize = optimize,
|
||||
});
|
||||
exe.root_module.addImport("{{ package_name }}", kreuzberg_dep.module("{{ package_name }}"));
|
||||
```
|
||||
|
||||
{% elif pm == "install.packages" %}
|
||||
Install from the kreuzberg R-universe:
|
||||
|
||||
```r
|
||||
install.packages("{{ package_name }}",
|
||||
repos = c("https://kreuzberg-dev.r-universe.dev", getOption("repos")))
|
||||
```
|
||||
|
||||
{% elif pm == "cargo" %}
|
||||
Build the shared library from the workspace:
|
||||
|
||||
```bash
|
||||
cargo build --release -p {{ package_name }}
|
||||
```
|
||||
|
||||
The built artifacts are emitted under `target/release/` (`lib{{ package_name | replace("-", "_") }}.{so,dylib,a}`) along with the C header at `crates/{{ package_name }}/include/kreuzberg.h`.
|
||||
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
### System Requirements
|
||||
{% if language == "python" %}
|
||||
- **Python 3.10+** required
|
||||
- Optional: [ONNX Runtime](https://github.com/microsoft/onnxruntime/releases) version 1.22.x for embeddings support
|
||||
- Optional: [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) for OCR functionality
|
||||
{% elif language == "typescript" %}
|
||||
- **Node.js 22+** required (NAPI-RS native bindings)
|
||||
- Optional: [ONNX Runtime](https://github.com/microsoft/onnxruntime/releases) version 1.22.x for embeddings support
|
||||
- Optional: [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) for OCR functionality
|
||||
|
||||
### Platform Support
|
||||
|
||||
Pre-built binaries available for:
|
||||
- macOS (arm64, x64)
|
||||
- Linux (x64)
|
||||
- Windows (x64)
|
||||
|
||||
{% elif language == "go" %}
|
||||
- **Go 1.19+** required
|
||||
- Optional: [ONNX Runtime](https://github.com/microsoft/onnxruntime/releases) version 1.22.x for embeddings support
|
||||
- Optional: [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) for OCR functionality
|
||||
{% elif language == "java" %}
|
||||
- **Java 25+** required (Foreign Function & Memory API; build run with `--enable-preview` and `--enable-native-access=ALL-UNNAMED`)
|
||||
- Native libraries bundled in the JAR for macOS (arm64, x64), Linux (x64, arm64), and Windows (x64)
|
||||
- Optional: [ONNX Runtime](https://github.com/microsoft/onnxruntime/releases) version 1.22.x for embeddings support
|
||||
- Optional: [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) for OCR functionality
|
||||
{% elif language == "kotlin" %}
|
||||
- **JDK 25+** required
|
||||
- Native libraries bundled via the Java facade JAR
|
||||
- Optional: [ONNX Runtime](https://github.com/microsoft/onnxruntime/releases) version 1.22.x for embeddings support
|
||||
- Optional: [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) for OCR functionality
|
||||
{% elif language == "ruby" %}
|
||||
- **Ruby 3.2.0 or higher** required (including Ruby 4.x)
|
||||
- Ruby 4.0+ is fully supported with no code changes required
|
||||
- Optional: [ONNX Runtime](https://github.com/microsoft/onnxruntime/releases) version 1.22.x for embeddings support
|
||||
- Optional: [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) for OCR functionality
|
||||
{% elif language == "php" %}
|
||||
- **PHP 8.2+** required
|
||||
- Optional: [ONNX Runtime](https://github.com/microsoft/onnxruntime/releases) version 1.22.x for embeddings support
|
||||
- Optional: [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) for OCR functionality
|
||||
{% elif language == "csharp" %}
|
||||
- **.NET 10.0+** required
|
||||
- Optional: [ONNX Runtime](https://github.com/microsoft/onnxruntime/releases) version 1.22.x for embeddings support
|
||||
- Optional: [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) for OCR functionality
|
||||
{% elif language == "elixir" %}
|
||||
- **Elixir 1.14+** and **Erlang/OTP 26+** required
|
||||
- Pre-compiled NIFs bundled via `rustler_precompiled` for macOS (arm64, x64), Linux (x64, arm64), and Windows (x64)
|
||||
- Optional: [ONNX Runtime](https://github.com/microsoft/onnxruntime/releases) version 1.22.x for embeddings support
|
||||
- Optional: [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) for OCR functionality
|
||||
{% elif language == "wasm" %}
|
||||
- Modern browser with WebAssembly support, or Deno 1.0+, or Cloudflare Workers
|
||||
- Optional: [Tesseract WASM](https://github.com/naptha/tesseract.js) for OCR functionality
|
||||
{% elif language == "r" %}
|
||||
- **R 4.1+** required (extendr bindings)
|
||||
- Optional: [ONNX Runtime](https://github.com/microsoft/onnxruntime/releases) version 1.22.x for embeddings support
|
||||
- Optional: [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) for OCR functionality
|
||||
{% elif language == "ffi" %}
|
||||
- A C/C++ toolchain (clang, gcc, or MSVC) and a Rust toolchain (`rustup`) for building from source
|
||||
- A `pkg-config` or CMake-aware build system that can locate `libkreuzberg_ffi` and `kreuzberg.h`
|
||||
- Optional: [ONNX Runtime](https://github.com/microsoft/onnxruntime/releases) version 1.22.x for embeddings support
|
||||
- Optional: [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) for OCR functionality
|
||||
{% elif language == "dart" %}
|
||||
- **Dart SDK 3.0+** for pure-Dart consumers
|
||||
- Flutter projects supported on macOS, iOS, Android, Linux, and Windows; Flutter Web is not supported
|
||||
- Native runtime delivered via `flutter_rust_bridge` with bundled binaries for the supported platforms
|
||||
- Optional: [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) for OCR functionality
|
||||
{% elif language == "swift" %}
|
||||
- **Swift 6.0+** (`swift-tools-version: 6.0`) on macOS 13+ or iOS 16+
|
||||
- Native runtime delivered through the C FFI surface from `kreuzberg-ffi`; published artifacts ship as a binary target
|
||||
- Optional: [ONNX Runtime](https://github.com/microsoft/onnxruntime/releases) version 1.22.x for embeddings support
|
||||
- Optional: [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) for OCR functionality
|
||||
{% elif language == "zig" %}
|
||||
- **Zig 0.16.0+** required (`minimum_zig_version` declared in `build.zig.zon`)
|
||||
- Links the C FFI surface from `kreuzberg-ffi`; the build resolves the library via `linkSystemLibrary` against the consumer-provided search path
|
||||
- Optional: [ONNX Runtime](https://github.com/microsoft/onnxruntime/releases) version 1.22.x for embeddings support
|
||||
- Optional: [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) for OCR functionality
|
||||
{% else %}
|
||||
- See [Installation Guide](https://docs.kreuzberg.dev/getting-started/installation/) for requirements
|
||||
{% endif %}
|
||||
24
templates/readme/partials/napi_implementation.md.jinja
Normal file
24
templates/readme/partials/napi_implementation.md.jinja
Normal file
@@ -0,0 +1,24 @@
|
||||
## NAPI-RS Implementation Details
|
||||
|
||||
### Native Performance
|
||||
|
||||
This binding uses NAPI-RS to provide native Node.js bindings with:
|
||||
|
||||
- **Zero-copy data transfer** between JavaScript and Rust layers
|
||||
- **Native thread pool** for concurrent document processing
|
||||
- **Direct memory management** for efficient large document handling
|
||||
- **Binary-compatible** pre-built native modules across platforms
|
||||
|
||||
### Threading Model
|
||||
|
||||
- Single documents are processed synchronously or asynchronously in a dedicated thread
|
||||
- Batch operations distribute work across available CPU cores
|
||||
- Thread count is configurable but defaults to system CPU count
|
||||
- Long-running extractions block the event loop unless using async APIs
|
||||
|
||||
### Memory Management
|
||||
|
||||
- Large documents (> 100 MB) are streamed to avoid loading entirely into memory
|
||||
- Temporary files are created in system temp directory for extraction
|
||||
- Memory is automatically released after extraction completion
|
||||
- ONNX models are cached in memory for repeated embeddings operations
|
||||
77
templates/readme/partials/quick_start.md.jinja
Normal file
77
templates/readme/partials/quick_start.md.jinja
Normal file
@@ -0,0 +1,77 @@
|
||||
### Basic Extraction
|
||||
|
||||
Extract text, metadata, and structure from any supported document format:
|
||||
|
||||
{{ snippets.basic_extraction | include_snippet(language) }}
|
||||
|
||||
### Common Use Cases
|
||||
|
||||
#### Extract with Custom Configuration
|
||||
|
||||
Most use cases benefit from configuration to control extraction behavior:
|
||||
|
||||
{% if snippets.ocr_configuration %}
|
||||
**With OCR (for scanned documents):**
|
||||
|
||||
{{ snippets.ocr_configuration | include_snippet(language) }}
|
||||
|
||||
{% endif %}
|
||||
|
||||
#### Table Extraction
|
||||
|
||||
{% if snippets.table_extraction %}
|
||||
{{ snippets.table_extraction | include_snippet(language) }}
|
||||
|
||||
{% else %}
|
||||
See [Configuration Guide](https://docs.kreuzberg.dev/guides/configuration/) for table extraction options.
|
||||
|
||||
{% endif %}
|
||||
|
||||
#### Processing Multiple Files
|
||||
|
||||
{% if snippets.batch_processing %}
|
||||
{{ snippets.batch_processing | include_snippet(language) }}
|
||||
|
||||
{% endif %}
|
||||
|
||||
{% if snippets.async_extraction %}
|
||||
#### Async Processing
|
||||
|
||||
For non-blocking document processing:
|
||||
|
||||
{{ snippets.async_extraction | include_snippet(language) }}
|
||||
|
||||
{% endif %}
|
||||
{% if snippets.config_discovery %}
|
||||
|
||||
#### Configuration Discovery
|
||||
|
||||
{{ snippets.config_discovery | include_snippet(language) }}
|
||||
|
||||
{% endif %}
|
||||
{% if snippets.worker_pool %}
|
||||
|
||||
#### Worker Thread Pool
|
||||
|
||||
{{ snippets.worker_pool | include_snippet(language) }}
|
||||
|
||||
**Performance Benefits:**
|
||||
- **Parallel Processing**: Multiple documents extracted simultaneously
|
||||
- **CPU Utilization**: Maximizes multi-core CPU usage for large batches
|
||||
- **Queue Management**: Automatically distributes work across available workers
|
||||
- **Resource Control**: Prevents thread exhaustion with configurable pool size
|
||||
|
||||
**Best Practices:**
|
||||
- Use worker pools for batches of 10+ documents
|
||||
- Set pool size to number of CPU cores (default behavior)
|
||||
- Always close pools with `closeWorkerPool()` to prevent resource leaks
|
||||
- Reuse pools across multiple batch operations for efficiency
|
||||
|
||||
{% endif %}
|
||||
|
||||
### Next Steps
|
||||
|
||||
- **[Installation Guide](https://docs.kreuzberg.dev/getting-started/installation/)** - Platform-specific setup
|
||||
- **[API Documentation](https://docs.kreuzberg.dev/reference/api-python/)** - Complete API reference
|
||||
- **[Examples & Guides](https://docs.kreuzberg.dev/)** - Full code examples and usage guides
|
||||
- **[Configuration Guide](https://docs.kreuzberg.dev/guides/configuration/)** - Advanced configuration options
|
||||
Reference in New Issue
Block a user