Nomad changes

crates/kreuzberg/src/api/config.rs

@@ -0,0 +1,68 @@
+//! API server configuration loading.
+
+use crate::{Result, core::ServerConfig};
+
+/// Load ServerConfig with proper precedence order.
+///
+/// This function implements the configuration hierarchy:
+/// 1. File (if provided)
+/// 2. Environment variables (via apply_env_overrides)
+/// 3. Defaults
+///
+/// The config file can be in flat format (server settings at root) or nested format
+/// (server settings under [server] section alongside other configs like [ocr]).
+///
+/// # Arguments
+///
+/// * `config_path` - Optional path to a ServerConfig file (TOML, YAML, or JSON)
+///
+/// # Returns
+///
+/// A configured ServerConfig with proper precedence applied.
+///
+/// # Errors
+///
+/// Returns an error if:
+/// - The config file path is provided but cannot be read
+/// - The config file contains invalid server configuration
+/// - Environment variable overrides contain invalid values
+///
+/// # Examples
+///
+/// ```ignore
+/// use kreuzberg::api::load_server_config;
+///
+/// # fn example() -> kreuzberg::Result<()> {
+/// // Load from file with env overrides
+/// let config = load_server_config(Some("server.toml"))?;
+///
+/// // Or use defaults with env overrides
+/// let config = load_server_config(None)?;
+/// # Ok(())
+/// # }
+/// ```
+pub(crate) fn load_server_config(config_path: Option<&str>) -> Result<ServerConfig> {
+    let mut config = if let Some(path) = config_path.map(std::path::Path::new) {
+        ServerConfig::from_file(path)?
+    } else {
+        ServerConfig::default()
+    };
+
+    // Apply environment variable overrides with proper logging
+    config.apply_env_overrides()?;
+
+    tracing::info!(
+        "Server configuration loaded: host={}, port={}, request_body_limit={} MB, multipart_field_limit={} MB, CORS={}",
+        config.host,
+        config.port,
+        config.max_request_body_mb(),
+        config.max_multipart_field_mb(),
+        if config.cors_allows_all() {
+            "allow all origins".to_string()
+        } else {
+            format!("{} specific origins", config.cors_origins.len())
+        }
+    );
+
+    Ok(config)
+}

crates/kreuzberg/src/api/error.rs

@@ -0,0 +1,215 @@
+//! API error handling.
+
+use axum::{
+    Json,
+    body::to_bytes,
+    extract::{FromRequest, Multipart, Request, rejection::JsonRejection},
+    http::StatusCode,
+    response::{IntoResponse, Response},
+};
+use serde::de::DeserializeOwned;
+
+use crate::error::KreuzbergError;
+
+use super::types::ErrorResponse;
+
+/// Custom JSON extractor that returns JSON error responses instead of plain text.
+///
+/// This wraps axum's `Json` extractor but uses `ApiError` as the rejection type,
+/// ensuring that all JSON parsing errors are returned as JSON with proper content type.
+///
+/// Additionally, this extractor validates that the root JSON value is an object (not an array),
+/// which prevents serde from incorrectly deserializing JSON arrays into struct fields.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct JsonApi<T>(pub T);
+
+impl<T, S> FromRequest<S> for JsonApi<T>
+where
+    T: DeserializeOwned,
+    S: Send + Sync,
+{
+    type Rejection = ApiError;
+
+    async fn from_request(req: Request, state: &S) -> Result<Self, Self::Rejection> {
+        // First, extract the body to check if it's a valid JSON object (not array)
+        let (parts, body) = req.into_parts();
+        let bytes = to_bytes(body, usize::MAX).await.map_err(|_| {
+            ApiError::new(
+                StatusCode::BAD_REQUEST,
+                KreuzbergError::Other("Failed to read request body".to_string()),
+            )
+        })?;
+
+        // Validate that the root JSON is an object, not an array
+        if !bytes.is_empty() {
+            let trimmed = std::str::from_utf8(&bytes).unwrap_or("").trim_start();
+            if trimmed.starts_with('[') {
+                return Err(ApiError::new(
+                    StatusCode::BAD_REQUEST,
+                    KreuzbergError::validation(
+                        "Expected JSON object, but received JSON array. \
+                         Please wrap your data in an object with appropriate fields.",
+                    ),
+                ));
+            }
+        }
+
+        // Reconstruct the request and use the standard Json extractor
+        let req = Request::from_parts(parts, axum::body::Body::from(bytes));
+        match Json::<T>::from_request(req, state).await {
+            Ok(Json(value)) => Ok(JsonApi(value)),
+            Err(rejection) => Err(ApiError::from(rejection)),
+        }
+    }
+}
+
+/// Custom Multipart extractor that returns JSON error responses instead of plain text.
+///
+/// This wraps axum's `Multipart` extractor but uses `ApiError` as the rejection type,
+/// ensuring that multipart parsing errors are returned as JSON with proper content type.
+pub struct MultipartApi(pub Multipart);
+
+impl<S> FromRequest<S> for MultipartApi
+where
+    S: Send + Sync,
+{
+    type Rejection = ApiError;
+
+    async fn from_request(req: Request, state: &S) -> Result<Self, Self::Rejection> {
+        match Multipart::from_request(req, state).await {
+            Ok(multipart) => Ok(MultipartApi(multipart)),
+            Err(rejection) => Err(ApiError {
+                status: StatusCode::BAD_REQUEST,
+                body: ErrorResponse {
+                    error_type: "MultipartError".to_string(),
+                    message: rejection.body_text(),
+                    traceback: None,
+                    status_code: StatusCode::BAD_REQUEST.as_u16(),
+                },
+            }),
+        }
+    }
+}
+
+/// API-specific error wrapper.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug)]
+pub struct ApiError {
+    /// HTTP status code
+    pub status: StatusCode,
+    /// Error response body
+    pub body: ErrorResponse,
+}
+
+impl ApiError {
+    /// Create a new API error.
+    pub(crate) fn new(status: StatusCode, error: KreuzbergError) -> Self {
+        let error_type = match &error {
+            KreuzbergError::Validation { .. } => "ValidationError",
+            KreuzbergError::Parsing { .. } => "ParsingError",
+            KreuzbergError::Ocr { .. } => "OCRError",
+            KreuzbergError::Io(_) => "IOError",
+            KreuzbergError::Cache { .. } => "CacheError",
+            KreuzbergError::ImageProcessing { .. } => "ImageProcessingError",
+            KreuzbergError::Serialization { .. } => "SerializationError",
+            KreuzbergError::MissingDependency(_) => "MissingDependencyError",
+            KreuzbergError::Plugin { .. } => "PluginError",
+            KreuzbergError::LockPoisoned(_) => "LockPoisonedError",
+            KreuzbergError::UnsupportedFormat(_) => "UnsupportedFormatError",
+            KreuzbergError::Embedding { .. } => "EmbeddingError",
+            KreuzbergError::Timeout { .. } => "TimeoutError",
+            KreuzbergError::Other(_) => "Error",
+            KreuzbergError::Cancelled => "CancelledError",
+            KreuzbergError::Security { .. } => "SecurityError",
+        };
+
+        Self {
+            status,
+            body: ErrorResponse {
+                error_type: error_type.to_string(),
+                message: error.to_string(),
+                traceback: None,
+                status_code: status.as_u16(),
+            },
+        }
+    }
+
+    /// Create a validation error (400).
+    #[cfg_attr(alef, alef(skip))]
+    pub(crate) fn validation(error: KreuzbergError) -> Self {
+        Self::new(StatusCode::BAD_REQUEST, error)
+    }
+
+    /// Create an unprocessable entity error (422).
+    #[cfg_attr(alef, alef(skip))]
+    pub(crate) fn unprocessable(error: KreuzbergError) -> Self {
+        Self::new(StatusCode::UNPROCESSABLE_ENTITY, error)
+    }
+
+    /// Create an internal server error (500).
+    #[cfg_attr(alef, alef(skip))]
+    pub(crate) fn internal(error: KreuzbergError) -> Self {
+        Self::new(StatusCode::INTERNAL_SERVER_ERROR, error)
+    }
+
+    /// Create a bad gateway error (502).
+    ///
+    /// Use when an upstream service (e.g., model download from HuggingFace) fails.
+    #[cfg(any(feature = "paddle-ocr", feature = "layout-detection", feature = "embeddings"))]
+    #[cfg_attr(alef, alef(skip))]
+    pub(crate) fn bad_gateway(error: KreuzbergError) -> Self {
+        Self::new(StatusCode::BAD_GATEWAY, error)
+    }
+}
+
+impl IntoResponse for ApiError {
+    fn into_response(self) -> Response {
+        (self.status, Json(self.body)).into_response()
+    }
+}
+
+impl From<KreuzbergError> for ApiError {
+    fn from(error: KreuzbergError) -> Self {
+        match &error {
+            KreuzbergError::Validation { .. } | KreuzbergError::UnsupportedFormat(_) => Self::validation(error),
+            KreuzbergError::Parsing { .. } | KreuzbergError::Ocr { .. } => Self::unprocessable(error),
+            _ => Self::internal(error),
+        }
+    }
+}
+
+impl From<JsonRejection> for ApiError {
+    fn from(rejection: JsonRejection) -> Self {
+        let (status, message) = match rejection {
+            JsonRejection::JsonDataError(err) => (
+                StatusCode::UNPROCESSABLE_ENTITY,
+                format!(
+                    "Failed to deserialize the JSON body into the target type: {}",
+                    err.body_text()
+                ),
+            ),
+            JsonRejection::JsonSyntaxError(err) => (
+                StatusCode::BAD_REQUEST,
+                format!("Failed to parse the request body as JSON: {}", err.body_text()),
+            ),
+            JsonRejection::MissingJsonContentType(_) => (
+                StatusCode::UNSUPPORTED_MEDIA_TYPE,
+                "Expected request with `Content-Type: application/json`".to_string(),
+            ),
+            JsonRejection::BytesRejection(err) => {
+                (StatusCode::BAD_REQUEST, format!("Failed to read request body: {}", err))
+            }
+            _ => (StatusCode::BAD_REQUEST, "Unknown JSON parsing error".to_string()),
+        };
+
+        Self {
+            status,
+            body: ErrorResponse {
+                error_type: "JsonParsingError".to_string(),
+                message,
+                traceback: None,
+                status_code: status.as_u16(),
+            },
+        }
+    }
+}

crates/kreuzberg/src/api/jobs.rs

@@ -0,0 +1,235 @@
+//! In-memory job store for async extraction polling.
+
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::time::Duration;
+
+use moka::sync::Cache;
+
+use crate::api::types::{JobState, JobStatus};
+
+/// Default time-to-live for completed/failed jobs (5 minutes).
+const JOB_TTL: Duration = Duration::from_secs(300);
+
+/// Maximum number of concurrent jobs held in the cache.
+const MAX_CAPACITY: u64 = 10_000;
+
+/// Maximum number of jobs in Pending or Running state at any one time.
+pub const MAX_ACTIVE_JOBS: usize = 100;
+
+/// Thread-safe in-memory store for async extraction jobs.
+///
+/// Uses [`moka::sync::Cache`] with built-in TTL eviction — no background
+/// eviction task required. Entries are evicted automatically after 5 minutes.
+/// Server restarts clear all active jobs.
+#[derive(Clone)]
+pub struct JobStore {
+    jobs: Cache<String, JobStatus>,
+    active: Arc<AtomicUsize>,
+}
+
+impl Default for JobStore {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl JobStore {
+    /// Create a new empty job store with default TTL and capacity.
+    pub fn new() -> Self {
+        let jobs = Cache::builder()
+            .max_capacity(MAX_CAPACITY)
+            .time_to_live(JOB_TTL)
+            .build();
+        Self {
+            jobs,
+            active: Arc::new(AtomicUsize::new(0)),
+        }
+    }
+
+    /// Return the number of jobs currently in Pending or Running state.
+    pub fn active_count(&self) -> usize {
+        self.active.load(Ordering::Relaxed)
+    }
+
+    /// Create a new job in the store and return its ID.
+    ///
+    /// This handles ID generation and initial state registration in one atomic step.
+    pub fn create_job(&self) -> String {
+        let job_id = generate_job_id();
+        let now = now_rfc3339();
+        self.active.fetch_add(1, Ordering::Relaxed);
+        self.create(job_id.clone(), now);
+        job_id
+    }
+
+    /// Register a new job in `Pending` state. Returns its initial `JobStatus`.
+    pub fn create(&self, job_id: String, timestamp: String) -> JobStatus {
+        let status = JobStatus {
+            job_id: job_id.clone(),
+            state: JobState::Pending,
+            created_at: timestamp.clone(),
+            updated_at: timestamp,
+            result: None,
+            error: None,
+        };
+        self.jobs.insert(job_id, status.clone());
+        status
+    }
+
+    /// Retrieve the current status of a job by ID.
+    pub fn get(&self, job_id: &str) -> Option<JobStatus> {
+        self.jobs.get(job_id)
+    }
+
+    /// Transition a job to the `Running` state.
+    pub fn set_running(&self, job_id: &str, timestamp: String) {
+        if let Some(mut status) = self.jobs.get(job_id) {
+            status.state = JobState::Running;
+            status.updated_at = timestamp;
+            self.jobs.insert(job_id.to_string(), status);
+        }
+    }
+
+    /// Mark a job as `Completed` and store its result.
+    pub fn complete(&self, job_id: &str, result: serde_json::Value, timestamp: String) {
+        if let Some(mut status) = self.jobs.get(job_id) {
+            status.state = JobState::Completed;
+            status.result = Some(result);
+            status.updated_at = timestamp;
+            self.jobs.insert(job_id.to_string(), status);
+            self.active.fetch_sub(1, Ordering::Relaxed);
+        }
+    }
+
+    /// Mark a job as `Failed` and store the error message.
+    pub fn fail(&self, job_id: &str, error: String, timestamp: String) {
+        if let Some(mut status) = self.jobs.get(job_id) {
+            status.state = JobState::Failed;
+            status.error = Some(error);
+            status.updated_at = timestamp;
+            self.jobs.insert(job_id.to_string(), status);
+            self.active.fetch_sub(1, Ordering::Relaxed);
+        }
+    }
+}
+
+/// Generate a new unique job ID (UUID v4).
+pub fn generate_job_id() -> String {
+    uuid::Uuid::new_v4().to_string()
+}
+
+/// Return the current time formatted as RFC 3339 (ISO 8601).
+pub fn now_rfc3339() -> String {
+    chrono::Utc::now().to_rfc3339()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::api::types::JobState;
+
+    #[test]
+    fn test_create_job_is_pending() {
+        let store = JobStore::new();
+        let ts = "2026-05-01T12:00:00Z".to_string();
+        let status = store.create("job-1".to_string(), ts.clone());
+        assert_eq!(status.state, JobState::Pending);
+        assert_eq!(status.job_id, "job-1");
+        assert_eq!(status.created_at, ts);
+    }
+
+    #[test]
+    fn test_get_existing_job() {
+        let store = JobStore::new();
+        store.create("job-2".to_string(), "2026-05-01T12:00:00Z".to_string());
+        let got = store.get("job-2");
+        assert!(got.is_some());
+        assert_eq!(got.unwrap().job_id, "job-2");
+    }
+
+    #[test]
+    fn test_get_missing_job_returns_none() {
+        let store = JobStore::new();
+        assert!(store.get("nope").is_none());
+    }
+
+    #[test]
+    fn test_set_running_transitions_state() {
+        let store = JobStore::new();
+        store.create("job-3".to_string(), "2026-05-01T12:00:00Z".to_string());
+        store.set_running("job-3", "2026-05-01T12:00:01Z".to_string());
+        let status = store.get("job-3").unwrap();
+        assert_eq!(status.state, JobState::Running);
+    }
+
+    #[test]
+    fn test_complete_stores_result() {
+        let store = JobStore::new();
+        store.create("job-4".to_string(), "2026-05-01T12:00:00Z".to_string());
+        store.complete(
+            "job-4",
+            serde_json::json!({"content": "hello"}),
+            "2026-05-01T12:00:02Z".to_string(),
+        );
+        let status = store.get("job-4").unwrap();
+        assert_eq!(status.state, JobState::Completed);
+        assert!(status.result.is_some());
+    }
+
+    #[test]
+    fn test_fail_stores_error() {
+        let store = JobStore::new();
+        store.create("job-5".to_string(), "2026-05-01T12:00:00Z".to_string());
+        store.fail(
+            "job-5",
+            "OCR unavailable".to_string(),
+            "2026-05-01T12:00:03Z".to_string(),
+        );
+        let status = store.get("job-5").unwrap();
+        assert_eq!(status.state, JobState::Failed);
+        assert_eq!(status.error.as_deref(), Some("OCR unavailable"));
+    }
+
+    #[test]
+    fn test_generate_job_id_is_valid_uuid() {
+        let id = generate_job_id();
+        assert!(!id.is_empty());
+        assert!(
+            uuid::Uuid::parse_str(&id).is_ok(),
+            "generated job ID must be a valid UUID: {id}"
+        );
+    }
+
+    #[test]
+    fn test_create_job_helper() {
+        let store = JobStore::new();
+        let id = store.create_job();
+        assert!(!id.is_empty());
+        let status = store.get(&id).expect("job must be created");
+        assert_eq!(status.state, JobState::Pending);
+    }
+}
+
+#[test]
+fn test_create_job_concurrent_uniqueness() {
+    use std::sync::Arc;
+    use std::thread;
+
+    let store = Arc::new(JobStore::new());
+    let mut handles = vec![];
+
+    for _ in 0..100 {
+        let store_clone = Arc::clone(&store);
+        handles.push(thread::spawn(move || store_clone.create_job()));
+    }
+
+    let mut job_ids = std::collections::HashSet::new();
+    for handle in handles {
+        let id = handle.join().unwrap();
+        assert!(job_ids.insert(id.clone()), "Duplicate job ID generated: {}", id);
+    }
+
+    assert_eq!(job_ids.len(), 100);
+}

crates/kreuzberg/src/api/mod.rs

@@ -0,0 +1,116 @@
+//! REST API server for Kreuzberg document extraction.
+//!
+//! This module provides an Axum-based HTTP server for document extraction
+//! with endpoints for single and batch extraction operations.
+//!
+//! # Endpoints
+//!
+//! - `POST /extract` - Extract text from uploaded files (multipart form data)
+//! - `POST /extract-structured` - Extract structured data via LLM with JSON schema (multipart form data)
+//! - `POST /detect` - Detect MIME type of an uploaded file (multipart form data)
+//! - `POST /embed` - Generate embeddings for text (JSON body with texts array)
+//! - `POST /chunk` - Chunk text into smaller pieces (JSON body with text and config)
+//! - `GET /health` - Health check endpoint
+//! - `GET /info` - Server information
+//! - `GET /version` - Version information
+//! - `GET /cache/stats` - Get cache statistics
+//! - `GET /cache/manifest` - Get model manifest with checksums and sizes
+//! - `POST /cache/warm` - Pre-download models to cache
+//! - `DELETE /cache/clear` - Clear all cached files
+//! - `PUT /process` - OpenWebUI "External" engine compatibility
+//! - `POST /v1/convert/file` - OpenWebUI "Docling" engine compatibility (docling-serve drop-in)
+//!
+//! # Examples
+//!
+//! ## Starting the server
+//!
+//! ```no_run
+//! use kreuzberg::api::serve;
+//!
+//! #[tokio::main]
+//! async fn main() -> kreuzberg::Result<()> {
+//!     // Local development
+//!     serve("127.0.0.1", 8000).await?;
+//!     Ok(())
+//! }
+//! ```
+//!
+//! ## Embedding the router in your app
+//!
+//! ```no_run
+//! use kreuzberg::{ExtractionConfig, api::create_router};
+//! use axum::Router;
+//!
+//! #[tokio::main]
+//! async fn main() -> kreuzberg::Result<()> {
+//!     // Load config (from file or use default)
+//!     let config = ExtractionConfig::default();
+//!     let kreuzberg_router = create_router(config);
+//!
+//!     // Nest under /api prefix
+//!     let app = Router::new().nest("/api", kreuzberg_router);
+//!
+//!     // Add your own routes
+//!     // ...
+//!
+//!     Ok(())
+//! }
+//! ```
+//!
+//! # cURL Examples
+//!
+//! ```bash
+//! # Single file extraction
+//! curl -F "files=@document.pdf" http://localhost:8000/extract
+//!
+//! # Multiple files with OCR config
+//! curl -F "files=@doc1.pdf" -F "files=@doc2.jpg" \
+//!      -F 'config={"ocr":{"language":"eng"}}' \
+//!      http://localhost:8000/extract
+//!
+//! # Health check
+//! curl http://localhost:8000/health
+//!
+//! # Server info
+//! curl http://localhost:8000/info
+//!
+//! # Cache statistics
+//! curl http://localhost:8000/cache/stats
+//!
+//! # Clear cache
+//! curl -X DELETE http://localhost:8000/cache/clear
+//!
+//! # Generate embeddings
+//! curl -X POST http://localhost:8000/embed \
+//!      -H "Content-Type: application/json" \
+//!      -d '{"texts":["Hello world","Second text"]}'
+//!
+//! # Chunk text
+//! curl -X POST http://localhost:8000/chunk \
+//!      -H "Content-Type: application/json" \
+//!      -d '{"text":"Long text to chunk...","chunker_type":"text"}'
+//! ```
+
+mod config;
+mod error;
+mod handlers;
+#[cfg(feature = "api")]
+pub(crate) mod jobs;
+#[cfg(feature = "api")]
+pub mod openapi;
+mod openweb;
+mod router;
+mod startup;
+mod types;
+
+pub use error::ApiError;
+#[allow(unused_imports)]
+pub use router::{create_router, create_router_with_limits};
+#[allow(unused_imports)]
+pub use startup::{serve, serve_default, serve_with_config, serve_with_config_and_limits, serve_with_server_config};
+pub use types::{
+    ApiSizeLimits, ApiState, CacheClearResponse, CacheStatsResponse, ChunkRequest, ChunkResponse, DetectResponse,
+    DoclingCompatDocument, DoclingCompatResponse, EmbedRequest, EmbedResponse, ErrorResponse, ExtractResponse,
+    HealthResponse, InfoResponse, ManifestEntryResponse, ManifestResponse, OpenWebDocumentMetadata,
+    OpenWebDocumentResponse, StructuredExtractionResponse, VersionResponse, WarmRequest, WarmResponse,
+};

crates/kreuzberg/src/api/openapi.rs

@@ -0,0 +1,193 @@
+//! OpenAPI 3.1 schema generation for Kreuzberg API.
+//!
+//! This module generates OpenAPI documentation from Rust types using utoipa.
+//! The schema is available at the `/openapi.json` endpoint.
+
+#[cfg(feature = "api")]
+use utoipa::OpenApi;
+
+/// OpenAPI documentation structure.
+///
+/// Defines all endpoints, request/response schemas, and examples
+/// for the Kreuzberg document extraction API.
+#[cfg(feature = "api")]
+#[derive(OpenApi)]
+#[openapi(
+    info(
+        title = "Kreuzberg API",
+        version = env!("CARGO_PKG_VERSION"),
+        description = "High-performance document intelligence API for extracting text, metadata, and structured data from PDFs, Office documents, images, and 90+ formats.",
+        contact(
+            name = "Kreuzberg",
+            url = "https://kreuzberg.dev"
+        ),
+        license(
+            name = "Apache-2.0 OR MIT"
+        )
+    ),
+    servers(
+        (url = "http://localhost:8000", description = "Local development server"),
+        (url = "https://api.kreuzberg.dev", description = "Production server (example)")
+    ),
+    paths(
+        crate::api::handlers::health_handler,
+        crate::api::handlers::info_handler,
+        crate::api::handlers::extract_handler,
+        crate::api::handlers::extract_structured_handler,
+        crate::api::handlers::detect_handler,
+        crate::api::handlers::formats_handler,
+        crate::api::handlers::cache_stats_handler,
+        crate::api::handlers::cache_clear_handler,
+        crate::api::handlers::cache_manifest_handler,
+        crate::api::handlers::cache_warm_handler,
+        crate::api::handlers::embed_handler,
+        crate::api::handlers::chunk_handler,
+        crate::api::handlers::version_handler,
+        crate::api::openweb::openweb_external_handler,
+        crate::api::openweb::openweb_docling_handler,
+    ),
+    components(
+        schemas(
+            crate::api::types::HealthResponse,
+            crate::api::types::PluginStatus,
+            crate::api::types::InfoResponse,
+            crate::api::types::ErrorResponse,
+            crate::api::types::CacheStatsResponse,
+            crate::api::types::CacheClearResponse,
+            crate::api::types::EmbedRequest,
+            crate::api::types::EmbedResponse,
+            crate::api::types::ChunkRequest,
+            crate::api::types::ChunkResponse,
+            crate::api::types::ChunkItem,
+            crate::api::types::ChunkingConfigRequest,
+            crate::api::types::ChunkingConfigResponse,
+            crate::api::types::VersionResponse,
+            crate::api::types::DetectResponse,
+            crate::api::types::ManifestResponse,
+            crate::api::types::ManifestEntryResponse,
+            crate::api::types::WarmRequest,
+            crate::api::types::WarmResponse,
+            crate::core::mime::SupportedFormat,
+            crate::types::extraction::ExtractionResult,
+            crate::types::extraction::Chunk,
+            crate::types::extraction::ChunkMetadata,
+            crate::types::extraction::ExtractedImage,
+            crate::types::extraction::Element,
+            crate::types::extraction::ElementMetadata,
+            crate::types::extraction::ElementId,
+            crate::types::extraction::ElementType,
+            crate::types::extraction::BoundingBox,
+            crate::types::ocr_elements::OcrElement,
+            crate::types::ocr_elements::OcrBoundingGeometry,
+            crate::types::ocr_elements::OcrConfidence,
+            crate::types::ocr_elements::OcrRotation,
+            crate::types::ocr_elements::OcrElementLevel,
+            crate::types::ocr_elements::OcrElementConfig,
+            crate::types::metadata::Metadata,
+            crate::types::tables::Table,
+            crate::types::page::PageContent,
+            crate::types::djot::DjotContent,
+            crate::api::types::OpenWebDocumentResponse,
+            crate::api::types::OpenWebDocumentMetadata,
+            crate::api::types::DoclingCompatResponse,
+            crate::api::types::DoclingCompatDocument,
+            crate::api::types::StructuredExtractionResponse,
+        )
+    ),
+    tags(
+        (name = "health", description = "Health and status endpoints"),
+        (name = "extraction", description = "Document extraction endpoints"),
+        (name = "cache", description = "Cache management endpoints"),
+        (name = "embeddings", description = "Text embedding generation"),
+        (name = "chunking", description = "Text chunking operations"),
+        (name = "openweb", description = "OpenWebUI compatibility endpoints")
+    )
+)]
+#[cfg_attr(alef, alef(skip))]
+pub struct ApiDoc;
+
+/// Generate OpenAPI JSON schema.
+///
+/// Returns the complete OpenAPI 3.1 specification as a JSON string.
+///
+/// # Examples
+///
+/// ```no_run
+/// use kreuzberg::api::openapi::openapi_json;
+///
+/// let schema = openapi_json();
+/// println!("{}", schema);
+/// ```
+#[cfg(feature = "api")]
+#[cfg_attr(alef, alef(skip))]
+pub fn openapi_json() -> String {
+    ApiDoc::openapi().to_pretty_json().unwrap_or_else(|_| "{}".to_string())
+}
+
+#[cfg(not(feature = "api"))]
+#[cfg_attr(alef, alef(skip))]
+pub(crate) fn openapi_json() -> String {
+    r#"{"error": "API feature not enabled"}"#.to_string()
+}
+
+#[cfg(test)]
+mod tests {
+    #[cfg(feature = "api")]
+    use super::*;
+
+    #[test]
+    #[cfg(feature = "api")]
+    fn test_openapi_schema_generation() {
+        let schema = openapi_json();
+        assert!(!schema.is_empty());
+        assert!(schema.contains("Kreuzberg API"));
+        assert!(schema.contains("/health"));
+        assert!(schema.contains("/extract"));
+    }
+
+    #[test]
+    #[cfg(feature = "api")]
+    fn test_openapi_schema_valid_json() {
+        let schema = openapi_json();
+        let parsed: serde_json::Value = serde_json::from_str(&schema).expect("Invalid JSON");
+        assert!(parsed.is_object());
+        assert!(parsed["openapi"].is_string());
+    }
+
+    #[test]
+    #[cfg(feature = "api")]
+    fn test_openapi_includes_all_endpoints() {
+        let schema = openapi_json();
+        // Health endpoints
+        assert!(schema.contains("/health"));
+        assert!(schema.contains("/info"));
+        assert!(schema.contains("/version"));
+        // Extraction
+        assert!(schema.contains("/extract"));
+        assert!(schema.contains("/detect"));
+        // Cache
+        assert!(schema.contains("/cache/stats"));
+        assert!(schema.contains("/cache/clear"));
+        assert!(schema.contains("/cache/manifest"));
+        assert!(schema.contains("/cache/warm"));
+        // Embeddings
+        assert!(schema.contains("/embed"));
+        // Chunking
+        assert!(schema.contains("/chunk"));
+    }
+
+    #[test]
+    #[cfg(feature = "api")]
+    fn test_openapi_includes_schemas() {
+        let schema = openapi_json();
+        assert!(schema.contains("HealthResponse"));
+        assert!(schema.contains("ErrorResponse"));
+        assert!(schema.contains("EmbedRequest"));
+        assert!(schema.contains("ChunkRequest"));
+        assert!(schema.contains("VersionResponse"));
+        assert!(schema.contains("DetectResponse"));
+        assert!(schema.contains("ManifestResponse"));
+        assert!(schema.contains("WarmRequest"));
+        assert!(schema.contains("WarmResponse"));
+    }
+}

crates/kreuzberg/src/api/openweb.rs

@@ -0,0 +1,175 @@
+//! OpenWebUI compatibility handlers.
+//!
+//! Provides endpoints compatible with OpenWebUI's Content Extraction Engine:
+//!
+//! - `PUT /process` — "External" engine: raw binary body, returns `{page_content, metadata}`
+//! - `POST /v1/convert/file` — "Docling" engine: multipart form-data, returns `{document: {md_content}, status}`
+
+use axum::{Json, body::Bytes, extract::State, http::HeaderMap};
+use tower::Service;
+
+use crate::service::ExtractionRequest;
+
+use super::{
+    error::{ApiError, MultipartApi},
+    types::{ApiState, DoclingCompatDocument, DoclingCompatResponse, OpenWebDocumentMetadata, OpenWebDocumentResponse},
+};
+
+/// OpenWebUI "External" engine handler.
+///
+/// PUT /process
+///
+/// Accepts raw binary file content in the request body.
+/// Uses `Content-Type` header for MIME type and `X-Filename` header for the filename.
+///
+/// Returns a JSON document matching OpenWebUI's external document loader contract.
+#[utoipa::path(
+    put,
+    path = "/process",
+    tag = "openweb",
+    request_body(content_type = "application/octet-stream", content = Vec<u8>),
+    responses(
+        (status = 200, description = "Document extracted", body = OpenWebDocumentResponse),
+        (status = 400, description = "Bad request", body = crate::api::types::ErrorResponse),
+        (status = 500, description = "Internal server error", body = crate::api::types::ErrorResponse),
+    )
+)]
+#[cfg_attr(
+    feature = "otel",
+    tracing::instrument(name = "api.openweb_process", skip(state, headers, body))
+)]
+pub(crate) async fn openweb_external_handler(
+    State(state): State<ApiState>,
+    headers: HeaderMap,
+    body: Bytes,
+) -> Result<Json<OpenWebDocumentResponse>, ApiError> {
+    if body.is_empty() {
+        return Err(ApiError::validation(crate::error::KreuzbergError::validation(
+            "Empty request body — upload a file as the raw request body",
+        )));
+    }
+
+    // Extract MIME type from Content-Type header, stripping any parameters (e.g. charset)
+    let mime_type = headers
+        .get(axum::http::header::CONTENT_TYPE)
+        .and_then(|v| v.to_str().ok())
+        .map(|v| v.split(';').next().unwrap_or(v).trim())
+        .unwrap_or(crate::core::mime::OCTET_STREAM_MIME_TYPE)
+        .to_string();
+
+    // Extract filename from X-Filename header (URL-encoded by OpenWebUI)
+    let filename = headers
+        .get("X-Filename")
+        .and_then(|v| v.to_str().ok())
+        .map(|v| urlencoding::decode(v).unwrap_or_else(|_| v.into()).into_owned())
+        .unwrap_or_else(|| "unknown".to_string());
+
+    // Detect MIME type from filename if the header was generic
+    let mime_type = if mime_type == crate::core::mime::OCTET_STREAM_MIME_TYPE {
+        crate::core::mime::detect_mime_type(&filename, false).unwrap_or(mime_type)
+    } else {
+        mime_type
+    };
+
+    // Build extraction config with markdown output
+    let mut config = (*state.default_config).clone();
+    config.output_format = crate::core::config::OutputFormat::Markdown;
+
+    let request = ExtractionRequest::bytes(body.to_vec(), mime_type, config);
+    let mut svc = state
+        .extraction_service
+        .lock()
+        .expect("extraction service lock poisoned")
+        .clone();
+    let result = svc.call(request).await?;
+
+    Ok(Json(OpenWebDocumentResponse {
+        page_content: result.content,
+        metadata: OpenWebDocumentMetadata { source: filename },
+    }))
+}
+
+/// OpenWebUI "Docling" engine handler (docling-serve compatible).
+///
+/// POST /v1/convert/file
+///
+/// Accepts multipart form-data with a `files` field containing the document.
+/// Returns a JSON response matching docling-serve's `/v1/convert/file` contract.
+///
+/// OpenWebUI reads only `document.md_content` from the response.
+#[utoipa::path(
+    post,
+    path = "/v1/convert/file",
+    tag = "openweb",
+    request_body(content_type = "multipart/form-data"),
+    responses(
+        (status = 200, description = "Document converted", body = DoclingCompatResponse),
+        (status = 400, description = "Bad request", body = crate::api::types::ErrorResponse),
+        (status = 500, description = "Internal server error", body = crate::api::types::ErrorResponse),
+    )
+)]
+#[cfg_attr(
+    feature = "otel",
+    tracing::instrument(name = "api.openweb_docling", skip(state, multipart))
+)]
+pub(crate) async fn openweb_docling_handler(
+    State(state): State<ApiState>,
+    MultipartApi(mut multipart): MultipartApi,
+) -> Result<Json<DoclingCompatResponse>, ApiError> {
+    let mut file_data: Option<(Vec<u8>, String)> = None;
+
+    while let Some(field) = multipart
+        .next_field()
+        .await
+        .map_err(|e| ApiError::validation(crate::error::KreuzbergError::validation(e.to_string())))?
+    {
+        let field_name = field.name().unwrap_or("").to_string();
+
+        if field_name == "files" || field_name == "file" {
+            let file_name = field.file_name().map(|s| s.to_string());
+            let content_type = field.content_type().map(|s| s.to_string());
+            let data = field
+                .bytes()
+                .await
+                .map_err(|e| ApiError::validation(crate::error::KreuzbergError::validation(e.to_string())))?;
+
+            let mut mime_type = content_type.unwrap_or_else(|| crate::core::mime::OCTET_STREAM_MIME_TYPE.to_string());
+
+            // Detect from filename if generic
+            if mime_type == crate::core::mime::OCTET_STREAM_MIME_TYPE
+                && let Some(ref name) = file_name
+                && let Ok(detected) = crate::core::mime::detect_mime_type(name, false)
+            {
+                mime_type = detected;
+            }
+
+            file_data = Some((data.to_vec(), mime_type));
+            break;
+        }
+    }
+
+    let (data, mime_type) = file_data.ok_or_else(|| {
+        ApiError::validation(crate::error::KreuzbergError::validation(
+            "No file provided. Upload a file with field name 'files'.",
+        ))
+    })?;
+
+    // Build extraction config with markdown output
+    let mut config = (*state.default_config).clone();
+    config.output_format = crate::core::config::OutputFormat::Markdown;
+
+    let request = ExtractionRequest::bytes(data, mime_type, config);
+    let mut svc = state
+        .extraction_service
+        .lock()
+        .expect("extraction service lock poisoned")
+        .clone();
+    let result = svc.call(request).await?;
+
+    Ok(Json(DoclingCompatResponse {
+        document: DoclingCompatDocument {
+            md_content: result.content,
+        },
+        status: "success".to_string(),
+    }))
+}

crates/kreuzberg/src/api/router.rs

@@ -0,0 +1,276 @@
+//! API router setup and configuration.
+
+use std::sync::Arc;
+
+use axum::{
+    Router,
+    extract::DefaultBodyLimit,
+    routing::{delete, get, post, put},
+};
+use tower_http::{
+    catch_panic::CatchPanicLayer,
+    compression::CompressionLayer,
+    cors::{AllowOrigin, Any, CorsLayer},
+    limit::RequestBodyLimitLayer,
+    request_id::{MakeRequestUuid, PropagateRequestIdLayer, SetRequestIdLayer},
+    sensitive_headers::SetSensitiveHeadersLayer,
+    trace::{DefaultMakeSpan, DefaultOnFailure, DefaultOnRequest, DefaultOnResponse, TraceLayer},
+};
+
+use crate::{ExtractionConfig, core::ServerConfig, service::ExtractionServiceBuilder};
+
+use super::{
+    handlers::{
+        cache_clear_handler, cache_manifest_handler, cache_stats_handler, cache_warm_handler, chunk_handler,
+        detect_handler, embed_handler, extract_async_handler, extract_handler, extract_structured_handler,
+        formats_handler, health_handler, info_handler, job_status_handler, not_found_handler, version_handler,
+    },
+    openweb::{openweb_docling_handler, openweb_external_handler},
+    types::{ApiSizeLimits, ApiState},
+};
+
+/// Create the API router with all routes configured.
+///
+/// This is public to allow users to embed the router in their own applications.
+///
+/// # Arguments
+///
+/// * `config` - Default extraction configuration. Per-request configs override these defaults.
+///
+/// # Examples
+///
+/// ```no_run
+/// use kreuzberg::{ExtractionConfig, api::create_router};
+///
+/// # #[tokio::main]
+/// # async fn main() {
+/// // Create router with default config and size limits
+/// let config = ExtractionConfig::default();
+/// let router = create_router(config);
+/// # }
+/// ```
+#[cfg_attr(alef, alef(skip))]
+pub fn create_router(config: ExtractionConfig) -> Router {
+    create_router_with_limits(config, ApiSizeLimits::default())
+}
+
+/// Create the API router with custom size limits.
+///
+/// This allows fine-grained control over request body and multipart field size limits.
+///
+/// # Arguments
+///
+/// * `config` - Default extraction configuration. Per-request configs override these defaults.
+/// * `limits` - Size limits for request bodies and multipart uploads.
+///
+/// # Examples
+///
+/// ```ignore
+/// use kreuzberg::{ExtractionConfig, api::{create_router_with_limits, ApiSizeLimits}};
+///
+/// # #[tokio::main]
+/// # async fn main() {
+/// // Create router with 50 MB limits
+/// let config = ExtractionConfig::default();
+/// let limits = ApiSizeLimits::from_mb(50, 50);
+/// let router = create_router_with_limits(config, limits);
+/// # }
+/// ```
+///
+/// ```ignore
+/// use kreuzberg::{ExtractionConfig, api::{create_router_with_limits, ApiSizeLimits}};
+/// use tower_http::limit::RequestBodyLimitLayer;
+///
+/// # #[tokio::main]
+/// # async fn main() {
+/// // Custom limits for very large documents (500 MB)
+/// let config = ExtractionConfig::default();
+/// let limits = ApiSizeLimits::from_mb(500, 500);
+/// let router = create_router_with_limits(config, limits);
+/// # }
+/// ```
+#[cfg_attr(alef, alef(skip))]
+pub fn create_router_with_limits(config: ExtractionConfig, limits: ApiSizeLimits) -> Router {
+    create_router_with_limits_and_server_config(config, limits, ServerConfig::default())
+}
+
+/// Create the API router with custom size limits and server configuration.
+///
+/// This function provides full control over request limits, CORS, and server settings via ServerConfig.
+///
+/// # Arguments
+///
+/// * `config` - Default extraction configuration. Per-request configs override these defaults.
+/// * `limits` - Size limits for request bodies and multipart uploads.
+/// * `server_config` - Server configuration including host, port, and CORS settings.
+///
+/// # Examples
+///
+/// ```ignore
+/// use kreuzberg::{ExtractionConfig, api::{create_router_with_limits_and_server_config, ApiSizeLimits}, core::ServerConfig};
+///
+/// # #[tokio::main]
+/// # async fn main() -> kreuzberg::Result<()> {
+/// let extraction_config = ExtractionConfig::default();
+/// let mut server_config = ServerConfig::default();
+/// server_config.cors_origins = vec!["https://example.com".to_string()];
+/// let router = create_router_with_limits_and_server_config(
+///     extraction_config,
+///     ApiSizeLimits::default(),
+///     server_config
+/// );
+/// # Ok(())
+/// # }
+/// ```
+pub(crate) fn create_router_with_limits_and_server_config(
+    config: ExtractionConfig,
+    limits: ApiSizeLimits,
+    server_config: ServerConfig,
+) -> Router {
+    let extraction_service = ExtractionServiceBuilder::new().with_tracing().with_metrics().build();
+
+    let state = ApiState {
+        default_config: Arc::new(config),
+        extraction_service: Arc::new(std::sync::Mutex::new(extraction_service)),
+        #[cfg(feature = "api")]
+        job_store: Arc::new(super::jobs::JobStore::new()),
+    };
+
+    // CORS configuration based on ServerConfig
+    let cors_layer = if server_config.cors_allows_all() {
+        tracing::warn!(
+            "CORS configured to allow all origins (default). This permits CSRF attacks. \
+             For production, set KREUZBERG_CORS_ORIGINS environment variable to comma-separated \
+             list of allowed origins (e.g., 'https://app.example.com,https://api.example.com')"
+        );
+        CorsLayer::new().allow_origin(Any).allow_methods(Any).allow_headers(Any)
+    } else {
+        let origins: Vec<_> = server_config
+            .cors_origins
+            .iter()
+            .filter_map(|s| s.trim().parse::<axum::http::HeaderValue>().ok())
+            .collect();
+
+        if !origins.is_empty() {
+            tracing::info!("CORS configured with {} explicit allowed origin(s)", origins.len());
+            CorsLayer::new()
+                .allow_origin(AllowOrigin::list(origins))
+                .allow_methods(Any)
+                .allow_headers(Any)
+        } else {
+            tracing::warn!(
+                "CORS origins configured but empty/invalid - falling back to permissive CORS. \
+                 This allows CSRF attacks. Set explicit origins for production."
+            );
+            CorsLayer::new().allow_origin(Any).allow_methods(Any).allow_headers(Any)
+        }
+    };
+
+    let mut router = Router::new()
+        .route("/extract", post(extract_handler))
+        .route("/extract-structured", post(extract_structured_handler))
+        .route("/extract-async", post(extract_async_handler))
+        .route("/jobs/{job_id}", get(job_status_handler))
+        .route("/detect", post(detect_handler))
+        .route("/embed", post(embed_handler))
+        .route("/chunk", post(chunk_handler))
+        .route("/formats", get(formats_handler))
+        .route("/health", get(health_handler))
+        .route("/info", get(info_handler))
+        .route("/version", get(version_handler))
+        .route("/cache/stats", get(cache_stats_handler))
+        .route("/cache/clear", delete(cache_clear_handler))
+        .route("/cache/manifest", get(cache_manifest_handler))
+        .route("/cache/warm", post(cache_warm_handler))
+        // OpenWebUI compatibility endpoints
+        .route("/process", put(openweb_external_handler))
+        .route("/v1/convert/file", post(openweb_docling_handler))
+        .fallback(not_found_handler);
+
+    // Add OpenAPI schema endpoint if API feature is enabled
+    #[cfg(feature = "api")]
+    {
+        router = router.route("/openapi.json", get(openapi_schema_handler));
+    }
+
+    router
+        .layer(DefaultBodyLimit::max(limits.max_request_body_bytes))
+        .layer(RequestBodyLimitLayer::new(limits.max_request_body_bytes))
+        .layer(cors_layer)
+        .layer(SetRequestIdLayer::x_request_id(MakeRequestUuid))
+        .layer(PropagateRequestIdLayer::x_request_id())
+        .layer(CompressionLayer::new())
+        .layer(CatchPanicLayer::new())
+        .layer(SetSensitiveHeadersLayer::new([axum::http::header::AUTHORIZATION]))
+        // Per-request and per-response events are demoted to DEBUG so the default
+        // `tower_http=info` filter keeps them out of normal logs. Failures stay at
+        // WARN so they surface without needing RUST_LOG overrides. Full transport
+        // tracing is restored with RUST_LOG=tower_http=debug.
+        .layer(
+            TraceLayer::new_for_http()
+                .make_span_with(DefaultMakeSpan::new().level(tracing::Level::DEBUG))
+                .on_request(DefaultOnRequest::new().level(tracing::Level::DEBUG))
+                .on_response(DefaultOnResponse::new().level(tracing::Level::DEBUG))
+                .on_failure(DefaultOnFailure::new().level(tracing::Level::WARN)),
+        )
+        .with_state(state)
+}
+
+/// OpenAPI schema handler.
+///
+/// Returns the OpenAPI 3.1 JSON schema for all documented endpoints.
+#[cfg(feature = "api")]
+async fn openapi_schema_handler() -> axum::Json<serde_json::Value> {
+    use crate::api::openapi::openapi_json;
+
+    let schema_str = openapi_json();
+    let schema: serde_json::Value = serde_json::from_str(&schema_str)
+        .unwrap_or_else(|_| serde_json::json!({"error": "Failed to generate OpenAPI schema"}));
+
+    axum::Json(schema)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_create_router() {
+        let config = ExtractionConfig::default();
+        let _router = create_router(config);
+    }
+
+    #[test]
+    fn test_router_has_routes() {
+        use std::mem::size_of_val;
+        let config = ExtractionConfig::default();
+        let router = create_router(config);
+        assert!(size_of_val(&router) > 0);
+    }
+
+    #[test]
+    fn test_create_router_with_limits() {
+        let config = ExtractionConfig::default();
+        let limits = ApiSizeLimits::from_mb(50, 50);
+        let _router = create_router_with_limits(config, limits);
+    }
+
+    #[test]
+    fn test_create_router_with_server_config() {
+        let extraction_config = ExtractionConfig::default();
+        let limits = ApiSizeLimits::from_mb(100, 100);
+        let server_config = ServerConfig::default();
+        let _router = create_router_with_limits_and_server_config(extraction_config, limits, server_config);
+    }
+
+    #[test]
+    fn test_server_config_cors_handling() {
+        let extraction_config = ExtractionConfig::default();
+        let limits = ApiSizeLimits::default();
+        let server_config = ServerConfig {
+            cors_origins: vec!["https://example.com".to_string()],
+            ..Default::default()
+        };
+        let _router = create_router_with_limits_and_server_config(extraction_config, limits, server_config);
+    }
+}

crates/kreuzberg/src/api/startup.rs

@@ -0,0 +1,265 @@
+//! API server startup functions.
+
+use std::net::{IpAddr, SocketAddr};
+
+use crate::{
+    ExtractionConfig, Result, core::ServerConfig, extractors, plugins::startup_validation::validate_plugins_at_startup,
+};
+
+use super::{config::load_server_config, router::create_router_with_limits_and_server_config, types::ApiSizeLimits};
+
+/// Start the API server with config file discovery.
+///
+/// Searches for kreuzberg.toml/yaml/json in current and parent directories.
+/// If no config file is found, uses default configuration.
+///
+/// # Arguments
+///
+/// * `host` - IP address to bind to (e.g., "127.0.0.1" or "0.0.0.0")
+/// * `port` - Port number to bind to (e.g., 8000)
+///
+/// # Examples
+///
+/// ```no_run
+/// use kreuzberg::api::serve;
+///
+/// #[tokio::main]
+/// async fn main() -> kreuzberg::Result<()> {
+///     // Local development
+///     serve("127.0.0.1", 8000).await?;
+///     Ok(())
+/// }
+/// ```
+///
+/// ```no_run
+/// use kreuzberg::api::serve;
+///
+/// #[tokio::main]
+/// async fn main() -> kreuzberg::Result<()> {
+///     // Docker/production (listen on all interfaces)
+///     serve("0.0.0.0", 8000).await?;
+///     Ok(())
+/// }
+/// ```
+///
+/// # Environment Variables
+///
+/// ```bash
+/// # Python/Docker usage
+/// export KREUZBERG_HOST=0.0.0.0
+/// export KREUZBERG_PORT=8000
+///
+/// # CORS configuration (IMPORTANT for production security)
+/// # Default: allows all origins (permits CSRF attacks)
+/// # Production: set to comma-separated list of allowed origins
+/// export KREUZBERG_CORS_ORIGINS="https://app.example.com,https://api.example.com"
+///
+/// # Upload size limits (default: 100 MB)
+/// # Modern approach (in bytes):
+/// export KREUZBERG_MAX_REQUEST_BODY_BYTES=104857600       # 100 MB
+/// export KREUZBERG_MAX_MULTIPART_FIELD_BYTES=104857600    # 100 MB per file
+///
+/// python -m kreuzberg.api
+/// ```
+#[cfg_attr(alef, alef(skip))]
+pub async fn serve(host: impl AsRef<str>, port: u16) -> Result<()> {
+    let extraction_config = match ExtractionConfig::discover()? {
+        Some(config) => {
+            tracing::info!("Loaded extraction config from discovered file");
+            config
+        }
+        None => {
+            tracing::info!("No config file found, using default configuration");
+            ExtractionConfig::default()
+        }
+    };
+
+    let server_config = load_server_config(None)?;
+    let limits = ApiSizeLimits::new(
+        server_config.max_request_body_bytes,
+        server_config.max_multipart_field_bytes,
+    );
+
+    // Initialize extractors and validate plugins at startup
+    extractors::ensure_initialized()?;
+    validate_plugins_at_startup()?;
+
+    serve_with_config_and_limits(host, port, extraction_config, limits).await
+}
+
+/// Start the API server with explicit config.
+///
+/// Uses default size limits (100 MB). For custom limits, use `serve_with_config_and_limits`.
+///
+/// # Arguments
+///
+/// * `host` - IP address to bind to (e.g., "127.0.0.1" or "0.0.0.0")
+/// * `port` - Port number to bind to (e.g., 8000)
+/// * `config` - Default extraction configuration for all requests
+///
+/// # Examples
+///
+/// ```no_run
+/// use kreuzberg::{ExtractionConfig, api::serve_with_config};
+///
+/// #[tokio::main]
+/// async fn main() -> kreuzberg::Result<()> {
+///     let config = ExtractionConfig::from_toml_file("config/kreuzberg.toml")?;
+///     serve_with_config("127.0.0.1", 8000, config).await?;
+///     Ok(())
+/// }
+/// ```
+#[cfg_attr(alef, alef(skip))]
+pub async fn serve_with_config(host: impl AsRef<str>, port: u16, config: ExtractionConfig) -> Result<()> {
+    let limits = ApiSizeLimits::default();
+    tracing::info!(
+        "Upload size limit: 100 MB (default, {} bytes)",
+        limits.max_request_body_bytes
+    );
+
+    // Initialize extractors and validate plugins at startup
+    extractors::ensure_initialized()?;
+    validate_plugins_at_startup()?;
+
+    serve_with_config_and_limits(host, port, config, limits).await
+}
+
+/// Start the API server with explicit config and size limits.
+///
+/// # Arguments
+///
+/// * `host` - IP address to bind to (e.g., "127.0.0.1" or "0.0.0.0")
+/// * `port` - Port number to bind to (e.g., 8000)
+/// * `config` - Default extraction configuration for all requests
+/// * `limits` - Size limits for request bodies and multipart uploads
+///
+/// # Examples
+///
+/// ```ignore
+/// use kreuzberg::{ExtractionConfig, api::{serve_with_config_and_limits, ApiSizeLimits}};
+///
+/// #[tokio::main]
+/// async fn main() -> kreuzberg::Result<()> {
+///     let config = ExtractionConfig::from_toml_file("config/kreuzberg.toml")?;
+///     let limits = ApiSizeLimits::from_mb(200, 200);
+///     serve_with_config_and_limits("127.0.0.1", 8000, config, limits).await?;
+///     Ok(())
+/// }
+/// ```
+#[cfg_attr(alef, alef(skip))]
+pub async fn serve_with_config_and_limits(
+    host: impl AsRef<str>,
+    port: u16,
+    config: ExtractionConfig,
+    limits: ApiSizeLimits,
+) -> Result<()> {
+    let ip: IpAddr = host
+        .as_ref()
+        .parse()
+        .map_err(|e| crate::error::KreuzbergError::validation(format!("Invalid host address: {}", e)))?;
+
+    let server_config = ServerConfig {
+        host: host.as_ref().to_string(),
+        port,
+        max_request_body_bytes: limits.max_request_body_bytes,
+        max_multipart_field_bytes: limits.max_multipart_field_bytes,
+        ..Default::default()
+    };
+
+    let addr = SocketAddr::new(ip, port);
+    let app = create_router_with_limits_and_server_config(config, limits, server_config);
+
+    // Initialize extractors and validate plugins at startup
+    extractors::ensure_initialized()?;
+    validate_plugins_at_startup()?;
+
+    tracing::info!("Starting Kreuzberg API server on http://{}:{}", ip, port);
+
+    let listener = tokio::net::TcpListener::bind(addr)
+        .await
+        .map_err(crate::error::KreuzbergError::Io)?;
+
+    axum::serve(listener, app)
+        .await
+        .map_err(|e| crate::error::KreuzbergError::Other(e.to_string()))?;
+
+    Ok(())
+}
+
+/// Start the API server with explicit extraction config and server config.
+///
+/// This function accepts a fully-configured ServerConfig, including CORS origins,
+/// size limits, host, and port. It respects all ServerConfig fields without
+/// re-parsing environment variables, making it ideal for CLI usage where
+/// configuration precedence has already been applied.
+///
+/// # Arguments
+///
+/// * `extraction_config` - Default extraction configuration for all requests
+/// * `server_config` - Server configuration including host, port, CORS, and size limits
+///
+/// # Examples
+///
+/// ```no_run
+/// use kreuzberg::{ExtractionConfig, api::serve_with_server_config, core::ServerConfig};
+///
+/// #[tokio::main]
+/// async fn main() -> kreuzberg::Result<()> {
+///     let extraction_config = ExtractionConfig::default();
+///     let mut server_config = ServerConfig::default();
+///     server_config.host = "0.0.0.0".to_string();
+///     server_config.port = 3000;
+///     server_config.cors_origins = vec!["https://example.com".to_string()];
+///
+///     serve_with_server_config(extraction_config, server_config).await?;
+///     Ok(())
+/// }
+/// ```
+#[cfg_attr(alef, alef(skip))]
+pub async fn serve_with_server_config(extraction_config: ExtractionConfig, server_config: ServerConfig) -> Result<()> {
+    let ip: IpAddr = server_config
+        .host
+        .parse()
+        .map_err(|e| crate::error::KreuzbergError::validation(format!("Invalid host address: {}", e)))?;
+
+    let limits = ApiSizeLimits::new(
+        server_config.max_request_body_bytes,
+        server_config.max_multipart_field_bytes,
+    );
+
+    let addr = SocketAddr::new(ip, server_config.port);
+    let app = create_router_with_limits_and_server_config(extraction_config, limits, server_config.clone());
+
+    // Initialize extractors and validate plugins at startup
+    extractors::ensure_initialized()?;
+    validate_plugins_at_startup()?;
+
+    tracing::info!(
+        "Starting Kreuzberg API server on http://{}:{} (request_body_limit={} MB, multipart_field_limit={} MB)",
+        ip,
+        server_config.port,
+        server_config.max_request_body_mb(),
+        server_config.max_multipart_field_mb()
+    );
+
+    let listener = tokio::net::TcpListener::bind(addr)
+        .await
+        .map_err(crate::error::KreuzbergError::Io)?;
+
+    axum::serve(listener, app)
+        .await
+        .map_err(|e| crate::error::KreuzbergError::Other(e.to_string()))?;
+
+    Ok(())
+}
+
+/// Start the API server with default host and port.
+///
+/// Defaults: host = "127.0.0.1", port = 8000
+///
+/// Uses config file discovery (searches current/parent directories for kreuzberg.toml/yaml/json).
+/// Validates plugins at startup to help diagnose configuration issues.
+#[cfg_attr(alef, alef(skip))]
+pub async fn serve_default() -> Result<()> {
+    serve("127.0.0.1", 8000).await
+}

crates/kreuzberg/src/api/types.rs

@@ -0,0 +1,550 @@
+//! API request and response types.
+
+use std::sync::{Arc, Mutex};
+
+use serde::{Deserialize, Serialize};
+use tower::util::BoxCloneService;
+
+use crate::{ExtractionConfig, KreuzbergError, service::ExtractionRequest, types::ExtractionResult};
+
+/// API server size limit configuration.
+///
+/// Controls maximum sizes for request bodies and multipart uploads.
+/// Default limits are set to 100 MB to accommodate typical document processing workloads.
+///
+/// # Default Values
+///
+/// - `max_request_body_bytes`: 100 MB (104,857,600 bytes)
+/// - `max_multipart_field_bytes`: 100 MB (104,857,600 bytes)
+///
+/// # Configuration via Environment Variables
+///
+/// You can override the defaults using these environment variables:
+///
+/// ```bash
+/// # Modern approach (in bytes):
+/// export KREUZBERG_MAX_REQUEST_BODY_BYTES=104857600     # 100 MB
+/// export KREUZBERG_MAX_MULTIPART_FIELD_BYTES=104857600  # 100 MB
+/// ```
+///
+/// # Examples
+///
+/// ```
+/// use kreuzberg::api::ApiSizeLimits;
+///
+/// // Default limits (100 MB)
+/// let limits = ApiSizeLimits::default();
+///
+/// // Custom limits (5 GB for both)
+/// let limits = ApiSizeLimits {
+///     max_request_body_bytes: 5 * 1024 * 1024 * 1024,
+///     max_multipart_field_bytes: 5 * 1024 * 1024 * 1024,
+/// };
+///
+/// // Very large documents (100 GB total, 50 GB per file)
+/// let limits = ApiSizeLimits {
+///     max_request_body_bytes: 100 * 1024 * 1024 * 1024,
+///     max_multipart_field_bytes: 50 * 1024 * 1024 * 1024,
+/// };
+/// ```
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Copy)]
+pub struct ApiSizeLimits {
+    /// Maximum size of the entire request body in bytes.
+    ///
+    /// This applies to the total size of all uploaded files and form data
+    /// in a single request. Default: 100 MB (104,857,600 bytes).
+    pub max_request_body_bytes: usize,
+
+    /// Maximum size of a single multipart field in bytes.
+    ///
+    /// This applies to individual files in a multipart upload.
+    /// Default: 100 MB (104,857,600 bytes).
+    pub max_multipart_field_bytes: usize,
+}
+
+impl Default for ApiSizeLimits {
+    fn default() -> Self {
+        Self {
+            max_request_body_bytes: 100 * 1024 * 1024,
+            max_multipart_field_bytes: 100 * 1024 * 1024,
+        }
+    }
+}
+
+impl ApiSizeLimits {
+    /// Create new size limits with custom values.
+    ///
+    /// # Arguments
+    ///
+    /// * `max_request_body_bytes` - Maximum total request size in bytes
+    /// * `max_multipart_field_bytes` - Maximum individual file size in bytes
+    pub(crate) fn new(max_request_body_bytes: usize, max_multipart_field_bytes: usize) -> Self {
+        Self {
+            max_request_body_bytes,
+            max_multipart_field_bytes,
+        }
+    }
+
+    /// Create size limits from MB values (convenience method).
+    ///
+    /// # Arguments
+    ///
+    /// * `max_request_body_mb` - Maximum total request size in megabytes
+    /// * `max_multipart_field_mb` - Maximum individual file size in megabytes
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use kreuzberg::api::ApiSizeLimits;
+    ///
+    /// // 50 MB limits
+    /// let limits = ApiSizeLimits::from_mb(50, 50);
+    /// ```
+    #[cfg(test)]
+    pub(crate) fn from_mb(max_request_body_mb: usize, max_multipart_field_mb: usize) -> Self {
+        Self {
+            max_request_body_bytes: max_request_body_mb * 1024 * 1024,
+            max_multipart_field_bytes: max_multipart_field_mb * 1024 * 1024,
+        }
+    }
+}
+
+/// Plugin status information in health response.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct PluginStatus {
+    /// Number of registered OCR backends
+    pub ocr_backends_count: usize,
+    /// Names of registered OCR backends
+    pub ocr_backends: Vec<String>,
+    /// Number of registered document extractors
+    pub extractors_count: usize,
+    /// Number of registered post-processors
+    pub post_processors_count: usize,
+}
+
+/// Health check response.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct HealthResponse {
+    /// Health status
+    #[cfg_attr(feature = "api", schema(example = "healthy"))]
+    pub status: String,
+    /// API version
+    #[cfg_attr(feature = "api", schema(example = "0.8.0"))]
+    pub version: String,
+    /// Plugin status (optional)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub plugins: Option<PluginStatus>,
+}
+
+/// Server information response.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct InfoResponse {
+    /// API version
+    #[cfg_attr(feature = "api", schema(example = "0.8.0"))]
+    pub version: String,
+    /// Whether using Rust backend
+    pub rust_backend: bool,
+}
+
+/// Extraction response (list of results).
+#[cfg_attr(alef, alef(skip))]
+pub type ExtractResponse = Vec<ExtractionResult>;
+
+/// Error response.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct ErrorResponse {
+    /// Error type name
+    #[cfg_attr(feature = "api", schema(example = "ValidationError"))]
+    pub error_type: String,
+    /// Error message
+    #[cfg_attr(feature = "api", schema(example = "Invalid input provided"))]
+    pub message: String,
+    /// Stack trace (if available)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub traceback: Option<String>,
+    /// HTTP status code
+    #[cfg_attr(feature = "api", schema(example = 400))]
+    pub status_code: u16,
+}
+
+/// API server state.
+///
+/// Holds the default extraction configuration loaded from config file
+/// (via discovery or explicit path). Per-request configs override these defaults.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Clone)]
+pub struct ApiState {
+    /// Default extraction configuration
+    pub default_config: Arc<ExtractionConfig>,
+    /// Tower service for extraction requests.
+    ///
+    /// Wrapped in `Arc<Mutex>` because `BoxCloneService` is `Send` but not `Sync`,
+    /// while `ApiState` must be `Clone + Sync` for Axum's state requirement.
+    /// The lock is held only long enough to clone the service.
+    pub extraction_service: Arc<Mutex<BoxCloneService<ExtractionRequest, ExtractionResult, KreuzbergError>>>,
+    /// In-memory job store for async extraction polling.
+    #[cfg(feature = "api")]
+    pub job_store: Arc<super::jobs::JobStore>,
+}
+
+/// Response from `POST /extract-async`: a job identifier the client polls.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct AsyncJobResponse {
+    /// Unique ID to pass to `GET /jobs/{job_id}`.
+    pub job_id: String,
+}
+
+/// The state of an async extraction job.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+#[serde(rename_all = "snake_case")]
+pub enum JobState {
+    /// The job has been accepted but not yet started.
+    Pending,
+    /// The job is currently being processed.
+    Running,
+    /// The job completed successfully.
+    Completed,
+    /// The job terminated with an error.
+    Failed,
+}
+
+/// The status of an async extraction job returned by `GET /jobs/{id}`.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct JobStatus {
+    /// Unique identifier of the job.
+    pub job_id: String,
+    /// Current lifecycle state of the job.
+    pub state: JobState,
+    /// ISO 8601 timestamp when the job was created.
+    pub created_at: String,
+    /// ISO 8601 timestamp of the last state change.
+    pub updated_at: String,
+    /// The extraction result, present only when `state == completed`.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub result: Option<serde_json::Value>,
+    /// Error message, present only when `state == failed`.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error: Option<String>,
+}
+
+/// Response from `GET /jobs/{job_id}`.
+pub type JobStatusResponse = JobStatus;
+
+/// Cache statistics response.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct CacheStatsResponse {
+    /// Cache directory path
+    #[cfg_attr(feature = "api", schema(example = "/tmp/kreuzberg-cache"))]
+    pub directory: String,
+    /// Total number of cache files
+    pub total_files: usize,
+    /// Total cache size in MB
+    pub total_size_mb: f64,
+    /// Available disk space in MB
+    pub available_space_mb: f64,
+    /// Age of oldest file in days
+    pub oldest_file_age_days: f64,
+    /// Age of newest file in days
+    pub newest_file_age_days: f64,
+}
+
+/// Cache clear response.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct CacheClearResponse {
+    /// Cache directory path
+    #[cfg_attr(feature = "api", schema(example = "/tmp/kreuzberg-cache"))]
+    pub directory: String,
+    /// Number of files removed
+    pub removed_files: usize,
+    /// Space freed in MB
+    pub freed_mb: f64,
+}
+
+/// Embedding request for generating embeddings from text.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct EmbedRequest {
+    /// Text strings to generate embeddings for (at least one non-empty string required)
+    #[cfg_attr(feature = "api", schema(min_items = 1))]
+    pub texts: Vec<String>,
+    /// Optional embedding configuration (model, batch size, etc.)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[cfg_attr(feature = "api", schema(value_type = Option<Object>))]
+    pub config: Option<crate::core::config::EmbeddingConfig>,
+}
+
+/// Embedding response containing generated embeddings.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct EmbedResponse {
+    /// Generated embeddings (one per input text)
+    pub embeddings: Vec<Vec<f32>>,
+    /// Model used for embedding generation
+    #[cfg_attr(feature = "api", schema(example = "all-MiniLM-L6-v2"))]
+    pub model: String,
+    /// Dimensionality of the embeddings
+    pub dimensions: usize,
+    /// Number of embeddings generated
+    pub count: usize,
+}
+
+/// Default chunker type.
+fn default_chunker_type() -> String {
+    "text".to_string()
+}
+
+/// Chunk request with text and configuration.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct ChunkRequest {
+    /// Text to chunk (must not be empty)
+    #[cfg_attr(feature = "api", schema(example = "This is sample text to chunk.", min_length = 1))]
+    pub text: String,
+    /// Optional chunking configuration
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub config: Option<ChunkingConfigRequest>,
+    /// Chunker type (text, markdown, yaml, or semantic)
+    #[serde(default = "default_chunker_type")]
+    #[cfg_attr(
+        feature = "api",
+        schema(example = "text", pattern = "^(text|markdown|yaml|semantic)$")
+    )]
+    pub chunker_type: String,
+}
+
+/// Chunking configuration request.
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct ChunkingConfigRequest {
+    /// Maximum characters per chunk (must be greater than overlap, default: 2000)
+    #[cfg_attr(feature = "api", schema(minimum = 101, example = 2000))]
+    pub max_characters: Option<usize>,
+    /// Overlap between chunks in characters (must be less than max_characters, default: 100)
+    #[cfg_attr(feature = "api", schema(minimum = 0, maximum = 1999, example = 100))]
+    pub overlap: Option<usize>,
+    /// Whether to trim whitespace
+    pub trim: Option<bool>,
+    /// Cosine similarity threshold for semantic topic detection (0.0-1.0)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub topic_threshold: Option<f32>,
+}
+
+/// Chunk response with chunks and metadata.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct ChunkResponse {
+    /// List of chunks
+    pub chunks: Vec<ChunkItem>,
+    /// Total number of chunks
+    pub chunk_count: usize,
+    /// Configuration used for chunking
+    pub config: ChunkingConfigResponse,
+    /// Input text size in bytes
+    pub input_size_bytes: usize,
+    /// Chunker type used for chunking
+    #[cfg_attr(feature = "api", schema(example = "text"))]
+    pub chunker_type: String,
+}
+
+/// Individual chunk item with metadata.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct ChunkItem {
+    /// Chunk content
+    pub content: String,
+    /// Byte offset start position
+    pub byte_start: usize,
+    /// Byte offset end position
+    pub byte_end: usize,
+    /// Index of this chunk (0-based)
+    pub chunk_index: usize,
+    /// Total number of chunks
+    pub total_chunks: usize,
+    /// First page number (optional, for PDF chunking)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub first_page: Option<u32>,
+    /// Last page number (optional, for PDF chunking)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub last_page: Option<u32>,
+}
+
+/// Version response.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct VersionResponse {
+    /// Kreuzberg version string
+    #[cfg_attr(feature = "api", schema(example = "0.8.0"))]
+    pub version: String,
+}
+
+/// MIME type detection response.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct DetectResponse {
+    /// Detected MIME type
+    #[cfg_attr(feature = "api", schema(example = "application/pdf"))]
+    pub mime_type: String,
+    /// Original filename (if provided)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub filename: Option<String>,
+}
+
+/// Model manifest entry for cache management.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct ManifestEntryResponse {
+    /// Relative path within the cache directory
+    #[cfg_attr(feature = "api", schema(example = "paddle-ocr/det/model.onnx"))]
+    pub relative_path: String,
+    /// SHA256 checksum of the model file
+    pub sha256: String,
+    /// Expected file size in bytes
+    pub size_bytes: u64,
+    /// HuggingFace source URL for downloading
+    pub source_url: String,
+}
+
+/// Model manifest response.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct ManifestResponse {
+    /// Kreuzberg version
+    #[cfg_attr(feature = "api", schema(example = "0.8.0"))]
+    pub kreuzberg_version: String,
+    /// Total size of all models in bytes
+    pub total_size_bytes: u64,
+    /// Number of models in the manifest
+    pub model_count: usize,
+    /// Individual model entries
+    pub models: Vec<ManifestEntryResponse>,
+}
+
+/// Cache warm request.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct WarmRequest {
+    /// Download all embedding model presets
+    #[serde(default)]
+    pub all_embeddings: bool,
+    /// Specific embedding model preset to download
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub embedding_model: Option<String>,
+}
+
+/// Cache warm response.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct WarmResponse {
+    /// Cache directory used
+    pub cache_dir: String,
+    /// Models that were downloaded
+    pub downloaded: Vec<String>,
+    /// Models that were already cached
+    pub already_cached: Vec<String>,
+}
+
+/// Response from structured extraction endpoint.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct StructuredExtractionResponse {
+    /// Structured data conforming to the provided JSON schema
+    pub structured_output: serde_json::Value,
+    /// Extracted document text content
+    pub content: String,
+    /// Detected MIME type of the input file
+    #[cfg_attr(feature = "api", schema(example = "application/pdf"))]
+    pub mime_type: String,
+}
+
+// ---------------------------------------------------------------------------
+// OpenWebUI compatibility types
+// ---------------------------------------------------------------------------
+
+/// OpenWebUI "External" engine response format.
+///
+/// Returned by `PUT /process` for the OpenWebUI external document loader.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct OpenWebDocumentResponse {
+    /// Extracted text content
+    pub page_content: String,
+    /// Document metadata
+    pub metadata: OpenWebDocumentMetadata,
+}
+
+/// Metadata for the OpenWebUI external document loader response.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct OpenWebDocumentMetadata {
+    /// Original filename
+    #[cfg_attr(feature = "api", schema(example = "document.pdf"))]
+    pub source: String,
+}
+
+/// OpenWebUI "Docling" engine response format.
+///
+/// Returned by `POST /v1/convert/file` for docling-serve compatibility.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct DoclingCompatResponse {
+    /// Converted document content
+    pub document: DoclingCompatDocument,
+    /// Processing status
+    #[cfg_attr(feature = "api", schema(example = "success"))]
+    pub status: String,
+}
+
+/// Document content in the docling-serve response format.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct DoclingCompatDocument {
+    /// Markdown content of the converted document
+    pub md_content: String,
+}
+
+/// Chunking configuration response.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct ChunkingConfigResponse {
+    /// Maximum characters per chunk
+    pub max_characters: usize,
+    /// Overlap between chunks in characters
+    pub overlap: usize,
+    /// Whether whitespace was trimmed
+    pub trim: bool,
+    /// Type of chunker used
+    #[cfg_attr(feature = "api", schema(example = "text"))]
+    pub chunker_type: String,
+    /// Topic threshold used for semantic chunking
+    #[cfg_attr(feature = "api", schema(example = "0.75"))]
+    pub topic_threshold: Option<f32>,
+}

crates/kreuzberg/src/cache/cleanup.rs

@@ -0,0 +1,245 @@
+//! Cache cleanup operations for managing cache size and age.
+
+use crate::error::{KreuzbergError, Result};
+use std::fs;
+use std::path::Path;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use super::core::{CacheEntry, CacheScanResult, CacheStats};
+use super::utilities::get_available_disk_space;
+
+pub(super) fn scan_cache_directory(cache_dir: &str) -> Result<CacheScanResult> {
+    let dir_path = Path::new(cache_dir);
+
+    if !dir_path.exists() {
+        return Ok(CacheScanResult {
+            stats: CacheStats {
+                total_files: 0,
+                total_size_mb: 0.0,
+                available_space_mb: get_available_disk_space(cache_dir)?,
+                oldest_file_age_days: 0.0,
+                newest_file_age_days: 0.0,
+            },
+            entries: Vec::new(),
+        });
+    }
+
+    let current_time = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs() as f64;
+
+    let read_dir =
+        fs::read_dir(dir_path).map_err(|e| KreuzbergError::cache(format!("Failed to read cache directory: {}", e)))?;
+
+    let mut total_size = 0u64;
+    let mut oldest_age = 0.0f64;
+    let mut newest_age = f64::INFINITY;
+    let mut entries = Vec::new();
+
+    for entry in read_dir {
+        let entry = match entry {
+            Ok(e) => e,
+            Err(e) => {
+                tracing::debug!("Error reading cache entry: {}", e);
+                continue;
+            }
+        };
+
+        let metadata = match entry.metadata() {
+            Ok(m) if m.is_file() => m,
+            _ => continue,
+        };
+
+        let path = entry.path();
+        if path.extension().and_then(|s| s.to_str()) != Some("msgpack") {
+            continue;
+        }
+
+        let modified = match metadata.modified() {
+            Ok(m) => m,
+            Err(e) => {
+                tracing::debug!("Error getting modification time for {:?}: {}", path, e);
+                continue;
+            }
+        };
+
+        let size = metadata.len();
+        total_size += size;
+
+        if let Ok(duration) = modified.duration_since(UNIX_EPOCH) {
+            let age_days = (current_time - duration.as_secs() as f64) / (24.0 * 3600.0);
+            oldest_age = oldest_age.max(age_days);
+            newest_age = newest_age.min(age_days);
+        }
+
+        entries.push(CacheEntry { path, size, modified });
+    }
+
+    if entries.is_empty() {
+        oldest_age = 0.0;
+        newest_age = 0.0;
+    }
+
+    Ok(CacheScanResult {
+        stats: CacheStats {
+            total_files: entries.len(),
+            total_size_mb: total_size as f64 / (1024.0 * 1024.0),
+            available_space_mb: get_available_disk_space(cache_dir)?,
+            oldest_file_age_days: oldest_age,
+            newest_file_age_days: newest_age,
+        },
+        entries,
+    })
+}
+
+#[cfg_attr(alef, alef(skip))]
+pub fn get_cache_metadata(cache_dir: &str) -> Result<CacheStats> {
+    let scan_result = scan_cache_directory(cache_dir)?;
+    Ok(scan_result.stats)
+}
+
+pub(crate) fn cleanup_cache(
+    cache_dir: &str,
+    max_age_days: f64,
+    max_size_mb: f64,
+    target_size_ratio: f64,
+) -> Result<(usize, f64)> {
+    let scan_result = scan_cache_directory(cache_dir)?;
+
+    if scan_result.entries.is_empty() {
+        return Ok((0, 0.0));
+    }
+
+    let current_time = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs() as f64;
+    let max_age_seconds = max_age_days * 24.0 * 3600.0;
+
+    let mut removed_count = 0;
+    let mut removed_size = 0.0;
+    let mut remaining_entries = Vec::new();
+    let mut total_remaining_size = 0u64;
+
+    for entry in scan_result.entries {
+        if let Ok(age) = entry.modified.duration_since(UNIX_EPOCH) {
+            let age_seconds = current_time - age.as_secs() as f64;
+            if age_seconds > max_age_seconds {
+                match fs::remove_file(&entry.path) {
+                    Ok(_) => {
+                        removed_count += 1;
+                        removed_size += entry.size as f64 / (1024.0 * 1024.0);
+                    }
+                    Err(e) => {
+                        tracing::debug!("Failed to remove {:?}: {}", entry.path, e);
+                    }
+                }
+            } else {
+                total_remaining_size += entry.size;
+                remaining_entries.push(entry);
+            }
+        }
+    }
+
+    let mut total_size_mb = total_remaining_size as f64 / (1024.0 * 1024.0);
+
+    if total_size_mb > max_size_mb {
+        remaining_entries.sort_by_key(|e| e.modified);
+
+        let target_size = max_size_mb * target_size_ratio;
+
+        for entry in remaining_entries {
+            if total_size_mb <= target_size {
+                break;
+            }
+
+            match fs::remove_file(&entry.path) {
+                Ok(_) => {
+                    let size_mb = entry.size as f64 / (1024.0 * 1024.0);
+                    removed_count += 1;
+                    removed_size += size_mb;
+                    total_size_mb -= size_mb;
+                }
+                Err(e) => {
+                    tracing::debug!("Failed to remove {:?}: {}", entry.path, e);
+                }
+            }
+        }
+    }
+
+    Ok((removed_count, removed_size))
+}
+
+pub(crate) fn smart_cleanup_cache(
+    cache_dir: &str,
+    max_age_days: f64,
+    max_size_mb: f64,
+    min_free_space_mb: f64,
+) -> Result<(usize, f64)> {
+    let stats = get_cache_metadata(cache_dir)?;
+
+    let needs_cleanup = stats.available_space_mb < min_free_space_mb
+        || stats.total_size_mb > max_size_mb
+        || stats.oldest_file_age_days > max_age_days;
+
+    if !needs_cleanup {
+        return Ok((0, 0.0));
+    }
+
+    let target_ratio = if stats.available_space_mb < min_free_space_mb {
+        0.5
+    } else {
+        0.8
+    };
+
+    cleanup_cache(cache_dir, max_age_days, max_size_mb, target_ratio)
+}
+
+#[cfg_attr(alef, alef(skip))]
+pub fn clear_cache_directory(cache_dir: &str) -> Result<(usize, f64)> {
+    let dir_path = Path::new(cache_dir);
+
+    if !dir_path.exists() {
+        return Ok((0, 0.0));
+    }
+
+    let mut removed_count = 0;
+    let mut removed_size = 0.0;
+
+    let read_dir =
+        fs::read_dir(dir_path).map_err(|e| KreuzbergError::cache(format!("Failed to read cache directory: {}", e)))?;
+
+    for entry in read_dir {
+        let entry = match entry {
+            Ok(e) => e,
+            Err(e) => {
+                tracing::debug!("Error reading entry: {}", e);
+                continue;
+            }
+        };
+
+        let metadata = match entry.metadata() {
+            Ok(m) if m.is_file() => m,
+            _ => continue,
+        };
+
+        let path = entry.path();
+        if path.extension().and_then(|s| s.to_str()) != Some("msgpack") {
+            continue;
+        }
+
+        let size_mb = metadata.len() as f64 / (1024.0 * 1024.0);
+        match fs::remove_file(&path) {
+            Ok(_) => {
+                removed_count += 1;
+                removed_size += size_mb;
+            }
+            Err(e) => {
+                tracing::debug!("Failed to remove {:?}: {}", path, e);
+            }
+        }
+    }
+
+    Ok((removed_count, removed_size))
+}

crates/kreuzberg/src/cache/core.rs

@@ -0,0 +1,544 @@
+//! Core cache implementation with GenericCache struct.
+//!
+//! # Lock Strategy
+//!
+//! This module uses `Arc<parking_lot::RwLock<T>>` for thread-safe state management.
+//! Read-heavy operations (membership checks on every `get`/`is_processing` call) acquire
+//! a shared read lock; mutations (insert/remove) acquire an exclusive write lock.
+//!
+//! `parking_lot::RwLock` is preferred over `std::sync::RwLock` because it is not
+//! susceptible to lock poisoning, making the API infallible and avoiding
+//! `KreuzbergError::LockPoisoned` error paths for these fields.
+
+use crate::error::{KreuzbergError, Result};
+use ahash::AHashSet;
+use parking_lot::RwLock;
+use serde::{Deserialize, Serialize};
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::time::SystemTime;
+
+use super::cleanup::smart_cleanup_cache;
+
+/// Minimum seconds between automatic cleanup runs (5 minutes).
+const CLEANUP_INTERVAL_SECS: u64 = 300;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CacheStats {
+    pub total_files: usize,
+    pub total_size_mb: f64,
+    pub available_space_mb: f64,
+    pub oldest_file_age_days: f64,
+    pub newest_file_age_days: f64,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct CacheEntry {
+    pub(super) path: PathBuf,
+    pub(super) size: u64,
+    pub(super) modified: SystemTime,
+}
+
+pub(super) struct CacheScanResult {
+    pub(super) stats: CacheStats,
+    pub(super) entries: Vec<CacheEntry>,
+}
+#[cfg_attr(alef, alef(skip))]
+pub struct GenericCache {
+    cache_dir: PathBuf,
+    #[cfg(test)]
+    cache_type: String,
+    max_age_days: f64,
+    max_cache_size_mb: f64,
+    min_free_space_mb: f64,
+    #[cfg(test)]
+    processing_locks: Arc<RwLock<AHashSet<String>>>,
+    /// Tracks cache keys being deleted to prevent read-during-delete race conditions
+    deleting_files: Arc<RwLock<AHashSet<PathBuf>>>,
+}
+
+impl GenericCache {
+    pub(crate) fn new(
+        cache_type: String,
+        cache_dir: Option<String>,
+        max_age_days: f64,
+        max_cache_size_mb: f64,
+        min_free_space_mb: f64,
+    ) -> Result<Self> {
+        let cache_dir_path = if let Some(dir) = cache_dir {
+            PathBuf::from(dir).join(&cache_type)
+        } else {
+            crate::cache_dir::resolve_cache_dir(&cache_type)
+        };
+
+        fs::create_dir_all(&cache_dir_path)
+            .map_err(|e| KreuzbergError::cache(format!("Failed to create cache directory: {}", e)))?;
+
+        Ok(Self {
+            cache_dir: cache_dir_path,
+            #[cfg(test)]
+            cache_type,
+            max_age_days,
+            max_cache_size_mb,
+            min_free_space_mb,
+            #[cfg(test)]
+            processing_locks: Arc::new(RwLock::new(AHashSet::new())),
+            deleting_files: Arc::new(RwLock::new(AHashSet::new())),
+        })
+    }
+
+    /// Acquire a shared read guard on `processing_locks`.
+    ///
+    /// `parking_lot::RwLock` is infallible (no poisoning), so this never returns an error.
+    #[cfg(test)]
+    fn read_processing_locks(&self) -> parking_lot::RwLockReadGuard<'_, AHashSet<String>> {
+        self.processing_locks.read()
+    }
+
+    /// Acquire an exclusive write guard on `processing_locks`.
+    #[cfg(test)]
+    fn write_processing_locks(&self) -> parking_lot::RwLockWriteGuard<'_, AHashSet<String>> {
+        self.processing_locks.write()
+    }
+
+    /// Acquire a shared read guard on `deleting_files`.
+    fn read_deleting_files(&self) -> parking_lot::RwLockReadGuard<'_, AHashSet<PathBuf>> {
+        self.deleting_files.read()
+    }
+
+    /// Acquire an exclusive write guard on `deleting_files`.
+    #[cfg(test)]
+    fn write_deleting_files(&self) -> parking_lot::RwLockWriteGuard<'_, AHashSet<PathBuf>> {
+        self.deleting_files.write()
+    }
+
+    /// Resolve the directory for a cache key, optionally within a namespace subdirectory.
+    fn resolve_dir(&self, namespace: Option<&str>) -> PathBuf {
+        match namespace {
+            Some(ns) => self.cache_dir.join(ns),
+            None => self.cache_dir.clone(),
+        }
+    }
+
+    fn get_cache_path(&self, cache_key: &str, namespace: Option<&str>) -> PathBuf {
+        self.resolve_dir(namespace).join(format!("{}.msgpack", cache_key))
+    }
+
+    fn get_metadata_path(&self, cache_key: &str, namespace: Option<&str>) -> PathBuf {
+        self.resolve_dir(namespace).join(format!("{}.meta", cache_key))
+    }
+
+    fn is_valid(&self, cache_path: &Path, source_file: Option<&str>, ttl_override_secs: Option<u64>) -> bool {
+        if !cache_path.exists() {
+            return false;
+        }
+
+        if let Ok(metadata) = fs::metadata(cache_path)
+            && let Ok(modified) = metadata.modified()
+            && let Ok(elapsed) = SystemTime::now().duration_since(modified)
+        {
+            // Check TTL from .meta file first, then override, then global max_age_days
+            let max_age_secs = if let Some(ttl) = ttl_override_secs {
+                ttl as f64
+            } else if let Some(meta_ttl) = self.read_meta_ttl(cache_path) {
+                if meta_ttl > 0 {
+                    meta_ttl as f64
+                } else {
+                    self.max_age_days * 86400.0
+                }
+            } else {
+                self.max_age_days * 86400.0
+            };
+
+            if elapsed.as_secs_f64() > max_age_secs {
+                return false;
+            }
+        }
+
+        if let Some(source_path) = source_file {
+            let Some(file_stem) = cache_path.file_stem().and_then(|s| s.to_str()) else {
+                return false;
+            };
+            let namespace = self.infer_namespace(cache_path);
+            let meta_path = self.get_metadata_path(file_stem, namespace.as_deref());
+
+            if meta_path.exists() {
+                if let Ok(cached_meta_bytes) = fs::read(&meta_path)
+                    && cached_meta_bytes.len() >= 16
+                {
+                    // SAFETY: slice is exactly 8 bytes; guaranteed by the `cached_meta_bytes.len() >= 16` check above.
+                    let cached_size = u64::from_le_bytes(cached_meta_bytes[0..8].try_into().unwrap());
+                    // SAFETY: slice is exactly 8 bytes; guaranteed by the `cached_meta_bytes.len() >= 16` check above.
+                    let cached_mtime = u64::from_le_bytes(cached_meta_bytes[8..16].try_into().unwrap());
+
+                    if let Ok(source_metadata) = fs::metadata(source_path) {
+                        let current_size = source_metadata.len();
+                        let Some(current_mtime) = source_metadata
+                            .modified()
+                            .ok()
+                            .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
+                            .map(|d| d.as_secs())
+                        else {
+                            return false;
+                        };
+
+                        return cached_size == current_size && cached_mtime == current_mtime;
+                    }
+                }
+                return false;
+            }
+        }
+
+        true
+    }
+
+    /// Read TTL from .meta file (bytes 16-23). Returns None if not present.
+    fn read_meta_ttl(&self, cache_path: &Path) -> Option<u64> {
+        let file_stem = cache_path.file_stem()?.to_str()?;
+        let namespace = self.infer_namespace(cache_path);
+        let meta_path = self.get_metadata_path(file_stem, namespace.as_deref());
+        let bytes = fs::read(&meta_path).ok()?;
+        if bytes.len() >= 24 {
+            // SAFETY: slice is exactly 8 bytes; guaranteed by the `bytes.len() >= 24` check above.
+            Some(u64::from_le_bytes(bytes[16..24].try_into().unwrap()))
+        } else {
+            None // Old-format 16-byte .meta, no TTL stored
+        }
+    }
+
+    /// Infer namespace from a cache path by checking if it's in a subdirectory.
+    fn infer_namespace(&self, cache_path: &Path) -> Option<String> {
+        let parent = cache_path.parent()?;
+        if parent == self.cache_dir {
+            None
+        } else {
+            parent.file_name()?.to_str().map(|s| s.to_string())
+        }
+    }
+
+    fn save_metadata(
+        &self,
+        cache_key: &str,
+        source_file: Option<&str>,
+        namespace: Option<&str>,
+        ttl_secs: Option<u64>,
+    ) {
+        let meta_path = self.get_metadata_path(cache_key, namespace);
+
+        let mut bytes = Vec::with_capacity(24);
+
+        if let Some(source_path) = source_file
+            && let Ok(metadata) = fs::metadata(source_path)
+        {
+            let size = metadata.len();
+            let mtime = metadata
+                .modified()
+                .ok()
+                .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
+                .map(|d| d.as_secs())
+                .unwrap_or(0);
+
+            bytes.extend_from_slice(&size.to_le_bytes());
+            bytes.extend_from_slice(&mtime.to_le_bytes());
+        } else {
+            bytes.extend_from_slice(&0u64.to_le_bytes());
+            bytes.extend_from_slice(&0u64.to_le_bytes());
+        }
+
+        // TTL in seconds (0 = use global default)
+        bytes.extend_from_slice(&ttl_secs.unwrap_or(0).to_le_bytes());
+
+        let _ = fs::write(meta_path, bytes);
+    }
+
+    #[cfg_attr(feature = "otel", tracing::instrument(
+        skip(self),
+        fields(
+            cache.hit = tracing::field::Empty,
+            cache.key = %cache_key,
+        )
+    ))]
+    pub(crate) fn get(
+        &self,
+        cache_key: &str,
+        source_file: Option<&str>,
+        namespace: Option<&str>,
+        ttl_override_secs: Option<u64>,
+    ) -> Result<Option<Vec<u8>>> {
+        let cache_path = self.get_cache_path(cache_key, namespace);
+
+        {
+            let deleting = self.read_deleting_files();
+            if deleting.contains(&cache_path) {
+                #[cfg(feature = "otel")]
+                tracing::Span::current().record("cache.hit", false);
+                return Ok(None);
+            }
+        }
+
+        if !self.is_valid(&cache_path, source_file, ttl_override_secs) {
+            #[cfg(feature = "otel")]
+            tracing::Span::current().record("cache.hit", false);
+            return Ok(None);
+        }
+
+        match fs::read(&cache_path) {
+            Ok(content) => {
+                #[cfg(feature = "otel")]
+                tracing::Span::current().record("cache.hit", true);
+                Ok(Some(content))
+            }
+            Err(_) => {
+                if let Err(e) = fs::remove_file(&cache_path) {
+                    tracing::debug!("Failed to remove corrupted cache file: {}", e);
+                }
+                let meta_path = self.get_metadata_path(cache_key, namespace);
+                if let Err(e) = fs::remove_file(meta_path) {
+                    tracing::debug!("Failed to remove corrupted metadata file: {}", e);
+                }
+                #[cfg(feature = "otel")]
+                tracing::Span::current().record("cache.hit", false);
+                Ok(None)
+            }
+        }
+    }
+
+    /// Backward-compatible get without namespace/TTL.
+    #[cfg(test)]
+    pub(crate) fn get_default(&self, cache_key: &str, source_file: Option<&str>) -> Result<Option<Vec<u8>>> {
+        self.get(cache_key, source_file, None, None)
+    }
+
+    #[cfg_attr(feature = "otel", tracing::instrument(
+        skip(self, data),
+        fields(
+            cache.key = %cache_key,
+            cache.size_bytes = data.len(),
+        )
+    ))]
+    pub(crate) fn set(
+        &self,
+        cache_key: &str,
+        data: Vec<u8>,
+        source_file: Option<&str>,
+        namespace: Option<&str>,
+        ttl_secs: Option<u64>,
+    ) -> Result<()> {
+        // create_dir_all is idempotent — safe for concurrent multi-worker calls
+        let dir = self.resolve_dir(namespace);
+        fs::create_dir_all(&dir)
+            .map_err(|e| KreuzbergError::cache(format!("Failed to create cache namespace dir: {}", e)))?;
+
+        let cache_path = self.get_cache_path(cache_key, namespace);
+
+        fs::write(&cache_path, &data)
+            .map_err(|e| KreuzbergError::cache(format!("Failed to write cache file: {}", e)))?;
+
+        self.save_metadata(cache_key, source_file, namespace, ttl_secs);
+
+        if self.should_run_cleanup() {
+            if let Some(cache_path_str) = self.cache_dir.to_str() {
+                let _ = smart_cleanup_cache(
+                    cache_path_str,
+                    self.max_age_days,
+                    self.max_cache_size_mb,
+                    self.min_free_space_mb,
+                );
+            }
+            self.touch_cleanup_marker();
+        }
+
+        Ok(())
+    }
+
+    /// Backward-compatible set without namespace/TTL.
+    #[cfg(test)]
+    pub(crate) fn set_default(&self, cache_key: &str, data: Vec<u8>, source_file: Option<&str>) -> Result<()> {
+        self.set(cache_key, data, source_file, None, None)
+    }
+
+    /// Check if cleanup should run based on filesystem marker timestamp.
+    ///
+    /// Multi-worker safe: uses filesystem mtime instead of in-memory counter.
+    fn should_run_cleanup(&self) -> bool {
+        let marker = self.cache_dir.join(".last_cleanup");
+        match fs::metadata(&marker) {
+            Ok(meta) => {
+                if let Ok(modified) = meta.modified() {
+                    let age = SystemTime::now().duration_since(modified).unwrap_or_default();
+                    age.as_secs() > CLEANUP_INTERVAL_SECS
+                } else {
+                    true
+                }
+            }
+            Err(_) => true,
+        }
+    }
+
+    /// Touch the cleanup marker file to record last cleanup time.
+    fn touch_cleanup_marker(&self) {
+        let marker = self.cache_dir.join(".last_cleanup");
+        let _ = fs::write(&marker, []);
+    }
+
+    #[cfg(test)]
+    pub(crate) fn is_processing(&self, cache_key: &str) -> Result<bool> {
+        Ok(self.read_processing_locks().contains(cache_key))
+    }
+
+    #[cfg(test)]
+    pub(crate) fn mark_processing(&self, cache_key: String) -> Result<()> {
+        self.write_processing_locks().insert(cache_key);
+        Ok(())
+    }
+
+    #[cfg(test)]
+    pub(crate) fn mark_complete(&self, cache_key: &str) -> Result<()> {
+        self.write_processing_locks().remove(cache_key);
+        Ok(())
+    }
+
+    #[cfg(test)]
+    fn mark_for_deletion(&self, path: &Path) -> Result<()> {
+        self.write_deleting_files().insert(path.to_path_buf());
+        Ok(())
+    }
+
+    #[cfg(test)]
+    fn unmark_deletion(&self, path: &Path) -> Result<()> {
+        self.write_deleting_files().remove(&path.to_path_buf());
+        Ok(())
+    }
+
+    #[cfg(test)]
+    pub(crate) fn clear(&self) -> Result<(usize, f64)> {
+        let dir_path = &self.cache_dir;
+
+        if !dir_path.exists() {
+            return Ok((0, 0.0));
+        }
+
+        let mut removed_count = 0;
+        let mut removed_size = 0.0;
+
+        let read_dir = fs::read_dir(dir_path)
+            .map_err(|e| KreuzbergError::cache(format!("Failed to read cache directory: {}", e)))?;
+
+        for entry in read_dir {
+            let entry = match entry {
+                Ok(e) => e,
+                Err(e) => {
+                    tracing::debug!("Error reading entry: {}", e);
+                    continue;
+                }
+            };
+
+            let path = entry.path();
+
+            // Skip the cleanup marker file
+            if path.file_name().and_then(|n| n.to_str()) == Some(".last_cleanup") {
+                continue;
+            }
+
+            let metadata = match entry.metadata() {
+                Ok(m) => m,
+                Err(_) => continue,
+            };
+
+            // Recursively clear namespace subdirectories
+            if metadata.is_dir() {
+                let (ns_removed, ns_freed) = self.delete_namespace_inner(&path)?;
+                removed_count += ns_removed;
+                removed_size += ns_freed;
+                continue;
+            }
+
+            if !metadata.is_file() {
+                continue;
+            }
+
+            let ext = path.extension().and_then(|s| s.to_str());
+            if ext != Some("msgpack") && ext != Some("meta") {
+                continue;
+            }
+
+            let size_mb = metadata.len() as f64 / (1024.0 * 1024.0);
+
+            if let Err(e) = self.mark_for_deletion(&path) {
+                tracing::debug!("Failed to mark file for deletion: {} (continuing anyway)", e);
+            }
+
+            match fs::remove_file(&path) {
+                Ok(_) => {
+                    removed_count += 1;
+                    removed_size += size_mb;
+                    if let Err(e) = self.unmark_deletion(&path) {
+                        tracing::debug!("Failed to unmark deleted file: {} (non-critical)", e);
+                    }
+                }
+                Err(e) => {
+                    tracing::debug!("Failed to remove {:?}: {}", path, e);
+                    if let Err(e) = self.unmark_deletion(&path) {
+                        tracing::debug!("Failed to unmark file after deletion error: {} (non-critical)", e);
+                    }
+                }
+            }
+        }
+
+        Ok((removed_count, removed_size))
+    }
+
+    /// Remove a directory and count its contents (used by `clear_all` to handle namespace dirs).
+    #[cfg(test)]
+    fn delete_namespace_inner(&self, dir: &Path) -> Result<(usize, f64)> {
+        if !dir.exists() {
+            return Ok((0, 0.0));
+        }
+
+        let mut removed_count = 0;
+        let mut removed_size = 0.0;
+
+        // Count files before removal
+        if let Ok(read_dir) = fs::read_dir(dir) {
+            for entry in read_dir.flatten() {
+                if let Ok(meta) = entry.metadata()
+                    && meta.is_file()
+                {
+                    removed_size += meta.len() as f64 / (1024.0 * 1024.0);
+                    removed_count += 1;
+                }
+            }
+        }
+
+        fs::remove_dir_all(dir)
+            .map_err(|e| KreuzbergError::cache(format!("Failed to remove directory {}: {}", dir.display(), e)))?;
+
+        Ok((removed_count, removed_size))
+    }
+
+    #[cfg(test)]
+    pub(crate) fn get_stats(&self) -> Result<CacheStats> {
+        self.get_stats_filtered(None)
+    }
+
+    /// Get cache stats, optionally filtered to a specific namespace.
+    #[cfg(test)]
+    pub(crate) fn get_stats_filtered(&self, namespace: Option<&str>) -> Result<CacheStats> {
+        let dir = self.resolve_dir(namespace);
+        let dir_str = dir
+            .to_str()
+            .ok_or_else(|| KreuzbergError::validation("Cache directory path contains invalid UTF-8".to_string()))?;
+        super::cleanup::get_cache_metadata(dir_str)
+    }
+
+    #[cfg(test)]
+    pub(crate) fn cache_dir(&self) -> &Path {
+        &self.cache_dir
+    }
+
+    #[cfg(test)]
+    pub(crate) fn cache_type(&self) -> &str {
+        &self.cache_type
+    }
+}

crates/kreuzberg/src/cache/mod.rs

@@ -0,0 +1,305 @@
+//! Generic cache implementation with lock poisoning recovery.
+//!
+//! This module provides a thread-safe caching system with automatic cleanup,
+//! processing locks, and validation capabilities.
+
+mod cleanup;
+mod core;
+mod utilities;
+
+// Re-export all public types and functions for backward compatibility
+pub use cleanup::{clear_cache_directory, get_cache_metadata};
+pub use core::{CacheStats, GenericCache};
+#[cfg(feature = "ocr")]
+pub(crate) use utilities::blake3_hash_bytes;
+pub(crate) use utilities::blake3_hash_file;
+#[cfg(test)]
+pub(crate) use utilities::{fast_hash, generate_cache_key, validate_cache_key};
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use cleanup::cleanup_cache;
+    use std::fs::File;
+    use tempfile::tempdir;
+
+    #[test]
+    fn test_generate_cache_key_empty() {
+        let result = generate_cache_key(&[]);
+        assert_eq!(result, "empty");
+    }
+
+    #[test]
+    fn test_generate_cache_key_consistent() {
+        let parts = vec![
+            ("key1".to_string(), "value1".to_string()),
+            ("key2".to_string(), "value2".to_string()),
+        ];
+        let key1 = generate_cache_key(&parts);
+        let key2 = generate_cache_key(&parts);
+        assert_eq!(key1, key2);
+        assert_eq!(key1.len(), 32);
+    }
+
+    #[test]
+    fn test_validate_cache_key() {
+        assert!(validate_cache_key("0123456789abcdef0123456789abcdef"));
+        assert!(!validate_cache_key("invalid_key"));
+        assert!(!validate_cache_key("0123456789abcdef"));
+        assert!(!validate_cache_key("0123456789abcdef0123456789abcdef0"));
+    }
+
+    #[test]
+    fn test_fast_hash() {
+        let data1 = b"test data";
+        let data2 = b"test data";
+        let data3 = b"different data";
+
+        assert_eq!(fast_hash(data1), fast_hash(data2));
+        assert_ne!(fast_hash(data1), fast_hash(data3));
+    }
+
+    #[test]
+    fn test_cache_metadata() {
+        let temp_dir = tempdir().unwrap();
+        let cache_dir = temp_dir.path().to_str().unwrap();
+
+        let file1 = temp_dir.path().join("test1.msgpack");
+        let file2 = temp_dir.path().join("test2.msgpack");
+        File::create(&file1).unwrap();
+        File::create(&file2).unwrap();
+
+        let stats = get_cache_metadata(cache_dir).unwrap();
+        assert_eq!(stats.total_files, 2);
+        assert!(stats.available_space_mb > 0.0);
+    }
+
+    #[test]
+    fn test_cleanup_cache() {
+        use std::io::Write;
+
+        let temp_dir = tempdir().unwrap();
+        let cache_dir = temp_dir.path().to_str().unwrap();
+
+        let file1 = temp_dir.path().join("old.msgpack");
+        let mut f = File::create(&file1).unwrap();
+        f.write_all(b"test data for cleanup").unwrap();
+        drop(f);
+
+        let (removed_count, _) = cleanup_cache(cache_dir, 1000.0, 0.000001, 0.8).unwrap();
+        assert_eq!(removed_count, 1);
+        assert!(!file1.exists());
+    }
+
+    #[test]
+    fn test_generic_cache_new() {
+        let temp_dir = tempdir().unwrap();
+        let cache = GenericCache::new(
+            "test".to_string(),
+            Some(temp_dir.path().to_str().unwrap().to_string()),
+            30.0,
+            500.0,
+            1000.0,
+        )
+        .unwrap();
+
+        assert_eq!(cache.cache_type(), "test");
+        assert!(cache.cache_dir().exists());
+    }
+
+    #[test]
+    fn test_generic_cache_get_set() {
+        let temp_dir = tempdir().unwrap();
+        let cache = GenericCache::new(
+            "test".to_string(),
+            Some(temp_dir.path().to_str().unwrap().to_string()),
+            30.0,
+            500.0,
+            1000.0,
+        )
+        .unwrap();
+
+        let cache_key = "test_key";
+        let data = b"test data".to_vec();
+
+        cache.set_default(cache_key, data.clone(), None).unwrap();
+
+        let result = cache.get_default(cache_key, None).unwrap();
+        assert_eq!(result, Some(data));
+    }
+
+    #[test]
+    fn test_generic_cache_get_miss() {
+        let temp_dir = tempdir().unwrap();
+        let cache = GenericCache::new(
+            "test".to_string(),
+            Some(temp_dir.path().to_str().unwrap().to_string()),
+            30.0,
+            500.0,
+            1000.0,
+        )
+        .unwrap();
+
+        let result = cache.get_default("nonexistent", None).unwrap();
+        assert_eq!(result, None);
+    }
+
+    #[test]
+    fn test_generic_cache_source_file_invalidation() {
+        use std::io::Write;
+        use std::thread::sleep;
+        use std::time::Duration;
+
+        let temp_dir = tempdir().unwrap();
+        let cache = GenericCache::new(
+            "test".to_string(),
+            Some(temp_dir.path().to_str().unwrap().to_string()),
+            30.0,
+            500.0,
+            1000.0,
+        )
+        .unwrap();
+
+        let source_file = temp_dir.path().join("source.txt");
+        let mut f = File::create(&source_file).unwrap();
+        f.write_all(b"original content").unwrap();
+        drop(f);
+
+        let cache_key = "test_key";
+        let data = b"cached data".to_vec();
+
+        cache
+            .set_default(cache_key, data.clone(), Some(source_file.to_str().unwrap()))
+            .unwrap();
+
+        let result = cache
+            .get_default(cache_key, Some(source_file.to_str().unwrap()))
+            .unwrap();
+        assert_eq!(result, Some(data.clone()));
+
+        sleep(Duration::from_millis(10));
+        use std::fs;
+        let mut f = fs::OpenOptions::new()
+            .write(true)
+            .truncate(true)
+            .open(&source_file)
+            .unwrap();
+        f.write_all(b"modified content with different size").unwrap();
+        drop(f);
+
+        let result = cache
+            .get_default(cache_key, Some(source_file.to_str().unwrap()))
+            .unwrap();
+        assert_eq!(result, None);
+    }
+
+    #[test]
+    fn test_generic_cache_processing_locks() {
+        let temp_dir = tempdir().unwrap();
+        let cache = GenericCache::new(
+            "test".to_string(),
+            Some(temp_dir.path().to_str().unwrap().to_string()),
+            30.0,
+            500.0,
+            1000.0,
+        )
+        .unwrap();
+
+        let cache_key = "test_key";
+
+        assert!(!cache.is_processing(cache_key).unwrap());
+
+        cache.mark_processing(cache_key.to_string()).unwrap();
+        assert!(cache.is_processing(cache_key).unwrap());
+
+        cache.mark_complete(cache_key).unwrap();
+        assert!(!cache.is_processing(cache_key).unwrap());
+    }
+
+    #[test]
+    fn test_generic_cache_clear() {
+        let temp_dir = tempdir().unwrap();
+        let cache = GenericCache::new(
+            "test".to_string(),
+            Some(temp_dir.path().to_str().unwrap().to_string()),
+            30.0,
+            500.0,
+            1000.0,
+        )
+        .unwrap();
+
+        cache.set_default("key1", b"data1".to_vec(), None).unwrap();
+        cache.set_default("key2", b"data2".to_vec(), None).unwrap();
+
+        let (removed, _freed) = cache.clear().unwrap();
+        assert!(removed >= 2, "Should remove at least 2 cache entries (got {})", removed);
+
+        assert_eq!(cache.get_default("key1", None).unwrap(), None);
+        assert_eq!(cache.get_default("key2", None).unwrap(), None);
+    }
+
+    #[test]
+    fn test_generic_cache_stats() {
+        let temp_dir = tempdir().unwrap();
+        let cache = GenericCache::new(
+            "test".to_string(),
+            Some(temp_dir.path().to_str().unwrap().to_string()),
+            30.0,
+            500.0,
+            1000.0,
+        )
+        .unwrap();
+
+        cache.set_default("key1", b"test data 1".to_vec(), None).unwrap();
+        cache.set_default("key2", b"test data 2".to_vec(), None).unwrap();
+
+        let stats = cache.get_stats().unwrap();
+        assert_eq!(stats.total_files, 2);
+        assert!(stats.total_size_mb > 0.0);
+        assert!(stats.available_space_mb > 0.0);
+    }
+
+    #[test]
+    fn test_generic_cache_expired_entry() {
+        use std::io::Write;
+
+        let temp_dir = tempdir().unwrap();
+        let cache = GenericCache::new(
+            "test".to_string(),
+            Some(temp_dir.path().to_str().unwrap().to_string()),
+            0.000001,
+            500.0,
+            1000.0,
+        )
+        .unwrap();
+
+        let cache_key = "test_key";
+
+        let cache_path = cache.cache_dir().join(format!("{}.msgpack", cache_key));
+        let mut f = File::create(&cache_path).unwrap();
+        f.write_all(b"test data").unwrap();
+        drop(f);
+
+        let old_time = std::time::SystemTime::now() - std::time::Duration::from_secs(60);
+        filetime::set_file_mtime(&cache_path, filetime::FileTime::from_system_time(old_time)).unwrap();
+
+        let result = cache.get_default(cache_key, None).unwrap();
+        assert_eq!(result, None);
+    }
+
+    #[test]
+    fn test_generic_cache_properties() {
+        let temp_dir = tempdir().unwrap();
+        let cache = GenericCache::new(
+            "test".to_string(),
+            Some(temp_dir.path().to_str().unwrap().to_string()),
+            30.0,
+            500.0,
+            1000.0,
+        )
+        .unwrap();
+
+        assert_eq!(cache.cache_type(), "test");
+        assert!(cache.cache_dir().to_string_lossy().contains("test"));
+    }
+}

crates/kreuzberg/src/cache/utilities.rs

@@ -0,0 +1,146 @@
+//! Cache utilities for key generation and disk space management.
+
+use crate::error::Result;
+use std::io::Read;
+use std::path::Path;
+
+#[cfg(unix)]
+use crate::error::KreuzbergError;
+
+/// Cache key hash format width (32 hex digits = 128 bits of blake3 output).
+const CACHE_KEY_HASH_WIDTH: usize = 32;
+
+/// Generate a deterministic cache key from configuration parameters.
+///
+/// # Algorithm
+///
+/// Uses blake3 (cryptographic, SIMD-accelerated) for collision-resistant cache keys.
+/// Cache keys are generated by:
+/// 1. Sorting key-value pairs by key (for determinism)
+/// 2. Concatenating as "key1=val1&key2=val2&..."
+/// 3. Hashing with blake3 and formatting as 32-character hex (first 128 bits)
+///
+/// # Example
+///
+/// ```rust
+/// use kreuzberg::cache::generate_cache_key;
+///
+/// let parts = vec![
+///     ("format".to_string(), "pdf".to_string()),
+///     ("ocr".to_string(), "true".to_string()),
+///     ("lang".to_string(), "en".to_string()),
+/// ];
+/// let key = generate_cache_key(&parts);
+/// assert_eq!(key.len(), 32);
+/// ```
+#[cfg(test)]
+pub(crate) fn generate_cache_key(parts: &[(String, String)]) -> String {
+    if parts.is_empty() {
+        return "empty".to_string();
+    }
+
+    let mut sorted_parts: Vec<(&str, &str)> = parts.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect();
+    sorted_parts.sort_by_key(|(k, _)| *k);
+
+    let estimated_size = sorted_parts.iter().map(|(k, v)| k.len() + v.len() + 2).sum::<usize>();
+    let mut cache_str = String::with_capacity(estimated_size);
+
+    for (i, (key, val)) in sorted_parts.iter().enumerate() {
+        if i > 0 {
+            cache_str.push('&');
+        }
+        cache_str.push_str(&format!("{}={}", key, val));
+    }
+
+    blake3_hash_to_hex(cache_str.as_bytes())
+}
+
+/// Hash arbitrary bytes with blake3, returning a 32-char hex string.
+#[cfg(feature = "ocr")]
+pub(crate) fn blake3_hash_bytes(data: &[u8]) -> String {
+    blake3_hash_to_hex(data)
+}
+
+/// Hash a file's content with blake3 using streaming 64 KiB reads.
+///
+/// Returns a 32-char hex string (128 bits of blake3 output).
+pub(crate) fn blake3_hash_file(path: &Path) -> Result<String> {
+    let file = std::fs::File::open(path)
+        .map_err(|e| crate::error::KreuzbergError::cache(format!("Failed to open file for hashing: {e}")))?;
+    let mut reader = std::io::BufReader::with_capacity(64 * 1024, file);
+    let mut hasher = blake3::Hasher::new();
+
+    let mut buf = [0u8; 64 * 1024];
+    loop {
+        let n = reader
+            .read(&mut buf)
+            .map_err(|e| crate::error::KreuzbergError::cache(format!("Failed to read file for hashing: {e}")))?;
+        if n == 0 {
+            break;
+        }
+        hasher.update(&buf[..n]);
+    }
+
+    let hash = hasher.finalize();
+    Ok(hex::encode(&hash.as_bytes()[..CACHE_KEY_HASH_WIDTH / 2]))
+}
+
+/// Hash bytes with blake3 and return first 128 bits as 32 hex chars.
+#[cfg(any(test, feature = "ocr"))]
+fn blake3_hash_to_hex(data: &[u8]) -> String {
+    let hash = blake3::hash(data);
+    hex::encode(&hash.as_bytes()[..CACHE_KEY_HASH_WIDTH / 2])
+}
+
+#[allow(unsafe_code)]
+pub(crate) fn get_available_disk_space(path: &str) -> Result<f64> {
+    #[cfg(unix)]
+    {
+        let path = Path::new(path);
+        let check_path = if path.exists() {
+            path
+        } else if let Some(parent) = path.parent() {
+            parent
+        } else {
+            Path::new("/")
+        };
+
+        use libc::{statvfs, statvfs as statvfs_struct};
+        use std::ffi::CString;
+
+        let path_str = check_path
+            .to_str()
+            .ok_or_else(|| KreuzbergError::validation("Path contains invalid UTF-8".to_string()))?;
+        let c_path = CString::new(path_str).map_err(|e| KreuzbergError::validation(format!("Invalid path: {}", e)))?;
+
+        let mut stat: statvfs_struct = unsafe { std::mem::zeroed() };
+
+        let result = unsafe { statvfs(c_path.as_ptr(), &mut stat) };
+
+        if result == 0 {
+            #[allow(clippy::unnecessary_cast)]
+            let available_bytes = stat.f_bavail as u64 * stat.f_frsize as u64;
+            Ok(available_bytes as f64 / (1024.0 * 1024.0))
+        } else {
+            tracing::debug!("Failed to get disk stats for {}: errno {}", path_str, result);
+            Ok(10000.0)
+        }
+    }
+
+    #[cfg(not(unix))]
+    {
+        let _ = path;
+        Ok(10000.0)
+    }
+}
+
+#[cfg(test)]
+pub(crate) fn fast_hash(data: &[u8]) -> u64 {
+    let hash = blake3::hash(data);
+    u64::from_le_bytes(hash.as_bytes()[..8].try_into().unwrap())
+}
+
+#[cfg(test)]
+pub(crate) fn validate_cache_key(key: &str) -> bool {
+    key.len() == 32 && key.chars().all(|c| c.is_ascii_hexdigit())
+}

crates/kreuzberg/src/cache_dir.rs

@@ -0,0 +1,54 @@
+//! Centralized cache directory resolution for all kreuzberg modules.
+//!
+//! Provides a single function that all modules use to determine where to store
+//! cached data (models, OCR results, tessdata, etc.). This avoids per-CWD
+//! `.kreuzberg/` directories and uses platform-appropriate global cache locations.
+
+use std::path::PathBuf;
+
+/// Resolve the kreuzberg cache base directory (without a module suffix).
+///
+/// Uses the same resolution order as [`resolve_cache_dir`] but returns
+/// the top-level kreuzberg cache directory.
+#[allow(dead_code)]
+pub(crate) fn resolve_cache_base() -> PathBuf {
+    if let Ok(env_path) = std::env::var("KREUZBERG_CACHE_DIR") {
+        return PathBuf::from(env_path);
+    }
+    if let Some(cache) = dirs::cache_dir() {
+        return cache.join("kreuzberg");
+    }
+    if let Some(home) = dirs::home_dir() {
+        return home.join(".cache").join("kreuzberg");
+    }
+    std::env::current_dir()
+        .unwrap_or_else(|_| PathBuf::from("."))
+        .join(".kreuzberg")
+}
+
+/// Resolve the kreuzberg cache directory for a given module.
+///
+/// Resolution order:
+/// 1. `KREUZBERG_CACHE_DIR` env var + `/{module}` (explicit override)
+/// 2. Platform-appropriate global cache directory:
+///    - macOS: `~/Library/Caches/kreuzberg/{module}`
+///    - Linux: `$XDG_CACHE_HOME/kreuzberg/{module}` or `~/.cache/kreuzberg/{module}`
+///    - Windows: `%LOCALAPPDATA%/kreuzberg/{module}`
+/// 3. Home directory fallback: `~/.cache/kreuzberg/{module}`
+/// 4. CWD-relative fallback: `.kreuzberg/{module}` (last resort, e.g. no HOME set)
+#[cfg_attr(alef, alef(skip))]
+pub(crate) fn resolve_cache_dir(module: &str) -> PathBuf {
+    if let Ok(env_path) = std::env::var("KREUZBERG_CACHE_DIR") {
+        return PathBuf::from(env_path).join(module);
+    }
+    if let Some(cache) = dirs::cache_dir() {
+        return cache.join("kreuzberg").join(module);
+    }
+    if let Some(home) = dirs::home_dir() {
+        return home.join(".cache").join("kreuzberg").join(module);
+    }
+    std::env::current_dir()
+        .unwrap_or_else(|_| PathBuf::from("."))
+        .join(".kreuzberg")
+        .join(module)
+}

crates/kreuzberg/src/cancellation.rs

@@ -0,0 +1,132 @@
+//! Cancellation token for extraction operations.
+//!
+//! Provides a lightweight, FFI-friendly cancellation primitive based on
+//! `Arc<AtomicBool>`. The token can be cloned and shared across threads;
+//! any holder can cancel the operation and all other holders will observe
+//! the cancellation on their next check.
+//!
+//! # Design
+//!
+//! - `Arc<AtomicBool>` is used rather than `tokio_util::CancellationToken` so
+//!   the type has no Tokio dependency at the type level and is usable from both
+//!   sync and async contexts.
+//! - `Ordering::Relaxed` is sufficient: we only need eventual visibility, not
+//!   happens-before ordering relative to other memory accesses.
+//! - The token wraps an `Arc<AtomicBool>` and can be stored in
+//!   `ExtractionConfig` without layout surprises.
+//!
+//! # FFI
+//!
+//! The FFI crate wraps this type in an opaque `*mut CancellationToken` handle
+//! (see `crates/kreuzberg-ffi/src/cancellation.rs`).
+
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+/// A lightweight, cloneable cancellation token.
+///
+/// Create one with [`CancellationToken::new`], pass clones to the extraction
+/// call (via [`ExtractionConfig::cancel_token`]) and to the caller. Call
+/// [`CancellationToken::cancel`] from the caller side when the operation
+/// should be aborted. The extraction code polls
+/// [`CancellationToken::is_cancelled`] at safe checkpoints and returns
+/// [`KreuzbergError::Cancelled`] if set.
+///
+/// Cloning is cheap (increments the `Arc` reference count only).
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Default)]
+pub struct CancellationToken {
+    cancelled: Arc<AtomicBool>,
+}
+
+impl CancellationToken {
+    /// Create a new, uncancelled token.
+    #[cfg(test)]
+    pub(crate) fn new() -> Self {
+        Self::default()
+    }
+
+    /// Signal cancellation.
+    ///
+    /// All clones of this token will observe [`is_cancelled`] returning `true`
+    /// on their next check. This operation is idempotent.
+    #[cfg(any(feature = "tokio-runtime", test))]
+    #[inline]
+    pub(crate) fn cancel(&self) {
+        self.cancelled.store(true, Ordering::Relaxed);
+    }
+
+    /// Returns `true` if [`cancel`] has been called on any clone of this token.
+    #[inline]
+    pub(crate) fn is_cancelled(&self) -> bool {
+        self.cancelled.load(Ordering::Relaxed)
+    }
+}
+
+impl Serialize for CancellationToken {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        // Serialize the current cancellation state.
+        // Note: This is a snapshot at serialization time; deserialized tokens
+        // are independent of the original token's future state.
+        let state = self.is_cancelled();
+        state.serialize(serializer)
+    }
+}
+
+impl<'de> Deserialize<'de> for CancellationToken {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        // Deserialize the cancellation state into a new token.
+        let cancelled = bool::deserialize(deserializer)?;
+        Ok(CancellationToken {
+            cancelled: Arc::new(AtomicBool::new(cancelled)),
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_new_token_is_not_cancelled() {
+        let token = CancellationToken::new();
+        assert!(!token.is_cancelled());
+    }
+
+    #[test]
+    fn test_cancel_sets_flag() {
+        let token = CancellationToken::new();
+        token.cancel();
+        assert!(token.is_cancelled());
+    }
+
+    #[test]
+    fn test_clone_shares_state() {
+        let token = CancellationToken::new();
+        let clone = token.clone();
+        assert!(!clone.is_cancelled());
+        token.cancel();
+        assert!(clone.is_cancelled());
+    }
+
+    #[test]
+    fn test_cancel_is_idempotent() {
+        let token = CancellationToken::new();
+        token.cancel();
+        token.cancel();
+        assert!(token.is_cancelled());
+    }
+
+    #[test]
+    fn test_default_is_not_cancelled() {
+        let token = CancellationToken::default();
+        assert!(!token.is_cancelled());
+    }
+}

crates/kreuzberg/src/chunking/boundaries.rs

@@ -0,0 +1,322 @@
+//! Page boundary handling and page range calculation for chunked text.
+//!
+//! This module provides functions to track which pages text chunks span,
+//! enabling accurate page-level metadata for document processing.
+
+use crate::error::{KreuzbergError, Result};
+use crate::types::PageBoundary;
+
+/// Validates the consistency and correctness of page boundaries.
+///
+/// # Validation Rules
+///
+/// 1. Boundaries must be sorted by byte_start (monotonically increasing)
+/// 2. Boundaries must not overlap (byte_end[i] <= byte_start[i+1])
+/// 3. Each boundary must have byte_start < byte_end
+///
+/// # Arguments
+///
+/// * `boundaries` - Page boundary markers to validate
+///
+/// # Returns
+///
+/// Returns `Ok(())` if all boundaries are valid.
+/// Returns `KreuzbergError::Validation` if any boundary is invalid.
+pub(crate) fn validate_page_boundaries(boundaries: &[PageBoundary]) -> Result<()> {
+    if boundaries.is_empty() {
+        return Ok(());
+    }
+
+    for (idx, boundary) in boundaries.iter().enumerate() {
+        if boundary.byte_start > boundary.byte_end {
+            return Err(KreuzbergError::validation(format!(
+                "Invalid boundary range at index {}: byte_start ({}) must be <= byte_end ({})",
+                idx, boundary.byte_start, boundary.byte_end
+            )));
+        }
+    }
+
+    for i in 0..boundaries.len() - 1 {
+        let current = &boundaries[i];
+        let next = &boundaries[i + 1];
+
+        if current.byte_start > next.byte_start {
+            return Err(KreuzbergError::validation(format!(
+                "Page boundaries not sorted: boundary at index {} (byte_start={}) comes after boundary at index {} (byte_start={})",
+                i,
+                current.byte_start,
+                i + 1,
+                next.byte_start
+            )));
+        }
+
+        if current.byte_end > next.byte_start {
+            return Err(KreuzbergError::validation(format!(
+                "Overlapping page boundaries: boundary {} ends at {} but boundary {} starts at {}",
+                i,
+                current.byte_end,
+                i + 1,
+                next.byte_start
+            )));
+        }
+    }
+
+    Ok(())
+}
+
+/// Calculate which pages a byte range spans.
+///
+/// # Arguments
+///
+/// * `byte_start` - Starting byte offset of the chunk
+/// * `byte_end` - Ending byte offset of the chunk
+/// * `boundaries` - Page boundary markers from the document
+///
+/// # Returns
+///
+/// A tuple of (first_page, last_page) where page numbers are 1-indexed.
+/// Returns (None, None) if boundaries are empty or chunk doesn't overlap any page.
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Validation` if boundaries are invalid.
+///
+/// # Examples
+///
+/// ```rust,ignore
+/// use kreuzberg::chunking::boundaries::calculate_page_range;
+/// use kreuzberg::types::PageBoundary;
+///
+/// let boundaries = vec![
+///     PageBoundary { byte_start: 0, byte_end: 100, page_number: 1 },
+///     PageBoundary { byte_start: 100, byte_end: 200, page_number: 2 },
+/// ];
+///
+/// let (first, last) = calculate_page_range(50, 150, &boundaries)?;
+/// assert_eq!(first, Some(1));
+/// assert_eq!(last, Some(2));
+/// # Ok::<(), kreuzberg::Result<()>>(())
+/// ```
+pub(crate) fn calculate_page_range(
+    byte_start: usize,
+    byte_end: usize,
+    boundaries: &[PageBoundary],
+) -> Result<(Option<u32>, Option<u32>)> {
+    if boundaries.is_empty() {
+        return Ok((None, None));
+    }
+
+    validate_page_boundaries(boundaries)?;
+
+    let mut first_page = None;
+    let mut last_page = None;
+
+    for boundary in boundaries {
+        if byte_start < boundary.byte_end && byte_end > boundary.byte_start {
+            if first_page.is_none() {
+                first_page = Some(boundary.page_number);
+            }
+            last_page = Some(boundary.page_number);
+        }
+    }
+
+    Ok((first_page, last_page))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_validate_page_boundaries_valid() {
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 20,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 20,
+                byte_end: 40,
+                page_number: 2,
+            },
+            PageBoundary {
+                byte_start: 40,
+                byte_end: 60,
+                page_number: 3,
+            },
+        ];
+
+        let result = validate_page_boundaries(&boundaries);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_validate_page_boundaries_empty() {
+        let boundaries: Vec<PageBoundary> = vec![];
+        let result = validate_page_boundaries(&boundaries);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_calculate_page_range_within_page() {
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 100,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 100,
+                byte_end: 200,
+                page_number: 2,
+            },
+        ];
+
+        let (first, last) = calculate_page_range(10, 50, &boundaries).unwrap();
+        assert_eq!(first, Some(1));
+        assert_eq!(last, Some(1));
+    }
+
+    #[test]
+    fn test_calculate_page_range_spanning_pages() {
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 100,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 100,
+                byte_end: 200,
+                page_number: 2,
+            },
+        ];
+
+        let (first, last) = calculate_page_range(50, 150, &boundaries).unwrap();
+        assert_eq!(first, Some(1));
+        assert_eq!(last, Some(2));
+    }
+
+    #[test]
+    fn test_calculate_page_range_empty_boundaries() {
+        let boundaries: Vec<PageBoundary> = vec![];
+
+        let (first, last) = calculate_page_range(0, 50, &boundaries).unwrap();
+        assert_eq!(first, None);
+        assert_eq!(last, None);
+    }
+
+    #[test]
+    fn test_calculate_page_range_no_overlap() {
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 100,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 100,
+                byte_end: 200,
+                page_number: 2,
+            },
+        ];
+
+        let (first, last) = calculate_page_range(200, 250, &boundaries).unwrap();
+        assert_eq!(first, None);
+        assert_eq!(last, None);
+    }
+
+    #[test]
+    fn test_calculate_page_range_three_pages() {
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 100,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 100,
+                byte_end: 200,
+                page_number: 2,
+            },
+            PageBoundary {
+                byte_start: 200,
+                byte_end: 300,
+                page_number: 3,
+            },
+        ];
+
+        let (first, last) = calculate_page_range(50, 250, &boundaries).unwrap();
+        assert_eq!(first, Some(1));
+        assert_eq!(last, Some(3));
+    }
+
+    #[test]
+    fn test_calculate_page_range_with_invalid_boundaries() {
+        let boundaries = vec![PageBoundary {
+            byte_start: 15,
+            byte_end: 10,
+            page_number: 1,
+        }];
+
+        let result = calculate_page_range(0, 20, &boundaries);
+        assert!(result.is_err());
+        let err = result.unwrap_err();
+        assert!(err.to_string().contains("Invalid boundary range"));
+    }
+
+    #[test]
+    fn test_page_boundaries_with_gaps() {
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 10,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 15,
+                byte_end: 25,
+                page_number: 2,
+            },
+        ];
+
+        let result = validate_page_boundaries(&boundaries);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_chunk_with_same_start_and_end() {
+        // Zero-length boundaries are valid (empty pages)
+        let boundaries = vec![PageBoundary {
+            byte_start: 10,
+            byte_end: 10,
+            page_number: 1,
+        }];
+
+        let result = validate_page_boundaries(&boundaries);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_zero_length_boundary_is_valid() {
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 100,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 100,
+                byte_end: 100,
+                page_number: 2,
+            }, // empty page
+            PageBoundary {
+                byte_start: 100,
+                byte_end: 200,
+                page_number: 3,
+            },
+        ];
+        assert!(validate_page_boundaries(&boundaries).is_ok());
+    }
+}

crates/kreuzberg/src/chunking/boundary_detection.rs

@@ -0,0 +1,499 @@
+//! Structural boundary detection for plain text.
+//!
+//! Detects header-like patterns in plain text (PDF/DOCX extracted prose)
+//! that `text-splitter` cannot identify. Used by the semantic chunker to
+//! force topic boundaries at section headers.
+
+use serde::{Deserialize, Serialize};
+
+/// Maximum line length for ALL CAPS header detection.
+const MAX_ALL_CAPS_LINE_LEN: usize = 80;
+
+/// Maximum line length for standalone title detection.
+const MAX_TITLE_LINE_LEN: usize = 60;
+
+/// Maximum number of words in a standalone title.
+const MAX_TITLE_WORD_COUNT: usize = 8;
+
+/// Minimum fraction of words that must start with uppercase for title detection.
+/// Expressed as numerator/denominator (3/5 = 60%).
+const TITLE_UPPERCASE_RATIO: (usize, usize) = (3, 5);
+
+/// A detected structural boundary in the text.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DetectedBoundary {
+    /// Byte offset of the start of the line in the original text.
+    pub byte_offset: usize,
+    /// Whether this boundary looks like a header/section title.
+    pub is_header: bool,
+}
+
+/// Detect structural boundaries in plain text.
+///
+/// Iterates lines and checks each against heuristics for ALL-CAPS headers,
+/// numbered sections, and title lines. Returns boundaries sorted by byte offset.
+pub(crate) fn detect_plain_text_boundaries(text: &str) -> Vec<DetectedBoundary> {
+    let mut boundaries = Vec::new();
+    let mut byte_offset: usize = 0;
+
+    for line in text.split('\n') {
+        let trimmed = line.trim();
+        if !trimmed.is_empty() && (is_all_caps_header(trimmed) || is_numbered_section(line) || is_title_line(trimmed)) {
+            boundaries.push(DetectedBoundary {
+                byte_offset,
+                is_header: true,
+            });
+        }
+        byte_offset += line.len() + 1; // +1 for the '\n' delimiter
+    }
+
+    boundaries
+}
+
+/// Check if a line is an ALL-CAPS header.
+///
+/// True when the line is < 80 chars, has >= 2 alphabetic chars, and every
+/// alphabetic char is uppercase.
+fn is_all_caps_header(line: &str) -> bool {
+    if line.len() >= MAX_ALL_CAPS_LINE_LEN {
+        return false;
+    }
+
+    let mut alpha_count = 0;
+    for ch in line.chars() {
+        if ch.is_alphabetic() {
+            if !ch.is_uppercase() {
+                return false;
+            }
+            alpha_count += 1;
+        }
+    }
+
+    alpha_count >= 2
+}
+
+/// Check if a line is a numbered section header.
+///
+/// Matches patterns like "1.", "1.2", "1.2.3" at the start of the line
+/// (not indented), followed by a space or end of line. Must contain at
+/// least one dot.
+fn is_numbered_section(line: &str) -> bool {
+    // Must not be indented
+    if line.starts_with(' ') || line.starts_with('\t') {
+        return false;
+    }
+
+    let trimmed = line.trim();
+    if trimmed.is_empty() {
+        return false;
+    }
+
+    // Find the numbering prefix: digits and dots
+    let prefix_end = trimmed
+        .find(|ch: char| !ch.is_ascii_digit() && ch != '.')
+        .unwrap_or(trimmed.len());
+
+    let prefix = &trimmed[..prefix_end];
+
+    // Must have at least one dot
+    if !prefix.contains('.') {
+        return false;
+    }
+
+    // Must start with a digit
+    if !prefix.starts_with(|ch: char| ch.is_ascii_digit()) {
+        return false;
+    }
+
+    // After the numbering prefix, must be followed by space + text.
+    // Bare prefixes like "1." or "1.2." alone are not section headers.
+    if prefix_end >= trimmed.len() {
+        return false;
+    }
+    if !trimmed[prefix_end..].starts_with(' ') {
+        return false;
+    }
+    // Must have non-whitespace text after the prefix + space.
+    if trimmed[prefix_end..].trim().is_empty() {
+        return false;
+    }
+
+    // No consecutive dots (reject malformed patterns like "1..2").
+    if prefix.contains("..") {
+        return false;
+    }
+
+    true
+}
+
+/// Check if a line looks like a standalone title.
+///
+/// Criteria: < 60 chars, starts with uppercase letter, title-case words
+/// (most words start with uppercase), no more than 8 words, no trailing
+/// sentence punctuation (. ! ? ; :), not indented.
+fn is_title_line(line: &str) -> bool {
+    if line.len() >= MAX_TITLE_LINE_LEN {
+        return false;
+    }
+
+    // Must start with uppercase alphabetic char (not indented).
+    let first = match line.chars().next() {
+        Some(ch) => ch,
+        None => return false,
+    };
+    if !first.is_uppercase() {
+        return false;
+    }
+
+    let words: Vec<&str> = line.split_whitespace().collect();
+
+    // Must have at least 2 words and no more than 8 (titles are short).
+    // Single words like "Note", "I", "To" are not titles.
+    if words.len() < 2 || words.len() > MAX_TITLE_WORD_COUNT {
+        return false;
+    }
+
+    // Most words must start with an uppercase letter (title-case).
+    let upper_start_count = words
+        .iter()
+        .filter(|w| w.chars().next().is_some_and(|c| c.is_uppercase()))
+        .count();
+    // Allow small connectors (of, and, the, etc.) — require title-case majority.
+    let (num, den) = TITLE_UPPERCASE_RATIO;
+    if upper_start_count * den < words.len() * num {
+        return false;
+    }
+
+    // Must not end with sentence punctuation.
+    if line.ends_with(['.', '!', '?', ';', ':']) {
+        return false;
+    }
+
+    true
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // --- is_all_caps_header ---
+
+    #[test]
+    fn all_caps_introduction() {
+        assert!(is_all_caps_header("INTRODUCTION"));
+    }
+
+    #[test]
+    fn all_caps_chapter_one() {
+        assert!(is_all_caps_header("CHAPTER ONE"));
+    }
+
+    #[test]
+    fn all_caps_terms_and_conditions() {
+        assert!(is_all_caps_header("TERMS AND CONDITIONS"));
+    }
+
+    #[test]
+    fn not_all_caps_mixed_case() {
+        assert!(!is_all_caps_header("Introduction"));
+    }
+
+    #[test]
+    fn not_all_caps_single_char() {
+        assert!(!is_all_caps_header("a"));
+    }
+
+    #[test]
+    fn not_all_caps_too_long() {
+        let long = "A".repeat(100);
+        assert!(!is_all_caps_header(&long));
+    }
+
+    #[test]
+    fn not_all_caps_digits_only() {
+        assert!(!is_all_caps_header("12345"));
+    }
+
+    // --- is_numbered_section ---
+
+    #[test]
+    fn numbered_simple() {
+        assert!(is_numbered_section("1. Introduction"));
+    }
+
+    #[test]
+    fn numbered_two_level() {
+        assert!(is_numbered_section("1.2 Background"));
+    }
+
+    #[test]
+    fn numbered_three_level() {
+        assert!(is_numbered_section("1.2.3 Methodology"));
+    }
+
+    #[test]
+    fn not_numbered_normal_text() {
+        assert!(!is_numbered_section("This is normal text."));
+    }
+
+    #[test]
+    fn not_numbered_no_dot() {
+        assert!(!is_numbered_section("1234 just a number"));
+    }
+
+    #[test]
+    fn not_numbered_indented() {
+        assert!(!is_numbered_section("  1.2 Indented"));
+    }
+
+    #[test]
+    fn not_numbered_bare_single_level() {
+        // "1." alone with no text is NOT a section header.
+        assert!(!is_numbered_section("1."));
+    }
+
+    #[test]
+    fn not_numbered_bare_two_level() {
+        // "1.2." alone with no text is NOT a section header.
+        assert!(!is_numbered_section("1.2."));
+    }
+
+    #[test]
+    fn not_numbered_whitespace_only_after_prefix() {
+        // "1. " with only whitespace after is NOT a section header.
+        assert!(!is_numbered_section("1.   "));
+    }
+
+    // --- is_title_line ---
+
+    #[test]
+    fn title_executive_summary() {
+        assert!(is_title_line("Executive Summary"));
+    }
+
+    #[test]
+    fn not_title_single_word() {
+        // Single words are not titles (require at least 2 words).
+        assert!(!is_title_line("Background"));
+        assert!(!is_title_line("Note"));
+        assert!(!is_title_line("Introduction"));
+    }
+
+    #[test]
+    fn title_risk_factors() {
+        assert!(is_title_line("Risk Factors"));
+    }
+
+    #[test]
+    fn not_title_ends_with_period() {
+        assert!(!is_title_line("This is a sentence."));
+    }
+
+    #[test]
+    fn not_title_ends_with_exclamation() {
+        assert!(!is_title_line("Watch out!"));
+    }
+
+    #[test]
+    fn not_title_too_long() {
+        let long = format!("A{}", "b".repeat(60));
+        assert!(!is_title_line(&long));
+    }
+
+    #[test]
+    fn not_title_indented() {
+        assert!(!is_title_line("  Indented Title"));
+    }
+
+    #[test]
+    fn not_title_ordinary_prose() {
+        // Ordinary short sentences should NOT be detected as titles.
+        assert!(!is_title_line("The cat sat on the mat"));
+        assert!(!is_title_line("I agree with you"));
+        assert!(!is_title_line("Call me later"));
+        assert!(!is_title_line("Total revenue was $5M"));
+    }
+
+    #[test]
+    fn not_title_starts_lowercase() {
+        assert!(!is_title_line("background"));
+    }
+
+    #[test]
+    fn not_title_too_many_words() {
+        assert!(!is_title_line("This Is A Title With Way Too Many Words In It"));
+    }
+
+    // --- detect_plain_text_boundaries ---
+
+    #[test]
+    fn detect_empty_text() {
+        let result = detect_plain_text_boundaries("");
+        assert!(result.is_empty());
+    }
+
+    #[test]
+    fn detect_no_headers() {
+        let text = "This is a normal sentence.\nAnother normal sentence here.\nNothing special going on.";
+        let result = detect_plain_text_boundaries(text);
+        assert!(result.is_empty());
+    }
+
+    #[test]
+    fn detect_byte_offsets_correct() {
+        let text = "INTRODUCTION\nSome body text here.\nCONCLUSION";
+        let result = detect_plain_text_boundaries(text);
+
+        assert_eq!(result.len(), 2);
+
+        // "INTRODUCTION" starts at offset 0
+        assert_eq!(result[0].byte_offset, 0);
+        assert!(result[0].is_header);
+
+        // "CONCLUSION" starts after "INTRODUCTION\nSome body text here.\n"
+        // = 12 + 1 + 21 + 1 = 35... let me compute:
+        // "INTRODUCTION" = 12 bytes, + 1 for \n = offset 13
+        // "Some body text here." = 20 bytes, + 1 for \n = offset 34 (13 + 21)
+        // Wait: "Some body text here." is 20 chars. 13 + 20 + 1 = 34.
+        let expected_offset = "INTRODUCTION".len() + 1 + "Some body text here.".len() + 1;
+        assert_eq!(result[1].byte_offset, expected_offset);
+        assert!(result[1].is_header);
+    }
+
+    #[test]
+    fn detect_mixed_boundary_types() {
+        let text = "CHAPTER ONE\n\n1.2 Background\n\nExecutive Summary\n\nThis is body text.";
+        let result = detect_plain_text_boundaries(text);
+
+        // CHAPTER ONE (all caps), 1.2 Background (numbered), Executive Summary (title)
+        assert_eq!(result.len(), 3);
+        assert!(result.iter().all(|b| b.is_header));
+
+        // Verify sorted by offset
+        for window in result.windows(2) {
+            assert!(window[0].byte_offset < window[1].byte_offset);
+        }
+    }
+
+    #[test]
+    fn not_title_starts_with_space() {
+        // is_title_line checks that first char is alphabetic — space is not
+        assert!(!is_title_line(" Leading space"));
+    }
+
+    #[test]
+    fn not_title_ends_with_colon() {
+        assert!(!is_title_line("Section:"));
+    }
+
+    #[test]
+    fn not_title_ends_with_semicolon() {
+        assert!(!is_title_line("Section;"));
+    }
+
+    #[test]
+    fn not_title_ends_with_question() {
+        assert!(!is_title_line("Is this a title?"));
+    }
+
+    // --- Mixed header types ---
+
+    #[test]
+    fn detect_all_caps_and_numbered_in_same_document() {
+        let text = "OVERVIEW\n\nSome body text.\n\n1.1 Subsection\n\nMore body text.";
+        let result = detect_plain_text_boundaries(text);
+        // OVERVIEW (all caps) + 1.1 Subsection (numbered) = 2 boundaries
+        assert_eq!(result.len(), 2, "should detect both ALL CAPS and numbered headers");
+    }
+
+    // --- Unicode / CJK text should NOT be detected as ALL CAPS ---
+
+    #[test]
+    fn unicode_cjk_not_all_caps() {
+        // Chinese characters are not uppercase Latin — should not match.
+        assert!(!is_all_caps_header("Chinese characters are not uppercase Latin"));
+        assert!(!is_all_caps_header("日本語テスト")); // has no uppercase Latin alpha
+    }
+
+    #[test]
+    fn unicode_hangul_not_all_caps() {
+        // Korean Hangul: alphabetic but has no uppercase concept.
+        // is_uppercase() returns false for Hangul, so it should not match.
+        assert!(!is_all_caps_header("한국어 테스트"));
+    }
+
+    #[test]
+    fn unicode_arabic_not_all_caps() {
+        assert!(!is_all_caps_header("مرحبا بالعالم"));
+    }
+
+    // --- Lines with only numbers / special chars ---
+
+    #[test]
+    fn numbers_and_special_chars_not_headers() {
+        assert!(!is_all_caps_header("12345"));
+        assert!(!is_all_caps_header("---"));
+        assert!(!is_all_caps_header("***"));
+        assert!(!is_all_caps_header("12.34.56"));
+    }
+
+    #[test]
+    fn detect_only_numbers_line_not_boundary() {
+        let text = "12345\n\nSome body text.\n\n67890";
+        let result = detect_plain_text_boundaries(text);
+        // None of these lines should be detected as headers.
+        assert!(
+            result.is_empty(),
+            "pure number/special-char lines should not be boundaries"
+        );
+    }
+
+    #[test]
+    fn detect_all_three_header_types_in_one_document() {
+        // ALL CAPS + numbered + title line in a single document.
+        let text = "ABSTRACT\n\nSome abstract text here.\n\n1.1 Methods\n\nMethodology description.\n\nFinal Remarks\n\nFinal remarks about the study.";
+        let result = detect_plain_text_boundaries(text);
+        // ABSTRACT (all caps) + 1.1 Methods (numbered) + Final Remarks (title) = 3 boundaries
+        assert_eq!(
+            result.len(),
+            3,
+            "should detect all three header types, got {} boundaries: {:?}",
+            result.len(),
+            result
+        );
+        // Verify offsets are strictly increasing.
+        for window in result.windows(2) {
+            assert!(
+                window[0].byte_offset < window[1].byte_offset,
+                "boundaries should be ordered by offset"
+            );
+        }
+    }
+
+    #[test]
+    fn unicode_chinese_text_not_all_caps() {
+        // Chinese characters have no uppercase concept — must not be detected.
+        assert!(!is_all_caps_header("\u{4e2d}\u{6587}\u{6d4b}\u{8bd5}")); // 中文测试
+        assert!(!is_all_caps_header("\u{7b2c}\u{4e00}\u{7ae0}")); // 第一章
+    }
+
+    #[test]
+    fn detect_chinese_text_not_boundary() {
+        // Full document with Chinese lines — none should be detected as ALL CAPS headers.
+        let text = "\u{7b2c}\u{4e00}\u{7ae0}\n\n\u{8fd9}\u{662f}\u{6b63}\u{6587}\u{5185}\u{5bb9}\u{3002}";
+        let result = detect_plain_text_boundaries(text);
+        // Chinese lines should not trigger is_all_caps_header, but may trigger
+        // is_title_line if they are short and don't end in sentence punctuation.
+        // The key point: they must not trigger is_all_caps_header.
+        for boundary in &result {
+            // Verify boundaries only come from title_line or numbered, not all_caps.
+            let line_at_offset = text[boundary.byte_offset..].lines().next().unwrap_or("");
+            assert!(
+                !is_all_caps_header(line_at_offset.trim()),
+                "Chinese text should not be detected as ALL CAPS: {:?}",
+                line_at_offset
+            );
+        }
+    }
+}

crates/kreuzberg/src/chunking/builder.rs

@@ -0,0 +1,221 @@
+//! Chunk construction and building logic.
+//!
+//! This module handles the construction of individual chunks from text segments,
+//! including overlap calculation, offset tracking, and metadata assembly.
+
+use crate::chunking::text_splitter::{Characters, ChunkCapacity, ChunkConfig};
+use crate::error::{KreuzbergError, Result};
+use crate::types::{Chunk, ChunkMetadata, PageBoundary};
+
+use super::boundaries::calculate_page_range;
+use super::classifier::classify_chunk;
+
+/// Default chunk size used when a zero value is passed (mirrors `default_chunk_size()`).
+const DEFAULT_CHUNK_SIZE: usize = 1000;
+
+/// Build a ChunkConfig from chunking parameters.
+///
+/// # Arguments
+///
+/// * `max_characters` - Maximum characters per chunk. A value of `0` is clamped to
+///   [`DEFAULT_CHUNK_SIZE`] (1000) to prevent panics from downstream `text-splitter`
+///   which requires a non-zero capacity.
+/// * `overlap` - Character overlap between consecutive chunks
+/// * `trim` - Whether to trim whitespace from boundaries
+///
+/// # Returns
+///
+/// A configured ChunkConfig ready for use with text splitters.
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Validation` if configuration is invalid.
+pub(crate) fn build_chunk_config(max_characters: usize, overlap: usize, trim: bool) -> Result<ChunkConfig<Characters>> {
+    let effective_max = if max_characters == 0 {
+        tracing::warn!(
+            clamped_to = DEFAULT_CHUNK_SIZE,
+            "chunk max_characters is 0; clamping to default to avoid panic"
+        );
+        DEFAULT_CHUNK_SIZE
+    } else {
+        max_characters
+    };
+    ChunkConfig::new(ChunkCapacity::new(effective_max))
+        .with_overlap(overlap)
+        .map(|config| config.with_trim(trim))
+        .map_err(|e| KreuzbergError::validation(format!("Invalid chunking configuration: {}", e)))
+}
+
+/// Build chunks from text segments with optional page boundary tracking.
+///
+/// This function takes a collection of text segments (produced by a text splitter)
+/// and constructs Chunk objects with proper metadata, including:
+/// - Byte offsets derived from the chunk's position in the source text
+/// - Chunk indices and total count
+/// - Page boundary information (if provided)
+///
+/// # Arguments
+///
+/// * `source_text` - The original text that the chunks were split from. Chunk
+///   slices must borrow from this text (as `text-splitter` guarantees).
+/// * `text_chunks` - Iterator of text segments to convert into chunks
+/// * `page_boundaries` - Optional page boundary markers for mapping chunks to pages
+///
+/// # Returns
+///
+/// A vector of Chunk objects with complete metadata.
+///
+/// # Errors
+///
+/// Returns an error if page boundary calculation fails.
+pub(crate) fn build_chunks<'a, I>(
+    source_text: &'a str,
+    text_chunks: I,
+    page_boundaries: Option<&[PageBoundary]>,
+) -> Result<Vec<Chunk>>
+where
+    I: IntoIterator<Item = &'a str>,
+{
+    let chunks_vec: Vec<&str> = text_chunks.into_iter().collect();
+    let total_chunks = chunks_vec.len();
+    let source_start = source_text.as_ptr() as usize;
+    let mut chunks = Vec::with_capacity(total_chunks);
+
+    for (index, chunk_text) in chunks_vec.into_iter().enumerate() {
+        let byte_start = chunk_text.as_ptr() as usize - source_start;
+        let byte_end = byte_start + chunk_text.len();
+
+        let (first_page, last_page) = if let Some(boundaries) = page_boundaries {
+            calculate_page_range(byte_start, byte_end, boundaries)?
+        } else {
+            (None, None)
+        };
+
+        chunks.push(Chunk {
+            content: chunk_text.to_string(),
+            chunk_type: classify_chunk(chunk_text, None),
+            embedding: None,
+            metadata: ChunkMetadata {
+                byte_start,
+                byte_end,
+                token_count: None,
+                chunk_index: index,
+                total_chunks,
+                first_page,
+                last_page,
+                heading_context: None,
+                image_indices: Vec::new(),
+            },
+        });
+    }
+
+    Ok(chunks)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_build_chunk_config_valid() {
+        let result = build_chunk_config(100, 10, true);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_build_chunk_config_zero_clamps_to_default() {
+        // Passing 0 must not panic — text-splitter requires non-zero capacity.
+        // The clamp should silently use DEFAULT_CHUNK_SIZE (1000).
+        let result = build_chunk_config(0, 0, true);
+        assert!(result.is_ok(), "zero max_characters must be clamped, not panic");
+    }
+
+    #[test]
+    fn test_build_chunk_config_invalid_overlap() {
+        let result = build_chunk_config(10, 20, true);
+        assert!(result.is_err());
+        let err = result.unwrap_err();
+        assert!(matches!(err, KreuzbergError::Validation { .. }));
+    }
+
+    #[test]
+    fn test_build_chunks_empty() {
+        let source = "";
+        let text_chunks: Vec<&str> = vec![];
+        let result = build_chunks(source, text_chunks, None).unwrap();
+        assert_eq!(result.len(), 0);
+    }
+
+    #[test]
+    fn test_build_chunks_single() {
+        let source = "Single chunk";
+        let result = build_chunks(source, vec![source], None).unwrap();
+        assert_eq!(result.len(), 1);
+        assert_eq!(result[0].content, "Single chunk");
+        assert_eq!(result[0].metadata.chunk_index, 0);
+        assert_eq!(result[0].metadata.total_chunks, 1);
+        assert_eq!(result[0].metadata.byte_start, 0);
+        assert_eq!(result[0].metadata.byte_end, 12);
+    }
+
+    #[test]
+    fn test_build_chunks_with_page_boundaries() {
+        let source = "First chunkSecond chunk";
+        let text_chunks = vec![&source[0..11], &source[11..23]];
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 11,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 11,
+                byte_end: 23,
+                page_number: 2,
+            },
+        ];
+
+        let result = build_chunks(source, text_chunks, Some(&boundaries)).unwrap();
+
+        assert_eq!(result.len(), 2);
+        assert_eq!(result[0].metadata.first_page, Some(1));
+        assert_eq!(result[1].metadata.first_page, Some(2));
+    }
+
+    #[test]
+    fn test_build_chunks_offset_from_source() {
+        let source = "AAAAABBBBBCCCCC";
+        // Overlapping slices from source
+        let text_chunks = vec![&source[0..5], &source[3..8], &source[6..11]];
+        let result = build_chunks(source, text_chunks, None).unwrap();
+
+        assert_eq!(result.len(), 3);
+
+        assert_eq!(result[0].metadata.byte_start, 0);
+        assert_eq!(result[0].metadata.byte_end, 5);
+
+        assert_eq!(result[1].metadata.byte_start, 3);
+        assert_eq!(result[1].metadata.byte_end, 8);
+
+        assert_eq!(result[2].metadata.byte_start, 6);
+        assert_eq!(result[2].metadata.byte_end, 11);
+    }
+
+    #[test]
+    fn test_build_chunks_no_overlap() {
+        let source = "AAAAABBBBBCCCCC";
+        let text_chunks = vec![&source[0..5], &source[5..10], &source[10..15]];
+        let result = build_chunks(source, text_chunks, None).unwrap();
+
+        assert_eq!(result.len(), 3);
+
+        assert_eq!(result[0].metadata.byte_start, 0);
+        assert_eq!(result[0].metadata.byte_end, 5);
+
+        assert_eq!(result[1].metadata.byte_start, 5);
+        assert_eq!(result[1].metadata.byte_end, 10);
+
+        assert_eq!(result[2].metadata.byte_start, 10);
+        assert_eq!(result[2].metadata.byte_end, 15);
+    }
+}

crates/kreuzberg/src/chunking/classifier.rs

@@ -0,0 +1,491 @@
+//! Heuristic semantic classifier for text chunks.
+//!
+//! Assigns a [`ChunkType`] to each text chunk based on structural signals
+//! (heading context, Markdown syntax) and content-level keyword patterns.
+//! Rules are evaluated in priority order; the first match wins.
+//!
+//! # Design
+//!
+//! - **No ML**: fully deterministic, zero-latency overhead, no external deps.
+//! - **Ordered rules**: higher-precision structural signals run before
+//!   lower-precision keyword heuristics.
+//! - **Extensible**: add new variants to [`ChunkType`] and insert new rules
+//!   without breaking existing classifications.
+
+use crate::types::{ChunkType, HeadingContext};
+
+/// Classify a single chunk based on its content and optional heading context.
+///
+/// Rules are evaluated in priority order. The first matching rule determines
+/// the returned [`ChunkType`]. When no rule matches, [`ChunkType::Unknown`]
+/// is returned.
+///
+/// # Arguments
+///
+/// * `content` - The text content of the chunk (may be trimmed or raw).
+/// * `heading_context` - Optional heading hierarchy this chunk falls under
+///   (only available when using `ChunkerType::Markdown`).
+///
+/// # Examples
+///
+/// ```ignore
+/// use kreuzberg::chunking::classifier::classify_chunk;
+/// use kreuzberg::types::ChunkType;
+///
+/// assert_eq!(classify_chunk("# Introduction", None), ChunkType::Heading);
+/// assert_eq!(
+///     classify_chunk("The Investor shall subscribe for the Shares and agrees to pay the subscription price. The Company shall deliver the Share certificates upon receipt.", None),
+///     ChunkType::OperativeClause,
+/// );
+/// assert_eq!(classify_chunk("Some unrecognized text.", None), ChunkType::Unknown);
+/// ```
+pub(crate) fn classify_chunk(content: &str, heading_context: Option<&HeadingContext>) -> ChunkType {
+    let trimmed = content.trim();
+
+    // ── 1. Heading ──────────────────────────────────────────────────────────
+    // A chunk that IS a heading (starts with `#`) or that sits at the top
+    // of the heading hierarchy (h1 only, very short content).
+    if is_heading(trimmed, heading_context) {
+        return ChunkType::Heading;
+    }
+
+    // ── 2. Code block ───────────────────────────────────────────────────────
+    // Use original content (not trimmed) so leading-indented blocks retain
+    // their 4-space prefix on every line.
+    if is_code_block(content) {
+        return ChunkType::CodeBlock;
+    }
+
+    // ── 3. Table-like ───────────────────────────────────────────────────────
+    if is_table_like(trimmed) {
+        return ChunkType::TableLike;
+    }
+
+    // ── 4. Formula ──────────────────────────────────────────────────────────
+    if is_formula(trimmed) {
+        return ChunkType::Formula;
+    }
+
+    // ── 5. Schedule / annex (heading context or keyword) ────────────────────
+    if is_schedule(trimmed, heading_context) {
+        return ChunkType::Schedule;
+    }
+
+    // ── 6. Definitions ──────────────────────────────────────────────────────
+    if is_definitions(trimmed) {
+        return ChunkType::Definitions;
+    }
+
+    // ── 7. Signature block ──────────────────────────────────────────────────
+    if is_signature_block(trimmed) {
+        return ChunkType::SignatureBlock;
+    }
+
+    // ── 8. Operative clause ─────────────────────────────────────────────────
+    if is_operative_clause(trimmed) {
+        return ChunkType::OperativeClause;
+    }
+
+    // ── 9. Party list ───────────────────────────────────────────────────────
+    if is_party_list(trimmed) {
+        return ChunkType::PartyList;
+    }
+
+    ChunkType::Unknown
+}
+
+// ─── Rule implementations ───────────────────────────────────────────────────
+
+fn is_heading(content: &str, _ctx: Option<&HeadingContext>) -> bool {
+    // Markdown ATX heading
+    if content.starts_with('#') {
+        return true;
+    }
+    // Setext-style heading: next line is `===` or `---`
+    let mut lines = content.lines();
+    if let (Some(_title), Some(underline)) = (lines.next(), lines.next()) {
+        let u = underline.trim();
+        if !u.is_empty() && (u.chars().all(|c| c == '=') || u.chars().all(|c| c == '-')) {
+            return true;
+        }
+    }
+    false
+}
+
+fn is_code_block(content: &str) -> bool {
+    // Fenced code block
+    if content.starts_with("```") || content.starts_with("~~~") {
+        return true;
+    }
+    // All non-empty lines indented ≥ 4 spaces (classic Markdown code block),
+    // but only when the block has ≥ 2 lines to avoid false positives.
+    let lines: Vec<&str> = content.lines().collect();
+    if lines.len() >= 2 {
+        let all_indented = lines
+            .iter()
+            .filter(|l| !l.trim().is_empty())
+            .all(|l| l.starts_with("    ") || l.starts_with('\t'));
+        if all_indented {
+            return true;
+        }
+    }
+    false
+}
+
+fn is_table_like(content: &str) -> bool {
+    let lines: Vec<&str> = content.lines().collect();
+    if lines.len() < 2 {
+        return false;
+    }
+    // Count lines that look like Markdown table rows: contain `|`
+    let pipe_lines = lines.iter().filter(|l| l.contains('|')).count();
+    if pipe_lines >= 2 {
+        return true;
+    }
+    // Count separator lines (`---`, `===`, repeated dashes ≥ 4)
+    let sep_lines = lines
+        .iter()
+        .filter(|l| {
+            let t = l.trim();
+            t.len() >= 4 && t.chars().all(|c| c == '-' || c == '+' || c == '|' || c == ' ')
+        })
+        .count();
+    sep_lines >= 3
+}
+
+fn is_formula(content: &str) -> bool {
+    // Unicode math symbols
+    const MATH_SYMBOLS: &[char] = &['∑', '∫', '√', '∂', '∏', '≤', '≥', '≠', '→', '←', '⊂', '⊃'];
+    if content.chars().any(|c| MATH_SYMBOLS.contains(&c)) {
+        return true;
+    }
+    // LaTeX-style patterns
+    let lower = content.to_lowercase();
+    let latex_patterns = [
+        r"\frac", r"\sum", r"\int", r"\sqrt", r"\alpha", r"\beta", r"\delta", r"$$", r"\[",
+    ];
+    if latex_patterns.iter().any(|p| lower.contains(p)) {
+        return true;
+    }
+    false
+}
+
+fn is_schedule(content: &str, ctx: Option<&HeadingContext>) -> bool {
+    const KEYWORDS: &[&str] = &["schedule", "annex", "appendix", "exhibit"];
+    let lower = content.to_lowercase();
+
+    // Check heading context for schedule keywords
+    if let Some(ctx) = ctx {
+        for h in &ctx.headings {
+            let hl = h.text.to_lowercase();
+            if KEYWORDS.iter().any(|k| hl.contains(k)) {
+                return true;
+            }
+        }
+    }
+    // First line starts with a schedule keyword
+    let first_line = content.lines().next().unwrap_or("").to_lowercase();
+    if KEYWORDS.iter().any(|k| first_line.starts_with(k)) {
+        return true;
+    }
+    // Content-level: strong keyword presence (e.g. "Schedule 1 –" or "Annex A:")
+
+    KEYWORDS.iter().any(|k| {
+        if let Some(idx) = lower.find(k) {
+            // Must be followed by a space + alphanumeric (e.g. "Schedule 1", "Annex A")
+            let rest = &lower[idx + k.len()..];
+            rest.starts_with(' ')
+                && rest
+                    .trim_start()
+                    .chars()
+                    .next()
+                    .map(|c| c.is_alphanumeric())
+                    .unwrap_or(false)
+        } else {
+            false
+        }
+    })
+}
+
+fn is_definitions(content: &str) -> bool {
+    let lower = content.to_lowercase();
+    // Classic legal definition patterns
+    let patterns = [
+        "\" means ",
+        "\" shall mean ",
+        "\" has the meaning",
+        "' means ",
+        "' shall mean ",
+        "means, for purposes",
+        "is defined as",
+        "shall be construed as",
+    ];
+    patterns.iter().any(|p| lower.contains(p))
+}
+
+fn is_signature_block(content: &str) -> bool {
+    let lower = content.to_lowercase();
+    let keywords = [
+        "signature",
+        "signed by",
+        "witnessed by",
+        "date:",
+        "in witness whereof",
+        "authorized signatory",
+        "duly authorized",
+        "____",
+    ];
+    let hits = keywords.iter().filter(|k| lower.contains(*k)).count();
+    hits >= 2
+}
+
+fn is_operative_clause(content: &str) -> bool {
+    let lower = content.to_lowercase();
+    // Action verbs commonly found in operative legal clauses
+    let verbs = [
+        "shall ",
+        "agree ",
+        "agrees ",
+        "transfer",
+        "grant ",
+        "grants ",
+        "undertake",
+        "obligat",
+        "covenant",
+        "warrant",
+        "represent",
+        "indemnif",
+        "assign ",
+        "assigns ",
+        "license ",
+        "licenses ",
+        "purchase",
+        "sell ",
+        "sells ",
+        "pay ",
+        "pays ",
+        "deliver",
+    ];
+    let hits = verbs.iter().filter(|v| lower.contains(*v)).count();
+    hits >= 3
+}
+
+fn is_party_list(content: &str) -> bool {
+    // Party lists tend to have multiple short lines with Title Case names,
+    // often mixed with addresses (contain digits) or role labels.
+    let lines: Vec<&str> = content.lines().map(|l| l.trim()).filter(|l| !l.is_empty()).collect();
+
+    if lines.len() < 3 {
+        return false;
+    }
+
+    let party_like = lines.iter().filter(|l| is_party_line(l)).count();
+    // Majority of lines should look party-like
+    party_like >= (lines.len() * 2 / 3).max(2)
+}
+
+/// Heuristic for a single "party-like" line.
+///
+/// A line looks like a party entry when it:
+/// - Is short (≤ 120 chars), AND
+/// - Starts with an uppercase letter (Title Case name or role), AND
+/// - Contains at least one of: a comma, a digit (address), or a role keyword.
+fn is_party_line(line: &str) -> bool {
+    if line.len() > 120 {
+        return false;
+    }
+    let starts_upper = line.chars().next().map(|c| c.is_uppercase()).unwrap_or(false);
+    if !starts_upper {
+        return false;
+    }
+    let has_digit = line.chars().any(|c| c.is_ascii_digit());
+    let has_comma = line.contains(',');
+    let lower = line.to_lowercase();
+    let has_role = [
+        "investor",
+        "company",
+        "borrower",
+        "lender",
+        "seller",
+        "buyer",
+        "party",
+        "subscriber",
+        "guarantor",
+    ]
+    .iter()
+    .any(|r| lower.contains(r));
+    has_digit || has_comma || has_role
+}
+
+// ─── Tests ──────────────────────────────────────────────────────────────────
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn classify(content: &str) -> ChunkType {
+        classify_chunk(content, None)
+    }
+
+    // ── Heading ──────────────────────────────────────────────────────────────
+
+    #[test]
+    fn test_heading_atx() {
+        assert_eq!(classify("# Introduction"), ChunkType::Heading);
+        assert_eq!(classify("## Section 2"), ChunkType::Heading);
+        assert_eq!(classify("### Sub-section"), ChunkType::Heading);
+    }
+
+    #[test]
+    fn test_heading_setext() {
+        assert_eq!(classify("Introduction\n============"), ChunkType::Heading);
+        assert_eq!(classify("Section 2\n---------"), ChunkType::Heading);
+    }
+
+    #[test]
+    fn test_not_heading_plain_text() {
+        assert_ne!(classify("This is plain paragraph text."), ChunkType::Heading);
+    }
+
+    // ── Code block ───────────────────────────────────────────────────────────
+
+    #[test]
+    fn test_code_block_fenced() {
+        assert_eq!(classify("```rust\nfn main() {}\n```"), ChunkType::CodeBlock);
+        assert_eq!(classify("~~~python\nprint('hi')\n~~~"), ChunkType::CodeBlock);
+    }
+
+    #[test]
+    fn test_code_block_indented() {
+        let indented = "    fn main() {\n        println!(\"hello\");\n    }";
+        assert_eq!(classify(indented), ChunkType::CodeBlock);
+    }
+
+    // ── Table-like ───────────────────────────────────────────────────────────
+
+    #[test]
+    fn test_table_markdown() {
+        let table = "| Name | Age |\n|------|-----|\n| Alice | 30 |";
+        assert_eq!(classify(table), ChunkType::TableLike);
+    }
+
+    #[test]
+    fn test_table_single_pipe_line_not_table() {
+        // Only one pipe line → not enough evidence
+        assert_ne!(classify("Just one | separator here"), ChunkType::TableLike);
+    }
+
+    // ── Formula ──────────────────────────────────────────────────────────────
+
+    #[test]
+    fn test_formula_unicode_symbols() {
+        assert_eq!(classify("The total ∑ of all values equals 1."), ChunkType::Formula);
+        assert_eq!(classify("∫ f(x) dx from 0 to ∞"), ChunkType::Formula);
+    }
+
+    #[test]
+    fn test_formula_latex() {
+        assert_eq!(classify(r"The result is $\frac{a}{b}$"), ChunkType::Formula);
+        assert_eq!(classify(r"$$\sum_{i=0}^{n} x_i$$"), ChunkType::Formula);
+    }
+
+    // ── Schedule ─────────────────────────────────────────────────────────────
+
+    #[test]
+    fn test_schedule_first_line() {
+        assert_eq!(
+            classify("Schedule 1 – Definitions\n\nThis schedule sets out..."),
+            ChunkType::Schedule
+        );
+        assert_eq!(classify("annex A: Technical Specifications"), ChunkType::Schedule);
+    }
+
+    // ── Definitions ──────────────────────────────────────────────────────────
+
+    #[test]
+    fn test_definitions_means() {
+        assert_eq!(
+            classify("\"Agreement\" means this Investment and Subscription Agreement."),
+            ChunkType::Definitions
+        );
+        assert_eq!(
+            classify("\"Closing Date\" shall mean the date on which..."),
+            ChunkType::Definitions
+        );
+    }
+
+    #[test]
+    fn test_definitions_is_defined_as() {
+        assert_eq!(
+            classify("The term 'Net Revenue' is defined as all revenue..."),
+            ChunkType::Definitions
+        );
+    }
+
+    // ── Signature block ───────────────────────────────────────────────────────
+
+    #[test]
+    fn test_signature_block() {
+        let sig = "Signed by: John Smith\nDate: 2026-03-30\nWitnessed by: Jane Doe";
+        assert_eq!(classify(sig), ChunkType::SignatureBlock);
+    }
+
+    #[test]
+    fn test_signature_block_in_witness() {
+        let sig = "In witness whereof the parties have duly authorized this agreement.\n____________________\nDate: ___________";
+        assert_eq!(classify(sig), ChunkType::SignatureBlock);
+    }
+
+    // ── Operative clause ─────────────────────────────────────────────────────
+
+    #[test]
+    fn test_operative_clause_basic() {
+        let clause = "The Investor shall subscribe for the Shares and agrees to pay the subscription price. The Company shall deliver the Share certificates upon receipt.";
+        assert_eq!(classify(clause), ChunkType::OperativeClause);
+    }
+
+    #[test]
+    fn test_operative_clause_grant() {
+        let clause = "The Licensor hereby grants, assigns, and transfers all right, title, and interest. The Licensee shall pay and deliver consideration.";
+        assert_eq!(classify(clause), ChunkType::OperativeClause);
+    }
+
+    // ── Party list ────────────────────────────────────────────────────────────
+
+    #[test]
+    fn test_party_list_basic() {
+        let parties = "Gregor Guggisberg, Winkelstrasse 12, Zurich\nInvestor\nAlpha Capital AG, Bahnhofstrasse 1, Zurich\nSubscriber\nBeta Holdings Ltd, 10 City Road, London\nBorrower";
+        assert_eq!(classify(parties), ChunkType::PartyList);
+    }
+
+    // ── Unknown ───────────────────────────────────────────────────────────────
+
+    #[test]
+    fn test_unknown_plain_text() {
+        assert_eq!(
+            classify("This document contains general information."),
+            ChunkType::Unknown
+        );
+    }
+
+    #[test]
+    fn test_unknown_empty() {
+        assert_eq!(classify(""), ChunkType::Unknown);
+    }
+
+    // ── Heading context ───────────────────────────────────────────────────────
+
+    #[test]
+    fn test_heading_context_schedule() {
+        use crate::types::{HeadingContext, HeadingLevel};
+        let ctx = HeadingContext {
+            headings: vec![HeadingLevel {
+                level: 1,
+                text: "Schedule 1 – Definitions".to_string(),
+            }],
+        };
+        // Content under a Schedule heading should be classified as Schedule
+        let result = classify_chunk("This schedule sets out the defined terms.", Some(&ctx));
+        assert_eq!(result, ChunkType::Schedule);
+    }
+}

crates/kreuzberg/src/chunking/config.rs

@@ -0,0 +1,18 @@
+//! Configuration types for text chunking.
+
+use serde::{Deserialize, Serialize};
+
+// Re-export ChunkingConfig and ChunkerType from core config (canonical location)
+pub use crate::core::config::processing::{ChunkSizing, ChunkerType, ChunkingConfig};
+
+/// Result of a text chunking operation.
+///
+/// Contains the generated chunks and metadata about the chunking.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ChunkingResult {
+    /// List of text chunks
+    pub chunks: Vec<crate::types::Chunk>,
+    /// Total number of chunks generated
+    pub chunk_count: usize,
+}

crates/kreuzberg/src/chunking/headings.rs

@@ -0,0 +1,174 @@
+//! Heading extraction for Markdown chunk metadata.
+//!
+//! Parses markdown text to build a heading map, then resolves
+//! which headings a chunk falls under based on its byte offset.
+
+use crate::types::{HeadingContext, HeadingLevel};
+use pulldown_cmark::{Event, Options, Parser, TagEnd};
+
+/// An entry in the heading map: `(byte_offset, level, text)`.
+type HeadingEntry = (usize, u8, String);
+
+/// Build a heading map from markdown text.
+///
+/// Returns a sorted Vec of `(byte_offset, level, heading_text)` for each heading found.
+pub(crate) fn build_heading_map(markdown: &str) -> Vec<HeadingEntry> {
+    let parser = Parser::new_ext(markdown, Options::all());
+    let mut headings = Vec::new();
+    let mut current_heading: Option<(usize, u8)> = None;
+    let mut heading_text = String::new();
+
+    for (event, range) in parser.into_offset_iter() {
+        match event {
+            Event::Start(pulldown_cmark::Tag::Heading { level, .. }) => {
+                current_heading = Some((range.start, heading_level_to_u8(level)));
+                heading_text.clear();
+            }
+            Event::Text(text) if current_heading.is_some() => {
+                heading_text.push_str(&text);
+            }
+            Event::Code(code) if current_heading.is_some() => {
+                heading_text.push_str(&code);
+            }
+            Event::End(TagEnd::Heading(_)) => {
+                if let Some((offset, level)) = current_heading.take() {
+                    headings.push((offset, level, heading_text.clone()));
+                }
+            }
+            _ => {}
+        }
+    }
+    headings
+}
+
+/// Resolve the heading context for a chunk at the given byte offset.
+///
+/// Walks the heading map to find all headings that precede `byte_start`,
+/// building a proper hierarchy stack (h1 > h2 > h3, etc.).
+pub(crate) fn resolve_heading_context(byte_start: usize, heading_map: &[HeadingEntry]) -> Option<HeadingContext> {
+    let mut stack: Vec<HeadingLevel> = Vec::new();
+
+    for &(offset, level, ref text) in heading_map {
+        if offset > byte_start {
+            break;
+        }
+        // Pop headings at same or deeper level (they've been superseded).
+        while stack.last().is_some_and(|h| h.level >= level) {
+            stack.pop();
+        }
+        stack.push(HeadingLevel {
+            level,
+            text: text.clone(),
+        });
+    }
+
+    if stack.is_empty() {
+        None
+    } else {
+        Some(HeadingContext { headings: stack })
+    }
+}
+
+fn heading_level_to_u8(level: pulldown_cmark::HeadingLevel) -> u8 {
+    match level {
+        pulldown_cmark::HeadingLevel::H1 => 1,
+        pulldown_cmark::HeadingLevel::H2 => 2,
+        pulldown_cmark::HeadingLevel::H3 => 3,
+        pulldown_cmark::HeadingLevel::H4 => 4,
+        pulldown_cmark::HeadingLevel::H5 => 5,
+        pulldown_cmark::HeadingLevel::H6 => 6,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_build_heading_map_basic() {
+        let md = "# Title\n\nSome text.\n\n## Section 1\n\nContent.\n\n## Section 2\n\nMore content.";
+        let map = build_heading_map(md);
+        assert_eq!(map.len(), 3);
+        assert_eq!(map[0], (0, 1, "Title".to_string()));
+        assert_eq!(map[1].1, 2);
+        assert_eq!(map[1].2, "Section 1");
+        assert_eq!(map[2].1, 2);
+        assert_eq!(map[2].2, "Section 2");
+    }
+
+    #[test]
+    fn test_build_heading_map_nested() {
+        let md = "# H1\n\n## H2\n\n### H3\n\nText.";
+        let map = build_heading_map(md);
+        assert_eq!(map.len(), 3);
+        assert_eq!(map[0].1, 1);
+        assert_eq!(map[1].1, 2);
+        assert_eq!(map[2].1, 3);
+    }
+
+    #[test]
+    fn test_build_heading_map_no_headings() {
+        let md = "Just plain text without any headings.";
+        let map = build_heading_map(md);
+        assert!(map.is_empty());
+    }
+
+    #[test]
+    fn test_build_heading_map_with_code_in_heading() {
+        let md = "# Title with `code`\n\nText.";
+        let map = build_heading_map(md);
+        assert_eq!(map.len(), 1);
+        assert_eq!(map[0].2, "Title with code");
+    }
+
+    #[test]
+    fn test_resolve_heading_context_under_h2() {
+        let map = vec![
+            (0, 1, "Title".to_string()),
+            (10, 2, "Section A".to_string()),
+            (30, 2, "Section B".to_string()),
+        ];
+        // Chunk at byte 15 is under h1 > h2("Section A")
+        let ctx = resolve_heading_context(15, &map).unwrap();
+        assert_eq!(ctx.headings.len(), 2);
+        assert_eq!(ctx.headings[0].level, 1);
+        assert_eq!(ctx.headings[0].text, "Title");
+        assert_eq!(ctx.headings[1].level, 2);
+        assert_eq!(ctx.headings[1].text, "Section A");
+    }
+
+    #[test]
+    fn test_resolve_heading_context_root() {
+        let map = vec![(10, 1, "Title".to_string())];
+        // Chunk at byte 0 is before any heading
+        let ctx = resolve_heading_context(0, &map);
+        assert!(ctx.is_none());
+    }
+
+    #[test]
+    fn test_resolve_heading_context_superseded() {
+        let map = vec![
+            (0, 1, "Title".to_string()),
+            (10, 2, "Section A".to_string()),
+            (20, 3, "Subsection".to_string()),
+            (30, 2, "Section B".to_string()), // This supersedes h3 "Subsection"
+        ];
+        // Chunk at byte 35 should be h1 > h2("Section B"), no h3
+        let ctx = resolve_heading_context(35, &map).unwrap();
+        assert_eq!(ctx.headings.len(), 2);
+        assert_eq!(ctx.headings[1].text, "Section B");
+    }
+
+    #[test]
+    fn test_resolve_heading_context_deep_nesting() {
+        let map = vec![
+            (0, 1, "H1".to_string()),
+            (5, 2, "H2".to_string()),
+            (10, 3, "H3".to_string()),
+            (15, 4, "H4".to_string()),
+        ];
+        let ctx = resolve_heading_context(20, &map).unwrap();
+        assert_eq!(ctx.headings.len(), 4);
+        assert_eq!(ctx.headings[3].level, 4);
+    }
+}

crates/kreuzberg/src/chunking/mod.rs

@@ -0,0 +1,112 @@
+//! Text chunking utilities.
+//!
+//! This module provides text chunking functionality using the `text-splitter` library.
+//! It splits long text into smaller chunks while preserving semantic boundaries.
+//!
+//! # Features
+//!
+//! - **Smart splitting**: Respects word and sentence boundaries
+//! - **Markdown-aware**: Preserves Markdown structure (headings, code blocks, lists)
+//! - **Configurable overlap**: Overlap chunks to maintain context
+//! - **Unicode support**: Handles CJK characters and emojis correctly
+//! - **Batch processing**: Process multiple texts efficiently
+//!
+//! # Chunker Types
+//!
+//! - **Text**: Generic text splitter, splits on whitespace and punctuation
+//! - **Markdown**: Markdown-aware splitter, preserves formatting and structure
+//! - **Yaml**: YAML-aware splitter, creates one chunk per top-level key
+//! - **Semantic**: Topic-aware chunker that groups paragraphs by semantic similarity
+//!
+//! # Example
+//!
+//! ```rust
+//! use kreuzberg::chunking::{chunk_text, ChunkingConfig, ChunkerType};
+//!
+//! # fn example() -> kreuzberg::Result<()> {
+//! let config = ChunkingConfig {
+//!     max_characters: 500,
+//!     overlap: 50,
+//!     trim: true,
+//!     chunker_type: ChunkerType::Text,
+//!     ..Default::default()
+//! };
+//!
+//! let long_text = "This is a very long document...".repeat(100);
+//! let result = chunk_text(&long_text, &config, None)?;
+//!
+//! println!("Split into {} chunks", result.chunk_count);
+//! for (i, chunk) in result.chunks.iter().enumerate() {
+//!     println!("Chunk {}: {} chars", i + 1, chunk.content.len());
+//! }
+//! # Ok(())
+//! # }
+//! ```
+//!
+//! # Use Cases
+//!
+//! - Splitting documents for LLM context windows
+//! - Creating overlapping chunks for semantic search
+//! - Processing large documents in batches
+//! - Maintaining context across chunk boundaries
+
+use once_cell::sync::OnceCell;
+use std::sync::Arc;
+
+// Module declarations
+pub mod boundaries;
+pub mod boundary_detection;
+mod builder;
+pub mod classifier;
+pub mod config;
+pub mod core;
+mod headings;
+pub mod processor;
+pub mod semantic;
+mod text_splitter;
+#[cfg(feature = "chunking-tokenizers")]
+mod tokenizer_cache;
+pub mod validation;
+mod yaml_section;
+
+// Re-export submodule types and functions
+pub use config::{ChunkSizing, ChunkerType, ChunkingConfig, ChunkingResult}; // ChunkingConfig re-exported from core::config::processing
+pub use core::chunk_text;
+pub(crate) use core::chunk_text_with_heading_source;
+pub use processor::ChunkingProcessor;
+
+use crate::error::Result;
+
+/// One-time initialization guard for the chunking processor registry.
+///
+/// Set to `()` once registration succeeds. If registration fails the cell remains
+/// empty, allowing the next call to retry.
+static PROCESSOR_INITIALIZED: OnceCell<()> = OnceCell::new();
+
+/// Ensure the chunking processor is registered.
+///
+/// This function is called automatically when needed.
+/// It's safe to call multiple times - registration only happens once.
+#[cfg_attr(alef, alef(skip))]
+pub(crate) fn ensure_initialized() -> Result<()> {
+    PROCESSOR_INITIALIZED
+        .get_or_try_init(register_chunking_processor)
+        .map(|_| ())
+}
+
+/// Register the chunking processor with the global registry.
+///
+/// This function should be called once at application startup to register
+/// the chunking post-processor.
+///
+/// **Note:** This is called automatically on first use.
+/// Explicit calling is optional.
+#[cfg_attr(alef, alef(skip))]
+pub(crate) fn register_chunking_processor() -> Result<()> {
+    let registry = crate::plugins::registry::get_post_processor_registry();
+    let mut registry = registry.write();
+
+    registry.register(Arc::new(ChunkingProcessor))?;
+
+    Ok(())
+}

crates/kreuzberg/src/chunking/processor.rs

@@ -0,0 +1,337 @@
+//! Text chunking post-processor.
+//!
+//! This module provides a PostProcessor plugin that chunks text content in
+//! extraction results.
+
+use crate::chunking::config::{ChunkerType, ChunkingConfig};
+use crate::plugins::{Plugin, PostProcessor, ProcessingStage};
+use crate::types::Metadata;
+use crate::{ExtractionConfig, ExtractionResult, KreuzbergError, Result};
+use async_trait::async_trait;
+
+/// Post-processor that chunks text in document content.
+///
+/// This processor:
+/// - Runs in the Middle processing stage
+/// - Only processes when `config.chunking` is configured
+/// - Stores chunks in `result.chunks`
+/// - Uses configurable chunk size and overlap
+///
+/// # Example
+///
+/// ```rust,no_run
+/// use kreuzberg::plugins::{Plugin, PostProcessor};
+/// use kreuzberg::chunking::processor::ChunkingProcessor;
+///
+/// let processor = ChunkingProcessor;
+/// assert_eq!(processor.name(), "text-chunking");
+/// ```
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Copy)]
+pub struct ChunkingProcessor;
+
+impl Plugin for ChunkingProcessor {
+    fn name(&self) -> &str {
+        "text-chunking"
+    }
+
+    fn version(&self) -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+
+    fn initialize(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn shutdown(&self) -> Result<()> {
+        Ok(())
+    }
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl PostProcessor for ChunkingProcessor {
+    async fn process(&self, result: &mut ExtractionResult, config: &ExtractionConfig) -> Result<()> {
+        let chunking_config = match &config.chunking {
+            Some(cfg) => cfg,
+            None => return Ok(()),
+        };
+
+        let inferred = maybe_infer_yaml_chunker(chunking_config, &result.metadata);
+        let effective_config = inferred.as_ref().unwrap_or(chunking_config);
+
+        let chunking_result = crate::chunking::chunk_text(&result.content, effective_config, None)
+            .map_err(|e| KreuzbergError::Other(format!("Chunking failed: {}", e)))?;
+        result.chunks = Some(chunking_result.chunks);
+
+        Ok(())
+    }
+
+    fn processing_stage(&self) -> ProcessingStage {
+        ProcessingStage::Middle
+    }
+
+    fn should_process(&self, _result: &ExtractionResult, config: &ExtractionConfig) -> bool {
+        config.chunking.is_some()
+    }
+
+    fn estimated_duration_ms(&self, result: &ExtractionResult) -> u64 {
+        let text_length = result.content.len();
+        (text_length / 10240).max(1) as u64
+    }
+}
+
+/// Returns an overridden config if auto-inference applies, or None to use the original.
+///
+/// If the user left the chunker type at default (`Text`) and the document metadata
+/// indicates a YAML or JSON data format, returns a config with `Yaml` chunker type.
+/// An explicit non-default choice by the user is never overridden.
+fn maybe_infer_yaml_chunker(config: &ChunkingConfig, metadata: &Metadata) -> Option<ChunkingConfig> {
+    if config.chunker_type != ChunkerType::Text {
+        return None;
+    }
+
+    let is_structured = metadata
+        .additional
+        .get("data_format")
+        .and_then(|v| v.as_str())
+        .is_some_and(|fmt| fmt == "yaml" || fmt == "json");
+
+    is_structured.then(|| ChunkingConfig {
+        chunker_type: ChunkerType::Yaml,
+        ..config.clone()
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::core::config::ChunkingConfig;
+    use crate::types::Metadata;
+    use std::borrow::Cow;
+
+    #[tokio::test]
+    async fn test_chunking_processor() {
+        let processor = ChunkingProcessor;
+        let config = ExtractionConfig {
+            chunking: Some(ChunkingConfig {
+                max_characters: 100,
+                overlap: 10,
+                trim: true,
+                chunker_type: crate::chunking::ChunkerType::Text,
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let mut result = ExtractionResult {
+            content: "This is a longer text that should be split into multiple chunks to test the chunking processor functionality.".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
+            ..Default::default()
+        };
+
+        processor.process(&mut result, &config).await.unwrap();
+
+        assert!(result.chunks.is_some());
+        let chunks = result.chunks.unwrap();
+        assert!(!chunks.is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_chunking_processor_no_config() {
+        let processor = ChunkingProcessor;
+        let config = ExtractionConfig::default();
+
+        let mut result = ExtractionResult {
+            content: "Some text".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
+            ..Default::default()
+        };
+
+        processor.process(&mut result, &config).await.unwrap();
+
+        assert!(result.chunks.is_none());
+    }
+
+    #[test]
+    fn test_chunking_processor_plugin_interface() {
+        let processor = ChunkingProcessor;
+        assert_eq!(processor.name(), "text-chunking");
+        assert!(!processor.version().is_empty());
+        assert!(processor.initialize().is_ok());
+        assert!(processor.shutdown().is_ok());
+    }
+
+    #[test]
+    fn test_chunking_processor_stage() {
+        let processor = ChunkingProcessor;
+        assert_eq!(processor.processing_stage(), ProcessingStage::Middle);
+    }
+
+    #[test]
+    fn test_chunking_processor_should_process() {
+        let processor = ChunkingProcessor;
+
+        let result = ExtractionResult {
+            content: "Sample text".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
+            ..Default::default()
+        };
+
+        let config_with_chunking = ExtractionConfig {
+            chunking: Some(crate::core::config::ChunkingConfig {
+                max_characters: 100,
+                overlap: 10,
+                trim: true,
+                chunker_type: crate::chunking::ChunkerType::Text,
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        assert!(processor.should_process(&result, &config_with_chunking));
+
+        let config_without_chunking = ExtractionConfig::default();
+        assert!(!processor.should_process(&result, &config_without_chunking));
+    }
+
+    #[test]
+    fn test_chunking_processor_estimated_duration() {
+        let processor = ChunkingProcessor;
+
+        let short_result = ExtractionResult {
+            content: "Short".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
+            ..Default::default()
+        };
+
+        let long_result = ExtractionResult {
+            content: "a".repeat(100000),
+            mime_type: Cow::Borrowed("text/plain"),
+            ..Default::default()
+        };
+
+        let short_duration = processor.estimated_duration_ms(&short_result);
+        let long_duration = processor.estimated_duration_ms(&long_result);
+
+        assert!(long_duration > short_duration);
+    }
+
+    fn make_metadata_with_format(format: &str) -> Metadata {
+        let mut metadata = Metadata::default();
+        metadata
+            .additional
+            .insert(Cow::Borrowed("data_format"), serde_json::json!(format));
+        metadata
+    }
+
+    #[tokio::test]
+    async fn test_auto_infer_yaml_from_metadata() {
+        let processor = ChunkingProcessor;
+        let config = ExtractionConfig {
+            chunking: Some(ChunkingConfig {
+                max_characters: 10000,
+                overlap: 0,
+                trim: true,
+                chunker_type: crate::chunking::ChunkerType::Text,
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let yaml_content = "server:\n  host: localhost\n  port: 8080";
+        let mut result = ExtractionResult {
+            content: yaml_content.to_string(),
+            mime_type: Cow::Borrowed("text/yaml"),
+            metadata: make_metadata_with_format("yaml"),
+            ..Default::default()
+        };
+
+        processor.process(&mut result, &config).await.unwrap();
+        let chunks = result.chunks.unwrap();
+        // Yaml chunker produces section-prefixed chunks
+        assert!(chunks[0].content.contains("# server > host"));
+    }
+
+    #[tokio::test]
+    async fn test_auto_infer_json_from_metadata() {
+        let processor = ChunkingProcessor;
+        let config = ExtractionConfig {
+            chunking: Some(ChunkingConfig {
+                max_characters: 10000,
+                overlap: 0,
+                trim: true,
+                chunker_type: crate::chunking::ChunkerType::Text,
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let json_content = r#"{"name": "test", "version": "1.0"}"#;
+        let mut result = ExtractionResult {
+            content: json_content.to_string(),
+            mime_type: Cow::Borrowed("application/json"),
+            metadata: make_metadata_with_format("json"),
+            ..Default::default()
+        };
+
+        processor.process(&mut result, &config).await.unwrap();
+        let chunks = result.chunks.unwrap();
+        // JSON chunker produces section-prefixed chunks
+        assert!(chunks[0].content.contains("# name"));
+    }
+
+    #[tokio::test]
+    async fn test_explicit_type_not_overridden() {
+        let processor = ChunkingProcessor;
+        let config = ExtractionConfig {
+            chunking: Some(ChunkingConfig {
+                max_characters: 10000,
+                overlap: 0,
+                trim: true,
+                chunker_type: crate::chunking::ChunkerType::Markdown,
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let yaml_content = "server:\n  host: localhost\n  port: 8080";
+        let mut result = ExtractionResult {
+            content: yaml_content.to_string(),
+            mime_type: Cow::Borrowed("text/yaml"),
+            metadata: make_metadata_with_format("yaml"),
+            ..Default::default()
+        };
+
+        processor.process(&mut result, &config).await.unwrap();
+        let chunks = result.chunks.unwrap();
+        // Markdown chunker does NOT produce "# server > host" section headers
+        assert!(!chunks[0].content.contains("# server > host"));
+    }
+
+    #[tokio::test]
+    async fn test_missing_data_format_no_inference() {
+        let processor = ChunkingProcessor;
+        let config = ExtractionConfig {
+            chunking: Some(ChunkingConfig {
+                max_characters: 10000,
+                overlap: 0,
+                trim: true,
+                chunker_type: crate::chunking::ChunkerType::Text,
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let yaml_content = "server:\n  host: localhost\n  port: 8080";
+        let mut result = ExtractionResult {
+            content: yaml_content.to_string(),
+            mime_type: Cow::Borrowed("text/yaml"),
+            ..Default::default()
+        };
+
+        processor.process(&mut result, &config).await.unwrap();
+        let chunks = result.chunks.unwrap();
+        // Without data_format metadata, should NOT auto-infer yaml chunking
+        assert!(!chunks[0].content.contains("# server > host"));
+    }
+}

crates/kreuzberg/src/chunking/semantic/merge.rs

@@ -0,0 +1,478 @@
+//! Segment merging for semantic chunking.
+//!
+//! Groups segments by topic boundaries, merges within groups, and falls back
+//! to `text_splitter::TextSplitter` when a merged group exceeds the budget.
+
+use crate::chunking::text_splitter::TextSplitter;
+
+/// A text segment with its byte offset in the original document.
+#[derive(Debug, Clone, Copy)]
+pub struct Segment<'a> {
+    pub text: &'a str,
+    pub byte_start: usize,
+}
+
+/// A merged chunk produced by [`merge_segments`].
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct MergedChunk {
+    pub text: String,
+    pub byte_start: usize,
+    pub byte_end: usize,
+}
+
+/// Extract up to `n` characters from the end of `s`, respecting UTF-8 boundaries.
+fn tail_chars(s: &str, n: usize) -> &str {
+    if n == 0 {
+        return "";
+    }
+    let total = s.chars().count();
+    if n >= total {
+        return s;
+    }
+    let skip = total - n;
+    let byte_offset = s.char_indices().nth(skip).map(|(i, _)| i).unwrap_or(0);
+    &s[byte_offset..]
+}
+
+/// Prepend overlap text from the previous group, staying within the ceiling.
+fn prepend_overlap(group_text: &str, prev_tail: &str, overlap: usize, ceiling: usize) -> String {
+    let tail = tail_chars(prev_tail, overlap);
+    if tail.is_empty() {
+        return group_text.to_string();
+    }
+
+    let candidate = format!("{tail}\n\n{group_text}");
+    if candidate.chars().count() <= ceiling {
+        return candidate;
+    }
+
+    let group_chars = group_text.chars().count();
+    let available = ceiling.saturating_sub(group_chars + 2);
+    if available > 0 {
+        let truncated = tail_chars(prev_tail, available);
+        format!("{truncated}\n\n{group_text}")
+    } else {
+        group_text.to_string()
+    }
+}
+
+/// Merge segments into chunks guided by topic boundaries.
+///
+/// # Arguments
+///
+/// * `source_text` – the original document text that segments reference.
+/// * `segments` – ordered text segments (subslices of `source_text`).
+/// * `boundaries` – parallel bool slice; `true` at index `i` means segment `i`
+///   starts a new topic group.
+/// * `max_characters` – maximum characters per output chunk.
+/// * `overlap` – number of characters from the tail of the previous group to
+///   prepend to the next group's first chunk.
+///
+/// # Panics (debug)
+///
+/// Debug-asserts that `segments.len() == boundaries.len()`.
+pub(crate) fn merge_segments(
+    source_text: &str,
+    segments: &[Segment<'_>],
+    boundaries: &[bool],
+    max_characters: usize,
+    overlap: usize,
+) -> Vec<MergedChunk> {
+    if segments.is_empty() {
+        return Vec::new();
+    }
+    debug_assert_eq!(
+        segments.len(),
+        boundaries.len(),
+        "segments and boundaries must have the same length"
+    );
+
+    let mut groups: Vec<std::ops::Range<usize>> = Vec::new();
+    let mut group_start = 0;
+    for (i, &is_boundary) in boundaries.iter().enumerate().skip(1) {
+        if is_boundary {
+            groups.push(group_start..i);
+            group_start = i;
+        }
+    }
+    groups.push(group_start..segments.len());
+
+    let mut result: Vec<MergedChunk> = Vec::new();
+    let mut prev_group_tail: Option<String> = None;
+
+    for group in &groups {
+        let group_segments = &segments[group.clone()];
+
+        // Groups are always non-empty: formed from `start..i` where start < i.
+        let group_byte_start = group_segments.first().unwrap().byte_start;
+        let last_seg = group_segments.last().unwrap();
+        let group_byte_end = last_seg.byte_start + last_seg.text.len();
+
+        // Use the original text slice so byte offsets stay valid.
+        let group_text = &source_text[group_byte_start..group_byte_end];
+
+        if group_text.chars().count() <= max_characters {
+            let text = match (overlap > 0, &prev_group_tail) {
+                (true, Some(prev_tail)) => prepend_overlap(group_text, prev_tail, overlap, max_characters),
+                _ => group_text.to_string(),
+            };
+
+            result.push(MergedChunk {
+                text,
+                byte_start: group_byte_start,
+                byte_end: group_byte_end,
+            });
+        } else {
+            let splitter = TextSplitter::new(max_characters);
+            let sub_chunks: Vec<&str> = splitter.chunks(group_text).collect();
+            let num_sub = sub_chunks.len();
+            let span = group_byte_end.saturating_sub(group_byte_start);
+
+            for (idx, chunk_text) in sub_chunks.iter().enumerate() {
+                let approx_start = group_byte_start + (span as f64 * idx as f64 / num_sub as f64) as usize;
+                let approx_end = group_byte_start + (span as f64 * (idx + 1) as f64 / num_sub as f64) as usize;
+
+                result.push(MergedChunk {
+                    text: (*chunk_text).to_string(),
+                    byte_start: approx_start,
+                    byte_end: approx_end,
+                });
+            }
+        }
+
+        prev_group_tail = Some(group_text.to_string());
+    }
+
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn single_segment_under_budget() {
+        let source = "hello world";
+        let segments = [Segment {
+            text: source,
+            byte_start: 0,
+        }];
+        let boundaries = [true];
+        let chunks = merge_segments(source, &segments, &boundaries, 100, 0);
+        assert_eq!(chunks.len(), 1);
+        assert_eq!(chunks[0].text, "hello world");
+        assert_eq!(chunks[0].byte_start, 0);
+        assert_eq!(chunks[0].byte_end, 11);
+    }
+
+    #[test]
+    fn two_segments_same_topic_merged() {
+        // Source text with segments at known byte offsets.
+        let source = "first     second";
+        let segments = [
+            Segment {
+                text: &source[0..5], // "first"
+                byte_start: 0,
+            },
+            Segment {
+                text: &source[10..16], // "second"
+                byte_start: 10,
+            },
+        ];
+        let boundaries = [true, false];
+        let chunks = merge_segments(source, &segments, &boundaries, 200, 0);
+        assert_eq!(chunks.len(), 1);
+        // The merged text is the original slice source[0..16].
+        assert_eq!(chunks[0].text, "first     second");
+        assert_eq!(chunks[0].byte_start, 0);
+        assert_eq!(chunks[0].byte_end, 16);
+    }
+
+    #[test]
+    fn two_segments_different_topics() {
+        let source = "topic one           topic two";
+        let segments = [
+            Segment {
+                text: &source[0..9], // "topic one"
+                byte_start: 0,
+            },
+            Segment {
+                text: &source[20..29], // "topic two"
+                byte_start: 20,
+            },
+        ];
+        let boundaries = [true, true];
+        let chunks = merge_segments(source, &segments, &boundaries, 200, 0);
+        assert_eq!(chunks.len(), 2);
+        assert_eq!(chunks[0].text, "topic one");
+        assert_eq!(chunks[1].text, "topic two");
+    }
+
+    #[test]
+    fn oversized_group_falls_back_to_splitter() {
+        let long_text = "word ".repeat(100); // ~500 chars
+        let trimmed = long_text.trim_end();
+        let segments = [Segment {
+            text: trimmed,
+            byte_start: 0,
+        }];
+        let boundaries = [true];
+        let max_chars = 50;
+        let chunks = merge_segments(trimmed, &segments, &boundaries, max_chars, 0);
+        assert!(chunks.len() > 1, "should split into multiple chunks");
+        for chunk in &chunks {
+            assert!(
+                chunk.text.len() <= max_chars,
+                "chunk exceeds budget: {} > {}",
+                chunk.text.len(),
+                max_chars
+            );
+        }
+    }
+
+    #[test]
+    fn overlap_at_topic_boundary() {
+        let source = "abcdefghij          klmnop";
+        let segments = [
+            Segment {
+                text: &source[0..10], // "abcdefghij"
+                byte_start: 0,
+            },
+            Segment {
+                text: &source[20..26], // "klmnop"
+                byte_start: 20,
+            },
+        ];
+        let boundaries = [true, true];
+        let chunks = merge_segments(source, &segments, &boundaries, 200, 5);
+        assert_eq!(chunks.len(), 2);
+        // First chunk has no overlap (no predecessor).
+        assert_eq!(chunks[0].text, "abcdefghij");
+        // Second chunk should start with tail of previous group's text.
+        assert!(
+            chunks[1].text.starts_with("fghij"),
+            "expected overlap prefix, got: {:?}",
+            chunks[1].text
+        );
+        assert!(chunks[1].text.contains("klmnop"));
+    }
+
+    #[test]
+    fn empty_segments() {
+        let chunks = merge_segments("", &[], &[], 100, 0);
+        assert!(chunks.is_empty());
+    }
+
+    #[test]
+    fn multiple_groups_with_merge() {
+        let source = "a1   a2   b1   b2   c1";
+        let segments = [
+            Segment {
+                text: &source[0..2], // "a1"
+                byte_start: 0,
+            },
+            Segment {
+                text: &source[5..7], // "a2"
+                byte_start: 5,
+            },
+            Segment {
+                text: &source[10..12], // "b1"
+                byte_start: 10,
+            },
+            Segment {
+                text: &source[15..17], // "b2"
+                byte_start: 15,
+            },
+            Segment {
+                text: &source[20..22], // "c1"
+                byte_start: 20,
+            },
+        ];
+        // Group A: [a1, a2], Group B: [b1, b2], Group C: [c1]
+        let boundaries = [true, false, true, false, true];
+        let chunks = merge_segments(source, &segments, &boundaries, 200, 0);
+        assert_eq!(chunks.len(), 3);
+        // Merged text is original slice source[0..7], source[10..17], source[20..22]
+        assert_eq!(chunks[0].text, "a1   a2");
+        assert_eq!(chunks[1].text, "b1   b2");
+        assert_eq!(chunks[2].text, "c1");
+    }
+
+    #[test]
+    fn tail_chars_basic() {
+        assert_eq!(tail_chars("hello", 3), "llo");
+        assert_eq!(tail_chars("hello", 10), "hello");
+        assert_eq!(tail_chars("hello", 0), "");
+    }
+
+    #[test]
+    fn tail_chars_unicode() {
+        // Each emoji is one char but multiple bytes.
+        let s = "abc🦀🐍";
+        assert_eq!(tail_chars(s, 2), "🦀🐍");
+        assert_eq!(tail_chars(s, 5), s);
+    }
+
+    #[test]
+    fn tail_chars_utf8_multibyte() {
+        // "café" — 'é' is a multi-byte char; must not split mid-char.
+        assert_eq!(tail_chars("café", 2), "fé");
+    }
+
+    #[test]
+    fn tail_chars_zero() {
+        assert_eq!(tail_chars("hello", 0), "");
+        assert_eq!(tail_chars("café", 0), "");
+    }
+
+    #[test]
+    fn tail_chars_exceeds_length() {
+        assert_eq!(tail_chars("hi", 10), "hi");
+        assert_eq!(tail_chars("café", 100), "café");
+    }
+
+    #[test]
+    fn single_segment_exceeds_max_characters() {
+        // A single segment larger than max_characters should be split by TextSplitter.
+        let big = "word ".repeat(200); // ~1000 chars
+        let trimmed = big.trim_end();
+        let segments = [Segment {
+            text: trimmed,
+            byte_start: 0,
+        }];
+        let boundaries = [true];
+        let max = 80;
+        let chunks = merge_segments(trimmed, &segments, &boundaries, max, 0);
+        assert!(chunks.len() > 1, "oversized single segment must be split");
+        for chunk in &chunks {
+            assert!(
+                chunk.text.len() <= max,
+                "chunk exceeds budget: {} > {}",
+                chunk.text.len(),
+                max
+            );
+        }
+    }
+
+    #[test]
+    fn three_groups_middle_oversized() {
+        let small_a = "Alpha content";
+        let big_b = "word ".repeat(200); // ~1000 chars — oversized
+        let big_b_trimmed = big_b.trim_end();
+        let small_c = "Gamma content";
+
+        // Build a source text that contains all segments at their byte offsets.
+        let mut source = String::new();
+        source.push_str(small_a); // 0..13
+        source.push_str(&" ".repeat(100 - small_a.len())); // pad to offset 100
+        source.push_str(big_b_trimmed); // 100..100+len
+        let pad_to = 1200usize.saturating_sub(source.len());
+        source.push_str(&" ".repeat(pad_to)); // pad to offset 1200
+        source.push_str(small_c); // 1200..1213
+
+        let segments = [
+            Segment {
+                text: &source[0..small_a.len()],
+                byte_start: 0,
+            },
+            Segment {
+                text: &source[100..100 + big_b_trimmed.len()],
+                byte_start: 100,
+            },
+            Segment {
+                text: &source[1200..1200 + small_c.len()],
+                byte_start: 1200,
+            },
+        ];
+        let boundaries = [true, true, true]; // each segment is its own group
+        let max = 80;
+        let chunks = merge_segments(&source, &segments, &boundaries, max, 0);
+
+        // First chunk fits as-is.
+        assert_eq!(chunks[0].text, small_a);
+        // Last chunk fits as-is.
+        assert_eq!(chunks.last().unwrap().text, small_c);
+        // Middle group must have been split into multiple chunks.
+        let middle_chunks: Vec<_> = chunks[1..chunks.len() - 1].to_vec();
+        assert!(
+            middle_chunks.len() > 1,
+            "oversized middle group should produce multiple chunks"
+        );
+        for mc in &middle_chunks {
+            assert!(mc.text.len() <= max, "middle chunk exceeds budget");
+        }
+    }
+
+    #[test]
+    fn merge_with_overlap_and_oversized_group() {
+        // Group A: small, Group B: oversized, Group C: small.
+        // Overlap is enabled — verify overlap is applied correctly alongside splitting.
+        let small_a = "alpha text here";
+        let big_b = "word ".repeat(100); // ~500 chars — oversized at max=60
+        let big_b_trimmed = big_b.trim_end();
+        let small_c = "gamma text";
+
+        // Build source text with segments at known offsets.
+        let mut source = String::new();
+        source.push_str(small_a); // 0..15
+        source.push_str(&" ".repeat(50 - small_a.len())); // pad to offset 50
+        source.push_str(big_b_trimmed); // 50..50+len
+        let pad_to = 600usize.saturating_sub(source.len());
+        source.push_str(&" ".repeat(pad_to)); // pad to offset 600
+        source.push_str(small_c); // 600..610
+
+        let segments = [
+            Segment {
+                text: &source[0..small_a.len()],
+                byte_start: 0,
+            },
+            Segment {
+                text: &source[50..50 + big_b_trimmed.len()],
+                byte_start: 50,
+            },
+            Segment {
+                text: &source[600..600 + small_c.len()],
+                byte_start: 600,
+            },
+        ];
+        let boundaries = [true, true, true];
+        let max = 60;
+        let overlap = 5;
+        let chunks = merge_segments(&source, &segments, &boundaries, max, overlap);
+
+        // First chunk: no overlap (no predecessor).
+        assert_eq!(chunks[0].text, small_a);
+
+        // Last chunk should contain overlap from the previous group's last segment.
+        let last = chunks.last().unwrap();
+        assert!(last.text.contains(small_c), "last chunk should contain its own text");
+
+        // Middle chunks (oversized group) should all respect the budget.
+        for chunk in &chunks[1..chunks.len() - 1] {
+            assert!(
+                chunk.text.len() <= max,
+                "middle chunk exceeds budget: {} > {}",
+                chunk.text.len(),
+                max
+            );
+        }
+    }
+
+    #[test]
+    fn tail_chars_cjk_characters() {
+        // CJK characters are multi-byte (3 bytes each in UTF-8).
+        let s = "hello\u{4e16}\u{754c}"; // "hello世界"
+        assert_eq!(tail_chars(s, 2), "\u{4e16}\u{754c}");
+        assert_eq!(tail_chars(s, 7), s);
+    }
+
+    #[test]
+    fn tail_chars_cafe_accent() {
+        // "cafe\u{0301}" — 'e' + combining acute accent = 2 chars but visually one.
+        // tail_chars works on chars, not grapheme clusters.
+        let s = "cafe\u{0301}";
+        // 5 chars: c, a, f, e, \u{0301}
+        assert_eq!(tail_chars(s, 3), "fe\u{0301}");
+    }
+}

crates/kreuzberg/src/chunking/semantic/mod.rs

@@ -0,0 +1,507 @@
+//! Semantic text chunking.
+//!
+//! Splits text into fine-grained segments, detects topic boundaries (optionally
+//! using embeddings), and merges segments into coherent chunks.
+
+pub mod merge;
+
+#[cfg(feature = "embeddings")]
+pub mod topic;
+
+use crate::chunking::boundaries::calculate_page_range;
+use crate::chunking::boundary_detection::detect_plain_text_boundaries;
+use crate::chunking::classifier::classify_chunk;
+use crate::chunking::config::{ChunkingConfig, ChunkingResult};
+use crate::chunking::headings::{build_heading_map, resolve_heading_context};
+use crate::chunking::text_splitter::{MarkdownSplitter, TextSplitter};
+use crate::error::Result;
+use crate::types::{Chunk, ChunkMetadata, PageBoundary};
+use merge::Segment;
+
+/// Default segment size (characters) for the initial fine-grained split.
+const SEGMENT_SIZE: usize = 200;
+
+/// Default cosine-similarity threshold for topic boundary detection.
+#[cfg(feature = "embeddings")]
+const DEFAULT_TOPIC_THRESHOLD: f32 = 0.75;
+
+/// Split text into semantically coherent chunks.
+///
+/// Splits text into fine-grained segments, detects structural (and optionally
+/// embedding-based) topic boundaries, then merges segments into chunks that
+/// respect those boundaries and the configured size budget.
+pub(crate) fn chunk_semantic(
+    text: &str,
+    config: &ChunkingConfig,
+    page_boundaries: Option<&[PageBoundary]>,
+) -> Result<ChunkingResult> {
+    if text.is_empty() {
+        return Ok(ChunkingResult {
+            chunks: vec![],
+            chunk_count: 0,
+        });
+    }
+
+    warn_if_fallback_path(config);
+
+    let seg_size = SEGMENT_SIZE;
+    let has_markdown_headers = text.lines().any(crate::utils::markdown_utils::is_markdown_header);
+    let splitter_segments: Vec<&str> = if has_markdown_headers {
+        let splitter = MarkdownSplitter::new(seg_size);
+        splitter.chunks(text).collect()
+    } else {
+        let splitter = TextSplitter::new(seg_size);
+        splitter.chunks(text).collect()
+    };
+
+    if splitter_segments.is_empty() {
+        return Ok(ChunkingResult {
+            chunks: vec![],
+            chunk_count: 0,
+        });
+    }
+
+    let source_start = text.as_ptr() as usize;
+    let segments: Vec<Segment<'_>> = splitter_segments
+        .iter()
+        .map(|&s| {
+            let byte_start = s.as_ptr() as usize - source_start;
+            debug_assert!(
+                byte_start + s.len() <= text.len(),
+                "text_splitter segment is not a subslice of the input"
+            );
+            Segment { text: s, byte_start }
+        })
+        .collect();
+
+    let detected = detect_plain_text_boundaries(text);
+    let mut forced: Vec<bool> = vec![false; segments.len()];
+    forced[0] = true;
+
+    // Both detected boundaries and segments are sorted by byte offset.
+    // Use a two-pointer merge for O(n+m) instead of O(n*m).
+    let mut seg_idx = 0;
+    for boundary in &detected {
+        while seg_idx < segments.len()
+            && segments[seg_idx].byte_start + segments[seg_idx].text.len() <= boundary.byte_offset
+        {
+            seg_idx += 1;
+        }
+        if seg_idx < segments.len() {
+            forced[seg_idx] = true;
+        }
+    }
+
+    // text_splitter returns subslices of the input text. Verify this invariant.
+    for seg in &segments {
+        debug_assert!(
+            seg.byte_start + seg.text.len() <= text.len(),
+            "segment byte range exceeds source text length"
+        );
+    }
+
+    let boundaries = compute_boundaries(&segments, &forced, config)?;
+
+    let ceiling = resolve_ceiling(config);
+    let merged = merge::merge_segments(text, &segments, &boundaries, ceiling, config.overlap);
+
+    let heading_map = build_heading_map(text);
+    let total_chunks = merged.len();
+    let mut chunks = Vec::with_capacity(total_chunks);
+
+    for (index, mc) in merged.into_iter().enumerate() {
+        let heading_ctx = resolve_heading_context(mc.byte_start, &heading_map);
+        let chunk_type = classify_chunk(&mc.text, heading_ctx.as_ref());
+
+        let (first_page, last_page) = if let Some(pb) = page_boundaries {
+            calculate_page_range(mc.byte_start, mc.byte_end, pb).unwrap_or((None, None))
+        } else {
+            (None, None)
+        };
+
+        chunks.push(Chunk {
+            content: mc.text,
+            chunk_type,
+            embedding: None,
+            metadata: ChunkMetadata {
+                byte_start: mc.byte_start,
+                byte_end: mc.byte_end,
+                token_count: None,
+                chunk_index: index,
+                total_chunks,
+                first_page,
+                last_page,
+                heading_context: heading_ctx,
+                image_indices: Vec::new(),
+            },
+        });
+    }
+
+    Ok(ChunkingResult {
+        chunk_count: chunks.len(),
+        chunks,
+    })
+}
+
+/// Compute final boundary vector, incorporating embeddings when available.
+#[cfg(feature = "embeddings")]
+fn compute_boundaries(segments: &[Segment<'_>], forced: &[bool], config: &ChunkingConfig) -> Result<Vec<bool>> {
+    if let Some(ref embedding_config) = config.embedding {
+        let segment_texts: Vec<&str> = segments.iter().map(|s| s.text).collect();
+        let threshold = config
+            .topic_threshold
+            .unwrap_or(DEFAULT_TOPIC_THRESHOLD)
+            .clamp(0.0, 1.0);
+        topic::detect_topic_boundaries(&segment_texts, forced, embedding_config, threshold)
+    } else {
+        Ok(forced.to_vec())
+    }
+}
+
+/// Compute final boundary vector (no embeddings available).
+#[cfg(not(feature = "embeddings"))]
+fn compute_boundaries(_segments: &[Segment<'_>], forced: &[bool], _config: &ChunkingConfig) -> Result<Vec<bool>> {
+    Ok(forced.to_vec())
+}
+
+/// Warn when the semantic chunker is invoked without an embedding model.
+///
+/// Without an embedding, `chunk_semantic` falls back to a structural-boundary
+/// heuristic (ALL-CAPS headers, numbered sections, blank-line paragraphs).
+/// Topic-similarity chunking requires an embedding model. This warning makes
+/// the fallback mode discoverable to callers who think they're getting
+/// embedding-driven topic detection.
+#[cfg(feature = "embeddings")]
+fn warn_if_fallback_path(config: &ChunkingConfig) {
+    if config.embedding.is_none() {
+        tracing::warn!(
+            "chunker_type='semantic' without an EmbeddingConfig falls back to a \
+             structural-boundary heuristic; topic-similarity chunking requires an \
+             embedding model. Either configure `embedding` or switch to \
+             chunker_type='text'/'markdown' to silence this warning."
+        );
+    }
+}
+
+#[cfg(not(feature = "embeddings"))]
+fn warn_if_fallback_path(_config: &ChunkingConfig) {}
+
+/// Resolve the size ceiling for merged chunks.
+///
+/// When an embedding preset is configured, use its `chunk_size` so chunks fit
+/// in the model's context window. Otherwise honor the caller's configured
+/// `max_characters`.
+fn resolve_ceiling(config: &ChunkingConfig) -> usize {
+    #[cfg(feature = "embeddings")]
+    if let Some(ref emb) = config.embedding
+        && let crate::EmbeddingModelType::Preset { ref name } = emb.model
+        && let Some(size) = crate::embeddings::preset_chunk_size(name)
+    {
+        return size;
+    }
+    config.max_characters
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::chunking::config::ChunkerType;
+
+    #[test]
+    fn chunk_semantic_empty_text() {
+        let config = ChunkingConfig {
+            max_characters: 500,
+            overlap: 0,
+            trim: true,
+            chunker_type: ChunkerType::Semantic,
+            ..Default::default()
+        };
+        let result = chunk_semantic("", &config, None).unwrap();
+        assert_eq!(result.chunks.len(), 0);
+        assert_eq!(result.chunk_count, 0);
+    }
+
+    #[test]
+    fn chunk_semantic_short_text() {
+        let config = ChunkingConfig {
+            max_characters: 500,
+            overlap: 0,
+            trim: true,
+            chunker_type: ChunkerType::Semantic,
+            ..Default::default()
+        };
+        let result = chunk_semantic("Hello world", &config, None).unwrap();
+        assert_eq!(result.chunks.len(), 1);
+        assert_eq!(result.chunks[0].content, "Hello world");
+    }
+
+    #[test]
+    fn chunk_semantic_multi_paragraph_merges() {
+        // No headers, no embeddings → all segments share one group → 1 chunk
+        let text = "First paragraph about cats.\n\nSecond paragraph about cats.\n\nThird paragraph about cats.";
+        let config = ChunkingConfig {
+            max_characters: 2000,
+            overlap: 0,
+            trim: true,
+            chunker_type: ChunkerType::Semantic,
+            ..Default::default()
+        };
+        let result = chunk_semantic(text, &config, None).unwrap();
+        assert_eq!(result.chunks.len(), 1, "all segments should merge into one chunk");
+    }
+
+    #[test]
+    fn chunk_semantic_all_caps_headers_force_boundaries() {
+        // Each section must be long enough to produce separate segments (SEGMENT_SIZE = 200).
+        let intro_body = "This is the introduction. ".repeat(12); // ~300 chars
+        let method_body = "Here we describe the methodology used. ".repeat(10); // ~390 chars
+        let results_body = "The results show improvements. ".repeat(10); // ~300 chars
+        let text = format!("INTRODUCTION\n\n{intro_body}\n\nMETHODOLOGY\n\n{method_body}\n\nRESULTS\n\n{results_body}");
+        let config = ChunkingConfig {
+            max_characters: 2000,
+            overlap: 0,
+            trim: true,
+            chunker_type: ChunkerType::Semantic,
+            ..Default::default()
+        };
+        let result = chunk_semantic(&text, &config, None).unwrap();
+        assert!(
+            result.chunks.len() >= 2,
+            "ALL CAPS headers should force boundaries, got {} chunks",
+            result.chunks.len()
+        );
+    }
+
+    #[test]
+    fn chunk_semantic_markdown_uses_markdown_splitter() {
+        // Content with ATX headings should trigger the MarkdownSplitter path.
+        let text = "# Introduction\n\nThis is the intro paragraph.\n\n## Details\n\nMore detail here.";
+        let config = ChunkingConfig {
+            max_characters: 2000,
+            overlap: 0,
+            trim: true,
+            chunker_type: ChunkerType::Semantic,
+            ..Default::default()
+        };
+        let result = chunk_semantic(text, &config, None).unwrap();
+        assert!(!result.chunks.is_empty(), "markdown content should produce chunks");
+        // The combined content should cover the full text.
+        let combined: String = result
+            .chunks
+            .iter()
+            .map(|c| c.content.as_str())
+            .collect::<Vec<_>>()
+            .join("");
+        assert!(combined.contains("Introduction"));
+        assert!(combined.contains("Details"));
+    }
+
+    #[test]
+    fn chunk_semantic_overlap_between_topic_groups() {
+        // Build text with two clear ALL-CAPS sections so boundaries are forced.
+        let body_a = "Alpha paragraph content. ".repeat(15); // ~375 chars
+        let body_b = "Beta paragraph content. ".repeat(15);
+        let text = format!("SECTION ONE\n\n{body_a}\n\nSECTION TWO\n\n{body_b}");
+        let config = ChunkingConfig {
+            max_characters: 2000,
+            overlap: 10,
+            trim: true,
+            chunker_type: ChunkerType::Semantic,
+            ..Default::default()
+        };
+        let result = chunk_semantic(&text, &config, None).unwrap();
+        assert!(
+            result.chunks.len() >= 2,
+            "should produce at least 2 chunks from 2 sections, got {}",
+            result.chunks.len()
+        );
+        // The second chunk should start with overlap characters from the previous group.
+        // We cannot predict exact overlap text, but the second chunk should contain
+        // content from SECTION TWO.
+        let second = &result.chunks[1].content;
+        assert!(
+            second.contains("SECTION TWO") || second.contains("Beta"),
+            "second chunk should contain second section content"
+        );
+    }
+
+    #[test]
+    fn max_characters_caps_oversized_headerless_text() {
+        // A large block of text with no headers must be split so every chunk
+        // respects the caller's configured max_characters.
+        let text = "word ".repeat(1500); // ~7500 chars
+        let max = 1000;
+        let config = ChunkingConfig {
+            max_characters: max,
+            overlap: 0,
+            trim: true,
+            chunker_type: ChunkerType::Semantic,
+            ..Default::default()
+        };
+        let result = chunk_semantic(&text, &config, None).unwrap();
+        assert!(result.chunks.len() >= 2, "should split at max_characters, got 1 chunk");
+        for (i, chunk) in result.chunks.iter().enumerate() {
+            assert!(
+                chunk.content.chars().count() <= max,
+                "chunk {} exceeds max_characters: {} > {}",
+                i,
+                chunk.content.chars().count(),
+                max
+            );
+        }
+    }
+
+    #[test]
+    fn max_characters_controls_fallback_chunk_size() {
+        // bb-yq35 repro: with no embedding configured, different max_characters
+        // values must produce different chunking output.
+        let sample = format!(
+            "{}{}{}",
+            "Solar panel efficiency improves. ".repeat(200),
+            "\n\nFDA clinical trials require double-blind. ".repeat(200),
+            "\n\nQuantum entanglement needs cooling. ".repeat(200),
+        );
+
+        let run = |max: usize| {
+            let config = ChunkingConfig {
+                max_characters: max,
+                overlap: 0,
+                trim: true,
+                chunker_type: ChunkerType::Semantic,
+                ..Default::default()
+            };
+            chunk_semantic(&sample, &config, None).unwrap()
+        };
+
+        let small = run(500);
+        let large = run(1500);
+
+        assert!(
+            small.chunks.len() > large.chunks.len(),
+            "smaller max_characters must yield more chunks: small={}, large={}",
+            small.chunks.len(),
+            large.chunks.len()
+        );
+        for chunk in &small.chunks {
+            assert!(
+                chunk.content.chars().count() <= 500,
+                "small chunk exceeds cap: {}",
+                chunk.content.chars().count()
+            );
+        }
+        for chunk in &large.chunks {
+            assert!(
+                chunk.content.chars().count() <= 1500,
+                "large chunk exceeds cap: {}",
+                chunk.content.chars().count()
+            );
+        }
+    }
+
+    #[cfg(feature = "embeddings")]
+    #[test]
+    fn semantic_without_embedding_warns() {
+        use std::io::Write;
+        use std::sync::{Arc, Mutex};
+
+        #[derive(Clone, Default)]
+        struct Buf(Arc<Mutex<Vec<u8>>>);
+        impl Write for Buf {
+            fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+                self.0.lock().unwrap().extend_from_slice(buf);
+                Ok(buf.len())
+            }
+            fn flush(&mut self) -> std::io::Result<()> {
+                Ok(())
+            }
+        }
+        impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for Buf {
+            type Writer = Buf;
+            fn make_writer(&'a self) -> Self::Writer {
+                self.clone()
+            }
+        }
+
+        let buffer = Buf::default();
+        let subscriber = tracing_subscriber::fmt()
+            .with_writer(buffer.clone())
+            .with_max_level(tracing::Level::WARN)
+            .with_ansi(false)
+            .finish();
+
+        tracing::subscriber::with_default(subscriber, || {
+            let config = ChunkingConfig {
+                chunker_type: ChunkerType::Semantic,
+                ..Default::default()
+            };
+            let _ = chunk_semantic("hello world", &config, None).unwrap();
+        });
+
+        let captured = String::from_utf8(buffer.0.lock().unwrap().clone()).unwrap();
+        assert!(
+            captured.contains("without an EmbeddingConfig"),
+            "expected fallback warning in captured logs, got: {captured:?}"
+        );
+    }
+
+    #[test]
+    fn sections_with_headers_produce_separate_chunks() {
+        // Each section has enough content that the segments span multiple paragraphs.
+        // Headers force boundaries, so each section should be its own chunk.
+        let body = "Content paragraph with sufficient text. ".repeat(8); // ~320 chars
+        let text = format!("SECTION A\n\n{body}\n\nSECTION B\n\n{body}\n\nSECTION C\n\n{body}");
+        let config = ChunkingConfig {
+            overlap: 0,
+            trim: true,
+            chunker_type: ChunkerType::Semantic,
+            ..Default::default()
+        };
+        let result = chunk_semantic(&text, &config, None).unwrap();
+        assert!(
+            result.chunks.len() >= 3,
+            "3 sections with headers should produce >= 3 chunks, got {}",
+            result.chunks.len()
+        );
+    }
+
+    #[test]
+    fn single_short_paragraph_one_chunk() {
+        let text = "A short paragraph.";
+        let config = ChunkingConfig {
+            chunker_type: ChunkerType::Semantic,
+            ..Default::default()
+        };
+        let result = chunk_semantic(text, &config, None).unwrap();
+        assert_eq!(result.chunks.len(), 1);
+    }
+
+    #[test]
+    fn topic_boundaries_keep_sections_separate() {
+        // Multi-section document with substantial content per section.
+        let energy_body = "Solar panels have improved significantly over the past decade. ".repeat(6);
+        let health_body = "AI diagnostics advanced rapidly during clinical trials. ".repeat(6);
+        let quantum_body = "Qubits crossed the thousand mark in recent experiments. ".repeat(6);
+        let text = format!(
+            "RENEWABLE ENERGY\n\n{energy_body}\n\nHEALTHCARE\n\n{health_body}\n\nQUANTUM COMPUTING\n\n{quantum_body}"
+        );
+
+        let config = ChunkingConfig {
+            chunker_type: ChunkerType::Semantic,
+            ..Default::default()
+        };
+        let result = chunk_semantic(&text, &config, None).unwrap();
+
+        assert!(
+            result.chunks.len() >= 3,
+            "3 sections should produce >= 3 chunks, got {}",
+            result.chunks.len()
+        );
+
+        // Energy chunk shouldn't contain healthcare content.
+        let energy = result.chunks.iter().find(|c| c.content.contains("Solar")).unwrap();
+        assert!(
+            !energy.content.contains("diagnostics"),
+            "energy chunk contains healthcare content"
+        );
+    }
+}

crates/kreuzberg/src/chunking/semantic/topic.rs

@@ -0,0 +1,224 @@
+//! Topic boundary detection via embedding similarity.
+//!
+//! Compares adjacent segment embeddings to find points where the topic shifts
+//! beyond a configurable threshold, producing boundary markers for the merge step.
+
+/// Compute cosine similarity between two vectors.
+///
+/// Returns a value in `[-1.0, 1.0]`. If either vector has near-zero magnitude
+/// the function returns `0.0` rather than producing `NaN`.
+pub(crate) fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
+    debug_assert_eq!(a.len(), b.len(), "vectors must have equal length");
+
+    let mut dot = 0.0_f32;
+    let mut norm_a = 0.0_f32;
+    let mut norm_b = 0.0_f32;
+
+    for (&ai, &bi) in a.iter().zip(b.iter()) {
+        dot += ai * bi;
+        norm_a += ai * ai;
+        norm_b += bi * bi;
+    }
+
+    let denom = norm_a.sqrt() * norm_b.sqrt();
+    if denom < f32::EPSILON { 0.0 } else { dot / denom }
+}
+
+/// Detect topic boundaries across a sequence of text segments.
+///
+/// Embeds all segments in a single batch, then marks a boundary wherever the
+/// cosine similarity between consecutive embeddings drops below `threshold`.
+/// Pre-existing forced boundaries (e.g. from structural cues) are preserved.
+///
+/// # Arguments
+///
+/// * `segment_texts` — ordered slice of segment strings
+/// * `forced_boundaries` — per-segment flags; `true` means a boundary is already
+///   decided and should not be overridden
+/// * `embedding_config` — model and batch-size configuration forwarded to
+///   [`crate::embeddings::embed_texts`]
+/// * `threshold` — similarity below this value triggers a new boundary
+///
+/// # Returns
+///
+/// A `Vec<bool>` of the same length as `segment_texts` where `true` marks the
+/// start of a new topic group.
+pub(crate) fn detect_topic_boundaries(
+    segment_texts: &[&str],
+    forced_boundaries: &[bool],
+    embedding_config: &crate::EmbeddingConfig,
+    threshold: f32,
+) -> crate::error::Result<Vec<bool>> {
+    let n = segment_texts.len();
+    if n == 0 {
+        return Ok(Vec::new());
+    }
+
+    let mut boundaries = vec![false; n];
+    boundaries[0] = true;
+
+    for (i, &forced) in forced_boundaries.iter().enumerate().take(n) {
+        if forced {
+            boundaries[i] = true;
+        }
+    }
+
+    if n < 2 {
+        return Ok(boundaries);
+    }
+
+    let embeddings = crate::embeddings::embed_texts(segment_texts, embedding_config)?;
+
+    if embeddings.len() != n {
+        return Err(crate::KreuzbergError::validation(format!(
+            "expected {} embeddings, got {}",
+            n,
+            embeddings.len()
+        )));
+    }
+
+    for i in 1..n {
+        if boundaries[i] {
+            continue;
+        }
+        let sim = cosine_similarity(&embeddings[i - 1], &embeddings[i]);
+        if sim < threshold {
+            boundaries[i] = true;
+        }
+    }
+
+    Ok(boundaries)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn cosine_similarity_identical() {
+        let v = vec![1.0, 2.0, 3.0];
+        let sim = cosine_similarity(&v, &v);
+        assert!(
+            (sim - 1.0).abs() < 1e-6,
+            "identical vectors should have similarity ~1.0, got {sim}"
+        );
+    }
+
+    #[test]
+    fn cosine_similarity_orthogonal() {
+        let a = vec![1.0, 0.0, 0.0];
+        let b = vec![0.0, 1.0, 0.0];
+        let sim = cosine_similarity(&a, &b);
+        assert!(
+            sim.abs() < 1e-6,
+            "orthogonal vectors should have similarity ~0.0, got {sim}"
+        );
+    }
+
+    #[test]
+    fn cosine_similarity_opposite() {
+        let a = vec![1.0, 2.0, 3.0];
+        let b = vec![-1.0, -2.0, -3.0];
+        let sim = cosine_similarity(&a, &b);
+        assert!(
+            (sim + 1.0).abs() < 1e-6,
+            "opposite vectors should have similarity ~-1.0, got {sim}"
+        );
+    }
+
+    #[test]
+    fn cosine_similarity_normalized() {
+        // For unit vectors, cosine similarity equals the dot product.
+        let norm = (1.0_f32 * 1.0 + 2.0 * 2.0 + 3.0 * 3.0).sqrt();
+        let a: Vec<f32> = vec![1.0 / norm, 2.0 / norm, 3.0 / norm];
+
+        let norm2 = (4.0_f32 * 4.0 + 5.0 * 5.0 + 6.0 * 6.0).sqrt();
+        let b: Vec<f32> = vec![4.0 / norm2, 5.0 / norm2, 6.0 / norm2];
+
+        let dot: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
+        let sim = cosine_similarity(&a, &b);
+        assert!(
+            (sim - dot).abs() < 1e-6,
+            "for unit vectors cosine_similarity should equal dot product: sim={sim}, dot={dot}"
+        );
+    }
+
+    #[test]
+    fn cosine_similarity_zero_vector() {
+        let a = vec![0.0, 0.0, 0.0];
+        let b = vec![1.0, 2.0, 3.0];
+        let sim = cosine_similarity(&a, &b);
+        assert!(sim.abs() < 1e-6, "zero vector should yield similarity 0.0, got {sim}");
+    }
+
+    #[test]
+    fn cosine_similarity_large_vectors() {
+        // 100-dimensional vectors — verify correctness at scale.
+        let a: Vec<f32> = (0..100).map(|i| (i as f32).sin()).collect();
+        let b: Vec<f32> = (0..100).map(|i| (i as f32).cos()).collect();
+
+        let sim = cosine_similarity(&a, &b);
+        // Just verify it produces a valid result in [-1, 1].
+        assert!(
+            (-1.0..=1.0).contains(&sim),
+            "similarity should be in [-1, 1], got {sim}"
+        );
+
+        // Verify against manual computation.
+        let dot: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
+        let na: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
+        let nb: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
+        let expected = dot / (na * nb);
+        assert!(
+            (sim - expected).abs() < 1e-5,
+            "mismatch: sim={sim}, expected={expected}"
+        );
+    }
+
+    #[test]
+    fn cosine_similarity_very_small_values() {
+        // Values near zero — should not produce NaN or panic.
+        let a = vec![1e-20_f32, 1e-20, 1e-20];
+        let b = vec![1e-20_f32, 1e-20, 1e-20];
+        let sim = cosine_similarity(&a, &b);
+        // Norms are ~1.7e-20 each, product ~3e-40 which is above f32::EPSILON (~1.2e-7)?
+        // Actually 3e-40 < f32::EPSILON, so we expect the guard to return 0.0.
+        // That's fine — the important thing is no NaN/panic.
+        assert!(!sim.is_nan(), "should not be NaN for very small values");
+    }
+
+    #[test]
+    #[should_panic(expected = "vectors must have equal length")]
+    fn cosine_similarity_mismatched_lengths_panics() {
+        // In debug builds, mismatched vector lengths trigger a debug_assert.
+        let a = vec![1.0, 2.0, 3.0];
+        let b = vec![1.0, 2.0];
+        cosine_similarity(&a, &b);
+    }
+
+    #[test]
+    fn cosine_similarity_topic_shift_simulation() {
+        // Simulate embeddings where segments 0-1 are similar (same topic)
+        // and segment 2 is different (topic shift).
+        let seg0 = vec![1.0, 0.9, 0.1, 0.0];
+        let seg1 = vec![0.95, 0.85, 0.15, 0.05];
+        let seg2 = vec![0.1, 0.0, 0.9, 1.0]; // different topic
+
+        let sim_same = cosine_similarity(&seg0, &seg1);
+        let sim_shift = cosine_similarity(&seg1, &seg2);
+
+        assert!(
+            sim_same > 0.9,
+            "same-topic segments should have high similarity, got {sim_same}"
+        );
+        assert!(
+            sim_shift < 0.5,
+            "topic-shift segments should have low similarity, got {sim_shift}"
+        );
+
+        // With threshold 0.75, only the shift should trigger a boundary.
+        let threshold = 0.75;
+        assert!(sim_same >= threshold, "same-topic pair should be above threshold");
+        assert!(sim_shift < threshold, "topic-shift pair should be below threshold");
+    }
+}

crates/kreuzberg/src/chunking/text_splitter/chunk_size.rs

@@ -0,0 +1,704 @@
+//! Vendored from text-splitter v0.30.1 (MIT, © 2023 Benjamin Brandt). See ATTRIBUTIONS.md.
+
+use std::{
+    borrow::Cow,
+    cell::{Ref, RefMut},
+    cmp::Ordering,
+    fmt,
+    ops::{Deref, Range, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive},
+    rc::Rc,
+    sync::Arc,
+};
+
+use ahash::AHashMap;
+use itertools::Itertools;
+use thiserror::Error;
+
+mod characters;
+#[cfg(feature = "chunking-tokenizers")]
+mod huggingface;
+
+use super::trim::Trim;
+pub use characters::Characters;
+
+/// Indicates there was an error with the chunk capacity configuration.
+/// The `Display` implementation will provide a human-readable error message to
+/// help debug the issue that caused the error.
+#[derive(Error, Debug)]
+#[error(transparent)]
+pub struct ChunkCapacityError(#[from] ChunkCapacityErrorRepr);
+
+/// Private error and free to change across minor version of the crate.
+#[derive(Error, Debug)]
+enum ChunkCapacityErrorRepr {
+    #[error("Max chunk size must be greater than or equal to the desired chunk size")]
+    MaxLessThanDesired,
+}
+
+/// Describes the valid chunk size(s) that can be generated.
+///
+/// The `desired` size is the target size for the chunk. In most cases, this
+/// will also serve as the maximum size of the chunk. It is always possible
+/// that a chunk may be returned that is less than the `desired` value, as
+/// adding the next piece of text may have made it larger than the `desired`
+/// capacity.
+///
+/// The `max` size is the maximum possible chunk size that can be generated.
+/// By setting this to a larger value than `desired`, it means that the chunk
+/// should be as close to `desired` as possible, but can be larger if it means
+/// staying at a larger semantic level.
+///
+/// The splitter will consume text until at maximum somewhere between `desired`
+/// and `max`, if they differ, but never above `max`.
+///
+/// If you need to ensure a fixed size, set `desired` and `max` to the same
+/// value. For example, if you are trying to maximize the context window for an
+/// embedding.
+///
+/// If you are loosely targeting a size, but have some extra room, for example
+/// in a RAG use case where you roughly want a certain part of a document, you
+/// can set `max` to your absolute maximum, and the splitter can stay at a
+/// higher semantic level when determining the chunk.
+#[derive(Copy, Clone, Debug, PartialEq)]
+pub struct ChunkCapacity {
+    pub(crate) desired: usize,
+    pub(crate) max: usize,
+}
+
+impl ChunkCapacity {
+    /// Create a new `ChunkCapacity` with the same `desired` and `max` size.
+    #[must_use]
+    pub fn new(size: usize) -> Self {
+        Self {
+            desired: size,
+            max: size,
+        }
+    }
+
+    /// If you need to ensure a fixed size, set `desired` and `max` to the same
+    /// value. For example, if you are trying to maximize the context window for an
+    /// embedding.
+    ///
+    /// If you are loosely targeting a size, but have some extra room, for example
+    /// in a RAG use case where you roughly want a certain part of a document, you
+    /// can set `max` to your absolute maximum, and the splitter can stay at a
+    /// higher semantic level when determining the chunk.
+    ///
+    /// # Errors
+    ///
+    /// If the `max` size is less than the `desired` size, an error is returned.
+    pub fn with_max(mut self, max: usize) -> Result<Self, ChunkCapacityError> {
+        if max < self.desired {
+            Err(ChunkCapacityError(ChunkCapacityErrorRepr::MaxLessThanDesired))
+        } else {
+            self.max = max;
+            Ok(self)
+        }
+    }
+
+    /// Validate if a given chunk fits within the capacity
+    ///
+    /// - `Ordering::Less` indicates more could be added
+    /// - `Ordering::Equal` indicates the chunk is within the capacity range
+    /// - `Ordering::Greater` indicates the chunk is larger than the capacity
+    #[must_use]
+    pub fn fits(&self, chunk_size: usize) -> Ordering {
+        if chunk_size < self.desired {
+            Ordering::Less
+        } else if chunk_size > self.max {
+            Ordering::Greater
+        } else {
+            Ordering::Equal
+        }
+    }
+}
+
+impl From<usize> for ChunkCapacity {
+    fn from(size: usize) -> Self {
+        ChunkCapacity::new(size)
+    }
+}
+
+impl From<Range<usize>> for ChunkCapacity {
+    fn from(range: Range<usize>) -> Self {
+        ChunkCapacity::new(range.start)
+            .with_max(range.end.saturating_sub(1).max(range.start))
+            .expect("invalid range")
+    }
+}
+
+impl From<RangeFrom<usize>> for ChunkCapacity {
+    fn from(range: RangeFrom<usize>) -> Self {
+        ChunkCapacity::new(range.start)
+            .with_max(usize::MAX)
+            .expect("invalid range")
+    }
+}
+
+impl From<RangeFull> for ChunkCapacity {
+    fn from(_: RangeFull) -> Self {
+        ChunkCapacity::new(usize::MIN)
+            .with_max(usize::MAX)
+            .expect("invalid range")
+    }
+}
+
+impl From<RangeInclusive<usize>> for ChunkCapacity {
+    fn from(range: RangeInclusive<usize>) -> Self {
+        ChunkCapacity::new(*range.start())
+            .with_max(*range.end())
+            .expect("invalid range")
+    }
+}
+
+impl From<RangeTo<usize>> for ChunkCapacity {
+    fn from(range: RangeTo<usize>) -> Self {
+        ChunkCapacity::new(usize::MIN)
+            .with_max(range.end.saturating_sub(1))
+            .expect("invalid range")
+    }
+}
+
+impl From<RangeToInclusive<usize>> for ChunkCapacity {
+    fn from(range: RangeToInclusive<usize>) -> Self {
+        ChunkCapacity::new(usize::MIN)
+            .with_max(range.end)
+            .expect("invalid range")
+    }
+}
+
+/// Determines the size of a given chunk.
+pub trait ChunkSizer {
+    /// Determine the size of a given chunk to use for validation
+    fn size(&self, chunk: &str) -> usize;
+}
+
+impl<T> ChunkSizer for &T
+where
+    T: ChunkSizer,
+{
+    fn size(&self, chunk: &str) -> usize {
+        (*self).size(chunk)
+    }
+}
+
+impl<T> ChunkSizer for Ref<'_, T>
+where
+    T: ChunkSizer,
+{
+    fn size(&self, chunk: &str) -> usize {
+        self.deref().size(chunk)
+    }
+}
+
+impl<T> ChunkSizer for RefMut<'_, T>
+where
+    T: ChunkSizer,
+{
+    fn size(&self, chunk: &str) -> usize {
+        self.deref().size(chunk)
+    }
+}
+
+impl<T> ChunkSizer for Box<T>
+where
+    T: ChunkSizer,
+{
+    fn size(&self, chunk: &str) -> usize {
+        self.deref().size(chunk)
+    }
+}
+
+impl<T> ChunkSizer for Cow<'_, T>
+where
+    T: ChunkSizer + ToOwned + ?Sized,
+    <T as ToOwned>::Owned: ChunkSizer,
+{
+    fn size(&self, chunk: &str) -> usize {
+        self.as_ref().size(chunk)
+    }
+}
+
+impl<T> ChunkSizer for Rc<T>
+where
+    T: ChunkSizer,
+{
+    fn size(&self, chunk: &str) -> usize {
+        self.deref().size(chunk)
+    }
+}
+
+impl<T> ChunkSizer for Arc<T>
+where
+    T: ChunkSizer,
+{
+    fn size(&self, chunk: &str) -> usize {
+        self.as_ref().size(chunk)
+    }
+}
+
+/// Indicates there was an error with the chunk configuration.
+/// The `Display` implementation will provide a human-readable error message to
+/// help debug the issue that caused the error.
+#[derive(Error, Debug)]
+#[error(transparent)]
+pub struct ChunkConfigError(#[from] ChunkConfigErrorRepr);
+
+/// Private error and free to change across minor version of the crate.
+#[derive(Error, Debug)]
+enum ChunkConfigErrorRepr {
+    #[error("The overlap is larger than or equal to the desired chunk capacity")]
+    OverlapLargerThanCapacity,
+}
+
+/// Configuration for how chunks should be created
+#[derive(Debug)]
+pub struct ChunkConfig<Sizer>
+where
+    Sizer: ChunkSizer,
+{
+    /// The chunk capacity to use for filling chunks
+    pub(crate) capacity: ChunkCapacity,
+    /// The amount of overlap between chunks. Defaults to 0.
+    pub(crate) overlap: usize,
+    /// The chunk sizer to use for determining the size of each chunk
+    pub(crate) sizer: Sizer,
+    /// Whether whitespace will be trimmed from the beginning and end of each chunk
+    pub(crate) trim: bool,
+}
+
+impl ChunkConfig<Characters> {
+    /// Create a basic configuration for chunking with only the required value a chunk capacity.
+    ///
+    /// By default, chunk sizes will be calculated based on the number of characters in each chunk.
+    /// You can set a custom chunk sizer by calling [`Self::with_sizer`].
+    ///
+    /// By default, chunks will be trimmed. If you want to preserve whitespace,
+    /// call [`Self::with_trim`] and set it to `false`.
+    #[must_use]
+    pub fn new(capacity: impl Into<ChunkCapacity>) -> Self {
+        Self {
+            capacity: capacity.into(),
+            overlap: 0,
+            sizer: Characters,
+            trim: true,
+        }
+    }
+}
+
+impl<Sizer> ChunkConfig<Sizer>
+where
+    Sizer: ChunkSizer,
+{
+    /// Set the amount of overlap between chunks.
+    ///
+    /// # Errors
+    ///
+    /// Will return an error if the overlap is larger than or equal to the chunk capacity.
+    pub fn with_overlap(mut self, overlap: usize) -> Result<Self, ChunkConfigError> {
+        if overlap >= self.capacity.desired {
+            Err(ChunkConfigError(ChunkConfigErrorRepr::OverlapLargerThanCapacity))
+        } else {
+            self.overlap = overlap;
+            Ok(self)
+        }
+    }
+
+    /// Set a custom chunk sizer to use for determining the size of each chunk
+    ///
+    /// ```text
+    /// let config = ChunkConfig::new(512).with_sizer(Characters);
+    /// ```
+    #[cfg(feature = "chunking-tokenizers")]
+    #[must_use]
+    pub fn with_sizer<S: ChunkSizer>(self, sizer: S) -> ChunkConfig<S> {
+        ChunkConfig {
+            capacity: self.capacity,
+            overlap: self.overlap,
+            sizer,
+            trim: self.trim,
+        }
+    }
+
+    /// Specify whether chunks should have whitespace trimmed from the
+    /// beginning and end or not.
+    ///
+    /// If `false` (default), joining all chunks should return the original
+    /// string.
+    /// If `true`, all chunks will have whitespace removed from beginning and end.
+    ///
+    /// ```text
+    /// let config = ChunkConfig::new(512).with_trim(false);
+    /// ```
+    #[must_use]
+    pub fn with_trim(mut self, trim: bool) -> Self {
+        self.trim = trim;
+        self
+    }
+}
+
+impl<T> From<T> for ChunkConfig<Characters>
+where
+    T: Into<ChunkCapacity>,
+{
+    fn from(capacity: T) -> Self {
+        Self::new(capacity)
+    }
+}
+
+/// A memoized chunk sizer that caches the size of chunks.
+/// Very helpful when the same chunk is being validated multiple times, which
+/// happens often, and can be expensive to compute, such as with tokenizers.
+#[derive(Debug)]
+pub struct MemoizedChunkSizer<'sizer, Sizer>
+where
+    Sizer: ChunkSizer,
+{
+    /// Cache of chunk sizes per byte offset range for base capacity
+    size_cache: AHashMap<Range<usize>, usize>,
+    /// The sizer used for calculating chunk sizes
+    sizer: &'sizer Sizer,
+}
+
+impl<'sizer, Sizer> MemoizedChunkSizer<'sizer, Sizer>
+where
+    Sizer: ChunkSizer,
+{
+    /// Wrap any chunk sizer for memoization
+    pub fn new(sizer: &'sizer Sizer) -> Self {
+        Self {
+            size_cache: AHashMap::new(),
+            sizer,
+        }
+    }
+
+    /// Determine the size of a given chunk to use for validation,
+    /// returning a cached value if it exists, and storing the result if not.
+    pub fn chunk_size(&mut self, offset: usize, chunk: &str, trim: Trim) -> usize {
+        let (offset, chunk) = trim.trim(offset, chunk);
+        *self
+            .size_cache
+            .entry(offset..(offset + chunk.len()))
+            .or_insert_with(|| self.sizer.size(chunk))
+    }
+
+    /// Find the best level to start splitting the text
+    pub fn find_correct_level<'text, L: fmt::Debug>(
+        &mut self,
+        offset: usize,
+        capacity: &ChunkCapacity,
+        levels_with_first_chunk: impl Iterator<Item = (L, &'text str)>,
+        trim: Trim,
+    ) -> (Option<L>, Option<usize>) {
+        let mut semantic_level = None;
+        let mut max_offset = None;
+
+        // We assume that larger levels are also longer. We can skip lower levels if going to a higher level would result in a shorter text
+        let levels_with_first_chunk = levels_with_first_chunk.coalesce(|(a_level, a_str), (b_level, b_str)| {
+            if a_str.len() >= b_str.len() {
+                Ok((b_level, b_str))
+            } else {
+                Err(((a_level, a_str), (b_level, b_str)))
+            }
+        });
+
+        for (level, str) in levels_with_first_chunk {
+            // Skip tokenizing levels that we know are too small anyway.
+            let len = str.len();
+            if len > capacity.max {
+                let chunk_size = self.chunk_size(offset, str, trim);
+                let fits = capacity.fits(chunk_size);
+                // If this no longer fits, we use the level we are at.
+                if fits.is_gt() {
+                    max_offset = Some(offset + len);
+                    break;
+                }
+            }
+            // Otherwise break up the text with the next level
+            semantic_level = Some(level);
+        }
+
+        (semantic_level, max_offset)
+    }
+
+    /// Clear the cached values. Once we've moved the cursor,
+    /// we don't need to keep the old values around.
+    pub fn clear_cache(&mut self) {
+        self.size_cache.clear();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{
+        cell::RefCell,
+        sync::atomic::{self, AtomicUsize},
+    };
+
+    use super::super::trim::Trim;
+
+    use super::*;
+
+    #[test]
+    fn check_chunk_capacity() {
+        let chunk = "12345";
+
+        assert_eq!(ChunkCapacity::from(4).fits(Characters.size(chunk)), Ordering::Greater);
+        assert_eq!(ChunkCapacity::from(5).fits(Characters.size(chunk)), Ordering::Equal);
+        assert_eq!(ChunkCapacity::from(6).fits(Characters.size(chunk)), Ordering::Less);
+    }
+
+    #[test]
+    fn check_chunk_capacity_for_range() {
+        let chunk = "12345";
+
+        assert_eq!(
+            ChunkCapacity::from(0..0).fits(Characters.size(chunk)),
+            Ordering::Greater
+        );
+        assert_eq!(
+            ChunkCapacity::from(0..5).fits(Characters.size(chunk)),
+            Ordering::Greater
+        );
+        assert_eq!(ChunkCapacity::from(5..6).fits(Characters.size(chunk)), Ordering::Equal);
+        assert_eq!(ChunkCapacity::from(6..100).fits(Characters.size(chunk)), Ordering::Less);
+    }
+
+    #[test]
+    fn check_chunk_capacity_for_range_from() {
+        let chunk = "12345";
+
+        assert_eq!(ChunkCapacity::from(0..).fits(Characters.size(chunk)), Ordering::Equal);
+        assert_eq!(ChunkCapacity::from(5..).fits(Characters.size(chunk)), Ordering::Equal);
+        assert_eq!(ChunkCapacity::from(6..).fits(Characters.size(chunk)), Ordering::Less);
+    }
+
+    #[test]
+    fn check_chunk_capacity_for_range_full() {
+        let chunk = "12345";
+
+        assert_eq!(ChunkCapacity::from(..).fits(Characters.size(chunk)), Ordering::Equal);
+    }
+
+    #[test]
+    fn check_chunk_capacity_for_range_inclusive() {
+        let chunk = "12345";
+
+        assert_eq!(
+            ChunkCapacity::from(0..=4).fits(Characters.size(chunk)),
+            Ordering::Greater
+        );
+        assert_eq!(ChunkCapacity::from(5..=6).fits(Characters.size(chunk)), Ordering::Equal);
+        assert_eq!(ChunkCapacity::from(4..=5).fits(Characters.size(chunk)), Ordering::Equal);
+        assert_eq!(
+            ChunkCapacity::from(6..=100).fits(Characters.size(chunk)),
+            Ordering::Less
+        );
+    }
+
+    #[test]
+    fn check_chunk_capacity_for_range_to() {
+        let chunk = "12345";
+
+        assert_eq!(ChunkCapacity::from(..0).fits(Characters.size(chunk)), Ordering::Greater);
+        assert_eq!(ChunkCapacity::from(..5).fits(Characters.size(chunk)), Ordering::Greater);
+        assert_eq!(ChunkCapacity::from(..6).fits(Characters.size(chunk)), Ordering::Equal);
+    }
+
+    #[test]
+    fn check_chunk_capacity_for_range_to_inclusive() {
+        let chunk = "12345";
+
+        assert_eq!(
+            ChunkCapacity::from(..=4).fits(Characters.size(chunk)),
+            Ordering::Greater
+        );
+        assert_eq!(ChunkCapacity::from(..=5).fits(Characters.size(chunk)), Ordering::Equal);
+        assert_eq!(ChunkCapacity::from(..=6).fits(Characters.size(chunk)), Ordering::Equal);
+    }
+
+    #[derive(Default)]
+    struct CountingSizer {
+        calls: AtomicUsize,
+    }
+
+    impl ChunkSizer for CountingSizer {
+        // Return character version, but count calls
+        fn size(&self, chunk: &str) -> usize {
+            self.calls.fetch_add(1, atomic::Ordering::SeqCst);
+            Characters.size(chunk)
+        }
+    }
+
+    #[test]
+    fn memoized_sizer_only_calculates_once_per_text() {
+        let sizer = CountingSizer::default();
+        let mut memoized_sizer = MemoizedChunkSizer::new(&sizer);
+        let text = "1234567890";
+        for _ in 0..10 {
+            memoized_sizer.chunk_size(0, text, Trim::All);
+        }
+
+        assert_eq!(memoized_sizer.sizer.calls.load(atomic::Ordering::SeqCst), 1);
+    }
+
+    #[test]
+    fn memoized_sizer_calculates_once_per_different_text() {
+        let sizer = CountingSizer::default();
+        let mut memoized_sizer = MemoizedChunkSizer::new(&sizer);
+        let text = "1234567890";
+        for i in 0..10 {
+            memoized_sizer.chunk_size(0, text.get(0..i).unwrap(), Trim::All);
+        }
+
+        assert_eq!(memoized_sizer.sizer.calls.load(atomic::Ordering::SeqCst), 10);
+    }
+
+    #[test]
+    fn can_clear_cache_on_memoized_sizer() {
+        let sizer = CountingSizer::default();
+        let mut memoized_sizer = MemoizedChunkSizer::new(&sizer);
+        let text = "1234567890";
+        for _ in 0..10 {
+            memoized_sizer.chunk_size(0, text, Trim::All);
+            memoized_sizer.clear_cache();
+        }
+
+        assert_eq!(memoized_sizer.sizer.calls.load(atomic::Ordering::SeqCst), 10);
+    }
+
+    #[test]
+    fn basic_chunk_config() {
+        let config = ChunkConfig::new(10);
+        assert_eq!(config.capacity, 10.into());
+        assert_eq!(config.sizer, Characters);
+        assert!(config.trim);
+    }
+
+    #[test]
+    fn disable_trimming() {
+        let config = ChunkConfig::new(10).with_trim(false);
+        assert!(!config.trim);
+    }
+
+    #[test]
+    fn new_sizer() {
+        #[derive(Debug, PartialEq)]
+        struct BasicSizer;
+
+        impl ChunkSizer for BasicSizer {
+            fn size(&self, _chunk: &str) -> usize {
+                unimplemented!()
+            }
+        }
+
+        let config = ChunkConfig::new(10).with_sizer(BasicSizer);
+        assert_eq!(config.capacity, 10.into());
+        assert_eq!(config.sizer, BasicSizer);
+        assert!(config.trim);
+    }
+
+    #[test]
+    fn chunk_capacity_max_and_desired_equal() {
+        let capacity = ChunkCapacity::new(10);
+        assert_eq!(capacity.desired, 10);
+        assert_eq!(capacity.max, 10);
+    }
+
+    #[test]
+    fn chunk_capacity_can_adjust_max() {
+        let capacity = ChunkCapacity::new(10).with_max(20).unwrap();
+        assert_eq!(capacity.desired, 10);
+        assert_eq!(capacity.max, 20);
+    }
+
+    #[test]
+    fn chunk_capacity_max_cant_be_less_than_desired() {
+        let capacity = ChunkCapacity::new(10);
+        let err = capacity.with_max(5).unwrap_err();
+        assert_eq!(
+            err.to_string(),
+            "Max chunk size must be greater than or equal to the desired chunk size"
+        );
+        assert_eq!(capacity.desired, 10);
+        assert_eq!(capacity.max, 10);
+    }
+
+    #[test]
+    fn set_chunk_overlap() {
+        let config = ChunkConfig::new(10).with_overlap(5).unwrap();
+        assert_eq!(config.overlap, 5);
+    }
+
+    #[test]
+    fn cant_set_overlap_larger_than_capacity() {
+        let chunk_config = ChunkConfig::new(5);
+        let err = chunk_config.with_overlap(10).unwrap_err();
+        assert_eq!(
+            err.to_string(),
+            "The overlap is larger than or equal to the desired chunk capacity"
+        );
+    }
+
+    #[test]
+    fn cant_set_overlap_larger_than_desired() {
+        let chunk_config = ChunkConfig::new(5..15);
+        let err = chunk_config.with_overlap(10).unwrap_err();
+        assert_eq!(
+            err.to_string(),
+            "The overlap is larger than or equal to the desired chunk capacity"
+        );
+    }
+
+    #[test]
+    fn chunk_size_reference() {
+        let config = ChunkConfig::new(1).with_sizer(&Characters);
+        config.sizer.size("chunk");
+    }
+
+    #[test]
+    fn chunk_size_cow() {
+        let sizer: Cow<'_, Characters> = Cow::Owned(Characters);
+        let config = ChunkConfig::new(1).with_sizer(sizer);
+        config.sizer.size("chunk");
+
+        let sizer = Cow::Borrowed(&Characters);
+        let config = ChunkConfig::new(1).with_sizer(sizer);
+        config.sizer.size("chunk");
+    }
+
+    #[test]
+    fn chunk_size_arc() {
+        let sizer = Arc::new(Characters);
+        let config = ChunkConfig::new(1).with_sizer(sizer);
+        config.sizer.size("chunk");
+    }
+
+    #[test]
+    fn chunk_size_ref() {
+        let sizer = RefCell::new(Characters);
+        let config = ChunkConfig::new(1).with_sizer(sizer.borrow());
+        config.sizer.size("chunk");
+    }
+
+    #[test]
+    fn chunk_size_ref_mut() {
+        let sizer = RefCell::new(Characters);
+        let config = ChunkConfig::new(1).with_sizer(sizer.borrow_mut());
+        config.sizer.size("chunk");
+    }
+
+    #[test]
+    fn chunk_size_box() {
+        let sizer = Box::new(Characters);
+        let config = ChunkConfig::new(1).with_sizer(sizer);
+        config.sizer.size("chunk");
+    }
+
+    #[test]
+    fn chunk_size_rc() {
+        let sizer = Rc::new(Characters);
+        let config = ChunkConfig::new(1).with_sizer(sizer);
+        config.sizer.size("chunk");
+    }
+}

crates/kreuzberg/src/chunking/text_splitter/chunk_size/characters.rs

@@ -0,0 +1,30 @@
+//! Vendored from text-splitter v0.30.1 (MIT, © 2023 Benjamin Brandt). See ATTRIBUTIONS.md.
+
+use super::super::ChunkSizer;
+
+/// Used for splitting a piece of text into chunks based on the number of
+/// characters in each chunk.
+///
+/// ```text
+/// let splitter = TextSplitter::new(10);
+/// ```
+#[derive(Copy, Clone, Debug, PartialEq)]
+pub struct Characters;
+
+impl ChunkSizer for Characters {
+    /// Determine the size of a given chunk to use for validation.
+    fn size(&self, chunk: &str) -> usize {
+        chunk.chars().count()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn returns_size() {
+        let offsets = Characters.size("eé");
+        assert_eq!(offsets, 2);
+    }
+}

crates/kreuzberg/src/chunking/text_splitter/chunk_size/huggingface.rs

@@ -0,0 +1,90 @@
+//! Vendored from text-splitter v0.30.1 (MIT, © 2023 Benjamin Brandt). See ATTRIBUTIONS.md.
+
+use tokenizers::{Encoding, Tokenizer};
+
+use super::super::ChunkSizer;
+
+/// Compute the number of tokens that exist within an entire [`Encoding`] object.
+///
+/// Take into account [`Encoding::get_overflowing`] for cases where the [`Tokenizer`] producing the [`Encoding`] has truncation parameters set.
+fn num_tokens_with_overflow(encoding: &Encoding, pad_id: Option<u32>) -> usize {
+    let base = encoding
+        .get_ids()
+        .iter()
+        // Skip padding tokens at beginning and end so they don't count towards the chunk size
+        .skip_while(|&id| pad_id.is_some_and(|pad_id| id == &pad_id))
+        .take_while(|&id| pad_id.is_none_or(|pad_id| id != &pad_id))
+        .count();
+
+    // If the [`Tokenizer`] has truncation, need to check overflow encodings to determine overall size.
+    let overflow: usize = encoding
+        .get_overflowing()
+        .iter()
+        .map(|enc| num_tokens_with_overflow(enc, pad_id))
+        .sum();
+
+    base + overflow
+}
+
+impl ChunkSizer for Tokenizer {
+    /// Returns the number of tokens in a given text after tokenization.
+    ///
+    /// # Panics
+    ///
+    /// Will panic if you don't have a byte-level tokenizer and the splitter
+    /// encounters text it can't tokenize.
+    fn size(&self, chunk: &str) -> usize {
+        let encoding = self
+            .encode_fast(chunk, false)
+            .expect("Unable to tokenize the following string {chunk}");
+
+        let pad_id = self.get_padding().map(|params| params.pad_id);
+        num_tokens_with_overflow(&encoding, pad_id)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn returns_size() {
+        let tokenizer = Tokenizer::from_pretrained("bert-base-cased", None).unwrap();
+        let size = tokenizer.size(" An apple a");
+        assert_eq!(size, 3);
+    }
+
+    #[test]
+    fn returns_size_handles_prefix() {
+        // Use a tokenizer that adds padding tokens: thenlper/gte-small has a PAD token
+        // so pad_id is Some, and leading/trailing padding is skipped in the count.
+        // The text "An apple a" should still yield exactly 3 content tokens.
+        let tokenizer = Tokenizer::from_pretrained("thenlper/gte-small", None)
+            .expect("Could not load tokenizer 'thenlper/gte-small'");
+
+        let size = tokenizer.size("An apple a");
+        assert_eq!(size, 3);
+    }
+
+    #[test]
+    fn handles_padding() {
+        let tokenizer = Tokenizer::from_pretrained("thenlper/gte-small", None).unwrap();
+        let size = tokenizer.size("An apple a");
+        assert_eq!(size, 3);
+    }
+
+    #[test]
+    fn handle_truncation() {
+        let tokenizer = Tokenizer::from_pretrained("sentence-transformers/all-MiniLM-L6-v2", None)
+            .expect("Could not load tokenizer 'sentence-transformers/all-MiniLM-L6-v2'");
+
+        // When the tokenizer has truncation (max_length=128) but no stride, encode_fast
+        // returns a single encoding capped at 128 tokens with no overflow entries.
+        // The ChunkSizer returns 128 for any text longer than the model's max_length,
+        // which is sufficient to signal to the text splitter that the chunk is too large.
+        assert_eq!(
+            tokenizer.size("An apple a day keeps the doctor away.".repeat(100).as_str()),
+            128
+        );
+    }
+}

crates/kreuzberg/src/chunking/text_splitter/mod.rs

@@ -0,0 +1,23 @@
+//! Vendored from <https://github.com/benbrandt/text-splitter> @ v0.30.1
+//! MIT, © 2023 Benjamin Brandt. See `ATTRIBUTIONS.md` for full provenance.
+//!
+//! Trimmed: dropped `code` (tree-sitter) and `tiktoken-rs` features and their
+//! files; rebuilt against `tokenizers 0.23`. The `markdown` splitter is always
+//! compiled because `pulldown-cmark` is already a non-optional kreuzberg
+//! dependency.
+
+#![allow(
+    clippy::pedantic,
+    clippy::cargo,
+    clippy::missing_errors_doc,
+    clippy::missing_panics_doc,
+    missing_docs,
+    rust_2018_idioms
+)]
+
+mod chunk_size;
+mod splitter;
+mod trim;
+
+pub(crate) use chunk_size::{Characters, ChunkCapacity, ChunkConfig, ChunkSizer};
+pub(crate) use splitter::{MarkdownSplitter, TextSplitter};

crates/kreuzberg/src/chunking/text_splitter/splitter.rs

@@ -0,0 +1,623 @@
+//! Vendored from text-splitter v0.30.1 (MIT, © 2023 Benjamin Brandt). See ATTRIBUTIONS.md.
+
+use std::{cmp::Ordering, fmt, iter::once, ops::Range};
+
+use either::Either;
+use itertools::Itertools;
+use strum::IntoEnumIterator;
+
+use self::fallback::FallbackLevel;
+use super::{ChunkCapacity, ChunkConfig, ChunkSizer, chunk_size::MemoizedChunkSizer, trim::Trim};
+
+mod fallback;
+mod markdown;
+mod text;
+
+pub(crate) use markdown::MarkdownSplitter;
+pub(crate) use text::TextSplitter;
+
+/// Shared interface for splitters that can generate chunks of text based on the
+/// associated semantic level.
+trait Splitter<Sizer>
+where
+    Sizer: ChunkSizer,
+{
+    type Level: SemanticLevel;
+
+    /// Trimming behavior to use when trimming chunks
+    const TRIM: Trim = Trim::All;
+
+    /// Retrieve the splitter chunk configuration
+    fn chunk_config(&self) -> &ChunkConfig<Sizer>;
+
+    /// Generate a list of offsets for each semantic level within the text.
+    fn parse(&self, text: &str) -> Vec<(Self::Level, Range<usize>)>;
+
+    /// Returns an iterator over chunks of the text and their byte offsets.
+    /// Each chunk will be up to the max size of the `ChunkConfig`.
+    fn chunk_indices<'splitter, 'text: 'splitter>(
+        &'splitter self,
+        text: &'text str,
+    ) -> impl Iterator<Item = (usize, &'text str)> + 'splitter
+    where
+        Sizer: 'splitter,
+    {
+        TextChunks::<Sizer, Self::Level>::new(self.chunk_config(), text, self.parse(text), Self::TRIM)
+    }
+
+    /// Generate a list of chunks from a given text.
+    /// Each chunk will be up to the max size of the `ChunkConfig`.
+    fn chunks<'splitter, 'text: 'splitter>(
+        &'splitter self,
+        text: &'text str,
+    ) -> impl Iterator<Item = &'text str> + 'splitter
+    where
+        Sizer: 'splitter,
+    {
+        self.chunk_indices(text).map(|(_, t)| t)
+    }
+}
+
+/// Custom-defined levels of semantic splitting for custom document types.
+trait SemanticLevel: Copy + fmt::Debug + Ord + PartialOrd + 'static {
+    /// Given a level, split the text into sections based on the level.
+    /// Level ranges are also provided of items that are equal to or greater than the current level.
+    /// Default implementation assumes that all level ranges should be treated
+    /// as their own item.
+    fn sections(
+        text: &str,
+        level_ranges: impl Iterator<Item = (Self, Range<usize>)>,
+    ) -> impl Iterator<Item = (usize, &str)> {
+        let mut cursor = 0;
+        let mut final_match = false;
+        level_ranges
+            .batching(move |it| {
+                loop {
+                    match it.next() {
+                        // If we've hit the end, actually return None
+                        None if final_match => return None,
+                        // First time we hit None, return the final section of the text
+                        None => {
+                            final_match = true;
+                            return text.get(cursor..).map(|t| Either::Left(once((cursor, t))));
+                        }
+                        // Return text preceding match + the match
+                        Some((_, range)) => {
+                            if range.start < cursor {
+                                continue;
+                            }
+                            let offset = cursor;
+                            let prev_section = text.get(offset..range.start).expect("invalid character sequence");
+                            let separator = text.get(range.start..range.end).expect("invalid character sequence");
+                            cursor = range.end;
+                            return Some(Either::Right(
+                                [(offset, prev_section), (range.start, separator)].into_iter(),
+                            ));
+                        }
+                    }
+                }
+            })
+            .flatten()
+            .filter(|(_, s)| !s.is_empty())
+    }
+}
+
+/// Captures information about document structure for a given text, and their
+/// various semantic levels
+#[derive(Debug)]
+struct SemanticSplitRanges<Level>
+where
+    Level: SemanticLevel,
+{
+    /// Current cursor in the ranges list, so that we can skip over items we've
+    /// already processed.
+    cursor: usize,
+    /// Range of each semantic item and its precalculated semantic level
+    ranges: Vec<(Level, Range<usize>)>,
+}
+
+impl<Level> SemanticSplitRanges<Level>
+where
+    Level: SemanticLevel,
+{
+    fn new(mut ranges: Vec<(Level, Range<usize>)>) -> Self {
+        // Sort by start. If start is equal, sort by end in reverse order, so larger ranges come first.
+        ranges.sort_unstable_by(|(_, a), (_, b)| a.start.cmp(&b.start).then_with(|| b.end.cmp(&a.end)));
+        Self { cursor: 0, ranges }
+    }
+
+    /// Retrieve ranges for all sections of a given level after an offset
+    fn ranges_after_offset(&self, offset: usize) -> impl Iterator<Item = (Level, Range<usize>)> + '_ {
+        self.ranges[self.cursor..]
+            .iter()
+            .filter(move |(_, sep)| sep.start >= offset)
+            .map(|(l, r)| (*l, r.start..r.end))
+    }
+    /// Retrieve ranges for all sections of a given level after an offset
+    fn level_ranges_after_offset(
+        &self,
+        offset: usize,
+        level: Level,
+    ) -> impl Iterator<Item = (Level, Range<usize>)> + '_ {
+        // Find the first item of this level. Allows us to skip larger items of a higher level that surround this one.
+        // Otherwise all lower levels would only return the first item of the higher level that wraps it.
+        let first_item = self
+            .ranges_after_offset(offset)
+            .position(|(l, _)| l == level)
+            .and_then(|i| {
+                self.ranges_after_offset(offset)
+                    .skip(i)
+                    .coalesce(|(a_level, a_range), (b_level, b_range)| {
+                        // If we are at the first item, if two neighboring elements have the same level and start, take the shorter one
+                        if a_level == b_level && a_range.start == b_range.start && i == 0 {
+                            Ok((b_level, b_range))
+                        } else {
+                            Err(((a_level, a_range), (b_level, b_range)))
+                        }
+                    })
+                    // Just take the first of these items
+                    .next()
+            });
+        // let first_item = self.ranges_after_offset(offset).find(|(l, _)| l == &level);
+        self.ranges_after_offset(offset)
+            .filter(move |(l, _)| l >= &level)
+            .skip_while(move |(l, r)| {
+                first_item.as_ref().is_some_and(|(_, fir)| {
+                    (l > &level && r.contains(&fir.start)) || (l == &level && r.start == fir.start && r.end > fir.end)
+                })
+            })
+    }
+
+    /// Return a unique, sorted list of all line break levels present before the next max level, added
+    /// to all of the base semantic levels, in order from smallest to largest
+    fn levels_in_remaining_text(&self, offset: usize) -> impl Iterator<Item = Level> + '_ {
+        self.ranges_after_offset(offset).map(|(l, _)| l).sorted().dedup()
+    }
+
+    /// Split a given text into iterator over each semantic chunk
+    fn semantic_chunks<'splitter, 'text: 'splitter>(
+        &'splitter self,
+        offset: usize,
+        text: &'text str,
+        semantic_level: Level,
+    ) -> impl Iterator<Item = (usize, &'text str)> + 'splitter {
+        Level::sections(
+            text,
+            self.level_ranges_after_offset(offset, semantic_level)
+                .map(move |(l, sep)| (l, sep.start - offset..sep.end - offset)),
+        )
+        .map(move |(i, str)| (offset + i, str))
+    }
+
+    /// Clear out ranges we have moved past so future iterations are faster
+    fn update_cursor(&mut self, cursor: usize) {
+        self.cursor += self.ranges[self.cursor..]
+            .iter()
+            .position(|(_, range)| range.start >= cursor)
+            .unwrap_or_else(|| self.ranges.len() - self.cursor);
+    }
+}
+
+/// Returns chunks of text with their byte offsets as an iterator.
+#[derive(Debug)]
+struct TextChunks<'text, 'sizer, Sizer, Level>
+where
+    Sizer: ChunkSizer,
+    Level: SemanticLevel,
+{
+    /// Overall capacity of the chunk
+    capacity: ChunkCapacity,
+    /// How to validate chunk sizes
+    chunk_sizer: MemoizedChunkSizer<'sizer, Sizer>,
+    /// Average number of sections in a chunk for each level
+    chunk_stats: ChunkStats,
+    /// Current byte offset in the `text`
+    cursor: usize,
+    /// Reusable container for next sections to avoid extra allocations
+    next_sections: Vec<(usize, &'text str)>,
+    /// Overlap capacity
+    overlap: ChunkCapacity,
+    /// Previous item's end byte offset
+    prev_item_end: usize,
+    /// Splitter used for determining semantic levels.
+    semantic_split: SemanticSplitRanges<Level>,
+    /// Original text to iterate over and generate chunks from
+    text: &'text str,
+    /// The trimming method to apply
+    trim: Trim,
+}
+
+impl<'sizer, 'text: 'sizer, Sizer, Level> TextChunks<'text, 'sizer, Sizer, Level>
+where
+    Sizer: ChunkSizer,
+    Level: SemanticLevel,
+{
+    /// Generate new [`TextChunks`] iterator for a given text.
+    /// Starts with an offset of 0
+    fn new(
+        chunk_config: &'sizer ChunkConfig<Sizer>,
+        text: &'text str,
+        offsets: Vec<(Level, Range<usize>)>,
+        trim: Trim,
+    ) -> Self {
+        let ChunkConfig {
+            capacity,
+            overlap,
+            sizer,
+            trim: trim_enabled,
+        } = chunk_config;
+        Self {
+            capacity: *capacity,
+            chunk_sizer: MemoizedChunkSizer::new(sizer),
+            chunk_stats: ChunkStats::new(),
+            cursor: 0,
+            next_sections: Vec::new(),
+            overlap: (*overlap).into(),
+            prev_item_end: 0,
+            semantic_split: SemanticSplitRanges::new(offsets),
+            text,
+            trim: if *trim_enabled { trim } else { Trim::None },
+        }
+    }
+
+    /// Generate the next chunk, applying trimming settings.
+    /// Returns final byte offset and str.
+    /// Will return `None` if given an invalid range.
+    fn next_chunk(&mut self) -> Option<(usize, &'text str)> {
+        self.semantic_split.update_cursor(self.cursor);
+        let low = self.update_next_sections();
+        let (start, end) = self.binary_search_next_chunk(low)?;
+        let chunk = self.text.get(start..end)?;
+        self.chunk_stats.update_max_chunk_size(end - start);
+
+        // Reset caches so we can reuse the memory allocation
+        self.chunk_sizer.clear_cache();
+        // Optionally move cursor back if overlap is desired
+        self.update_cursor(end);
+
+        // Trim whitespace if user requested it
+        Some(self.trim.trim(start, chunk))
+    }
+
+    /// Use binary search to find the next chunk that fits within the chunk size
+    fn binary_search_next_chunk(&mut self, mut low: usize) -> Option<(usize, usize)> {
+        let start = self.cursor;
+        let mut end = self.cursor;
+        let mut equals_found = false;
+        let mut high = self.next_sections.len().saturating_sub(1);
+        let mut successful_index = None;
+        let mut successful_chunk_size = None;
+
+        while low <= high {
+            let mid = low + (high - low) / 2;
+            let (offset, str) = self.next_sections[mid];
+            let text_end = offset + str.len();
+            let chunk = self.text.get(start..text_end)?;
+            let chunk_size = self.chunk_sizer.chunk_size(start, chunk, self.trim);
+            let fits = self.capacity.fits(chunk_size);
+
+            match fits {
+                Ordering::Less => {
+                    // We got further than the last one, so update end
+                    if text_end > end {
+                        end = text_end;
+                        successful_index = Some(mid);
+                        successful_chunk_size = Some(chunk_size);
+                    }
+                }
+                Ordering::Equal => {
+                    // If we found a smaller equals use it. Or if this is the first equals we found
+                    if text_end < end || !equals_found {
+                        end = text_end;
+                        successful_index = Some(mid);
+                        successful_chunk_size = Some(chunk_size);
+                    }
+                    equals_found = true;
+                }
+                Ordering::Greater => {
+                    // If we're too big on our smallest run, we must return at least one section
+                    if mid == 0 && start == end {
+                        end = text_end;
+                        successful_index = Some(mid);
+                        successful_chunk_size = Some(chunk_size);
+                    }
+                }
+            }
+
+            // Adjust search area
+            if fits.is_lt() {
+                low = mid + 1;
+            } else if mid > 0 {
+                high = mid - 1;
+            } else {
+                // Nothing to adjust
+                break;
+            }
+        }
+
+        if let (Some(successful_index), Some(chunk_size)) = (successful_index, successful_chunk_size) {
+            let mut range = successful_index..self.next_sections.len();
+            // We've already checked the successful index
+            range.next();
+
+            for index in range {
+                let (offset, str) = self.next_sections[index];
+                let text_end = offset + str.len();
+                let chunk = self.text.get(start..text_end)?;
+                let size = self.chunk_sizer.chunk_size(start, chunk, self.trim);
+                if size <= chunk_size {
+                    if text_end > end {
+                        end = text_end;
+                    }
+                } else {
+                    break;
+                }
+            }
+        }
+
+        Some((start, end))
+    }
+
+    /// Use binary search to find the sections that fit within the overlap size.
+    /// If no overlap deisired, return end.
+    fn update_cursor(&mut self, end: usize) {
+        if self.overlap.max == 0 {
+            self.cursor = end;
+            return;
+        }
+
+        // Binary search for overlap
+        let mut start = end;
+        let mut low = 0;
+        // Find closest index that would work
+        let mut high = match self
+            .next_sections
+            .binary_search_by_key(&end, |(offset, str)| offset + str.len())
+        {
+            Ok(i) | Err(i) => i,
+        };
+
+        while low <= high {
+            let mid = low + (high - low) / 2;
+            let (offset, _) = self.next_sections[mid];
+            let chunk_size =
+                self.chunk_sizer
+                    .chunk_size(offset, self.text.get(offset..end).expect("Invalid range"), self.trim);
+            let fits = self.overlap.fits(chunk_size);
+
+            // We got further than the last one, so update start
+            if fits.is_le() && offset < start && offset > self.cursor {
+                start = offset;
+            }
+
+            // Adjust search area
+            if fits.is_lt() && mid > 0 {
+                high = mid - 1;
+            } else {
+                low = mid + 1;
+            }
+        }
+
+        self.cursor = start;
+    }
+
+    /// Find the ideal next sections, breaking it up until we find the largest chunk.
+    /// Increasing length of chunk until we find biggest size to minimize validation time
+    /// on huge chunks
+    fn update_next_sections(&mut self) -> usize {
+        // First thing, clear out the list, but reuse the allocated memory
+        self.next_sections.clear();
+
+        let remaining_text = self.text.get(self.cursor..).unwrap();
+
+        let (semantic_level, mut max_offset) = self.chunk_sizer.find_correct_level(
+            self.cursor,
+            &self.capacity,
+            self.semantic_split
+                .levels_in_remaining_text(self.cursor)
+                .filter_map(|level| {
+                    self.semantic_split
+                        .semantic_chunks(self.cursor, remaining_text, level)
+                        .next()
+                        .map(|(_, str)| (level, str))
+                }),
+            self.trim,
+        );
+
+        let sections = if let Some(semantic_level) = semantic_level {
+            Either::Left(
+                self.semantic_split
+                    .semantic_chunks(self.cursor, remaining_text, semantic_level),
+            )
+        } else {
+            let (semantic_level, fallback_max_offset) = self.chunk_sizer.find_correct_level(
+                self.cursor,
+                &self.capacity,
+                FallbackLevel::iter()
+                    .filter_map(|level| level.sections(remaining_text).next().map(|(_, str)| (level, str))),
+                self.trim,
+            );
+
+            max_offset = match (fallback_max_offset, max_offset) {
+                (Some(fallback), Some(max)) => Some(fallback.min(max)),
+                (fallback, max) => fallback.or(max),
+            };
+
+            let fallback_level = semantic_level.unwrap_or(FallbackLevel::Char);
+
+            Either::Right(
+                fallback_level
+                    .sections(remaining_text)
+                    .map(|(offset, text)| (self.cursor + offset, text)),
+            )
+        };
+
+        let mut sections = sections
+            .take_while(move |(offset, _)| max_offset.is_none_or(|max| *offset <= max))
+            .filter(|(_, str)| !str.is_empty());
+
+        // Start filling up the next sections. Since calculating the size of the chunk gets more expensive
+        // the farther we go, we conservatively check for a smaller range to do the later binary search in.
+        let mut low = 0;
+        let mut prev_equals: Option<usize> = None;
+        let max = self.capacity.max;
+        let mut target_offset = self.chunk_stats.max_chunk_size.unwrap_or(max);
+
+        loop {
+            let prev_num = self.next_sections.len();
+            for (offset, str) in sections.by_ref() {
+                self.next_sections.push((offset, str));
+                if offset + str.len() > (self.cursor.saturating_add(target_offset)) {
+                    break;
+                }
+            }
+            let new_num = self.next_sections.len();
+            // If we've iterated through the whole iterator, break here.
+            if new_num - prev_num == 0 {
+                break;
+            }
+
+            // Check if the last item fits
+            if let Some(&(offset, str)) = self.next_sections.last() {
+                let text_end = offset + str.len();
+                if (text_end - self.cursor) < target_offset {
+                    break;
+                }
+                let chunk_size = self.chunk_sizer.chunk_size(
+                    offset,
+                    self.text.get(self.cursor..text_end).expect("Invalid range"),
+                    self.trim,
+                );
+                let fits = self.capacity.fits(chunk_size);
+
+                if fits.is_le() {
+                    let final_offset = offset + str.len() - self.cursor;
+                    let size = chunk_size.max(1);
+                    let diff = (max - size).max(1);
+                    let avg_size = final_offset.div_ceil(size);
+
+                    target_offset = final_offset
+                        .saturating_add(diff.saturating_mul(avg_size))
+                        .saturating_add(final_offset.div_ceil(10));
+                }
+
+                match fits {
+                    Ordering::Less => {
+                        // We know we can go higher
+                        low = new_num.saturating_sub(1);
+                    }
+                    Ordering::Equal => {
+                        // Don't update low because it could be a range
+                        // If we've seen a previous equals, we can break if the size is bigger already
+                        if let Some(prev) = prev_equals
+                            && prev < chunk_size
+                        {
+                            break;
+                        }
+                        prev_equals = Some(chunk_size);
+                    }
+                    Ordering::Greater => {
+                        break;
+                    }
+                }
+            }
+        }
+
+        low
+    }
+}
+
+impl<'sizer, 'text: 'sizer, Sizer, Level> Iterator for TextChunks<'text, 'sizer, Sizer, Level>
+where
+    Sizer: ChunkSizer,
+    Level: SemanticLevel,
+{
+    type Item = (usize, &'text str);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            // Make sure we haven't reached the end
+            if self.cursor >= self.text.len() {
+                return None;
+            }
+
+            match self.next_chunk()? {
+                // Make sure we didn't get an empty chunk. Should only happen in
+                // cases where we trim.
+                (_, "") => {}
+                c => {
+                    let item_end = c.0 + c.1.len();
+                    // Skip because we've emitted a chunk whose content we've already emitted
+                    if item_end <= self.prev_item_end {
+                        continue;
+                    }
+                    self.prev_item_end = item_end;
+                    return Some(c);
+                }
+            }
+        }
+    }
+}
+
+/// Keeps track of the average size of chunks as we go
+#[derive(Debug, Default)]
+struct ChunkStats {
+    /// The size of the biggest chunk we've seen, if we have seen at least one
+    max_chunk_size: Option<usize>,
+}
+
+impl ChunkStats {
+    fn new() -> Self {
+        Self::default()
+    }
+
+    /// Update statistics after the chunk has been produced
+    fn update_max_chunk_size(&mut self, size: usize) {
+        self.max_chunk_size = self.max_chunk_size.map(|s| s.max(size)).or(Some(size));
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn chunk_stats_empty() {
+        let stats = ChunkStats::new();
+        assert_eq!(stats.max_chunk_size, None);
+    }
+
+    #[test]
+    fn chunk_stats_one() {
+        let mut stats = ChunkStats::new();
+        stats.update_max_chunk_size(10);
+        assert_eq!(stats.max_chunk_size, Some(10));
+    }
+
+    #[test]
+    fn chunk_stats_multiple() {
+        let mut stats = ChunkStats::new();
+        stats.update_max_chunk_size(10);
+        stats.update_max_chunk_size(20);
+        stats.update_max_chunk_size(30);
+        assert_eq!(stats.max_chunk_size, Some(30));
+    }
+
+    impl SemanticLevel for usize {}
+
+    #[test]
+    fn semantic_ranges_are_sorted() {
+        let ranges = SemanticSplitRanges::new(vec![(0, 0..1), (1, 0..2), (0, 1..2), (2, 0..4)]);
+
+        assert_eq!(ranges.ranges, vec![(2, 0..4), (1, 0..2), (0, 0..1), (0, 1..2)]);
+    }
+
+    #[test]
+    fn semantic_ranges_skip_previous_ranges() {
+        let mut ranges = SemanticSplitRanges::new(vec![(0, 0..1), (1, 0..2), (0, 1..2), (2, 0..4)]);
+
+        ranges.update_cursor(1);
+
+        assert_eq!(ranges.ranges_after_offset(0).collect::<Vec<_>>(), vec![(0, 1..2)]);
+    }
+}

crates/kreuzberg/src/chunking/text_splitter/splitter/fallback.rs

@@ -0,0 +1,57 @@
+//! Vendored from text-splitter v0.30.1 (MIT, © 2023 Benjamin Brandt). See ATTRIBUTIONS.md.
+
+use std::sync::LazyLock;
+
+use auto_enums::auto_enum;
+use icu_segmenter::{
+    GraphemeClusterSegmenter, GraphemeClusterSegmenterBorrowed, SentenceSegmenter, SentenceSegmenterBorrowed,
+    WordSegmenter, WordSegmenterBorrowed,
+    options::{SentenceBreakInvariantOptions, WordBreakInvariantOptions},
+};
+use itertools::Itertools;
+use strum::EnumIter;
+
+pub const GRAPHEME_SEGMENTER: GraphemeClusterSegmenterBorrowed<'static> = GraphemeClusterSegmenter::new();
+static WORD_SEGMENTER: LazyLock<WordSegmenterBorrowed<'static>> =
+    LazyLock::new(|| WordSegmenter::new_dictionary(WordBreakInvariantOptions::default()));
+static SENTENCE_SEGMENTER: LazyLock<SentenceSegmenterBorrowed<'static>> =
+    LazyLock::new(|| SentenceSegmenter::new(SentenceBreakInvariantOptions::default()));
+
+/// When using a custom semantic level, it is possible that none of them will
+/// be small enough to fit into the chunk size. In order to make sure we can
+/// still move the cursor forward, we fallback to unicode segmentation.
+#[derive(Clone, Copy, Debug, EnumIter, Eq, PartialEq, Ord, PartialOrd)]
+pub enum FallbackLevel {
+    /// Split by individual chars. May be larger than a single byte,
+    /// but we don't go lower so we always have valid UTF str's.
+    Char,
+    /// Split by [unicode grapheme clusters](https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries)    Grapheme,
+    GraphemeCluster,
+    /// Split by [unicode words](https://www.unicode.org/reports/tr29/#Word_Boundaries)
+    Word,
+    /// Split by [unicode sentences](https://www.unicode.org/reports/tr29/#Sentence_Boundaries)
+    Sentence,
+}
+
+impl FallbackLevel {
+    #[auto_enum(Iterator)]
+    pub fn sections(self, text: &str) -> impl Iterator<Item = (usize, &str)> {
+        match self {
+            Self::Char => text
+                .char_indices()
+                .map(move |(i, c)| (i, text.get(i..i + c.len_utf8()).expect("char should be valid"))),
+            Self::GraphemeCluster => GRAPHEME_SEGMENTER
+                .segment_str(text)
+                .tuple_windows()
+                .map(|(i, j)| (i, &text[i..j])),
+            Self::Word => WORD_SEGMENTER
+                .segment_str(text)
+                .tuple_windows()
+                .map(|(i, j)| (i, &text[i..j])),
+            Self::Sentence => SENTENCE_SEGMENTER
+                .segment_str(text)
+                .tuple_windows()
+                .map(|(i, j)| (i, &text[i..j])),
+        }
+    }
+}

crates/kreuzberg/src/chunking/text_splitter/splitter/markdown.rs

@@ -0,0 +1,272 @@
+//! Vendored from text-splitter v0.30.1 (MIT, © 2023 Benjamin Brandt). See ATTRIBUTIONS.md.
+/*!
+# [`MarkdownSplitter`]
+Semantic splitting of Markdown documents. Tries to use as many semantic units from Markdown
+as possible, according to the Common Mark specification.
+*/
+
+use std::{iter::once, ops::Range};
+
+use either::Either;
+use itertools::Itertools;
+use pulldown_cmark::{Event, Options, Parser, Tag};
+
+use super::super::{
+    ChunkConfig, ChunkSizer,
+    splitter::{SemanticLevel, Splitter},
+    trim::Trim,
+};
+
+/// Markdown splitter. Recursively splits chunks into the largest
+/// semantic units that fit within the chunk size. Also will
+/// attempt to merge neighboring chunks if they can fit within the
+/// given chunk size.
+#[derive(Debug)]
+pub struct MarkdownSplitter<Sizer>
+where
+    Sizer: ChunkSizer,
+{
+    /// Method of determining chunk sizes.
+    chunk_config: ChunkConfig<Sizer>,
+}
+
+impl<Sizer> MarkdownSplitter<Sizer>
+where
+    Sizer: ChunkSizer,
+{
+    /// Creates a new [`MarkdownSplitter`].
+    ///
+    /// ```text
+    /// let splitter = MarkdownSplitter::new(512);
+    /// ```
+    #[must_use]
+    pub fn new(chunk_config: impl Into<ChunkConfig<Sizer>>) -> Self {
+        Self {
+            chunk_config: chunk_config.into(),
+        }
+    }
+
+    /// Generate a list of chunks from a given text. Each chunk will be up to
+    /// the `max_chunk_size`.
+    ///
+    /// ## Method
+    ///
+    /// To preserve as much semantic meaning within a chunk as possible, each chunk is composed of the largest semantic units that can fit in the next given chunk. For each splitter type, there is a defined set of semantic levels. Here is an example of the steps used:
+    ///
+    /// 1. Characters
+    /// 2. [Unicode Grapheme Cluster Boundaries](https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries)
+    /// 3. [Unicode Word Boundaries](https://www.unicode.org/reports/tr29/#Word_Boundaries)
+    /// 4. [Unicode Sentence Boundaries](https://www.unicode.org/reports/tr29/#Sentence_Boundaries)
+    /// 5. Soft line breaks (single newline) which isn't necessarily a new element in Markdown.
+    /// 6. Inline elements such as: text nodes, emphasis, strong, strikethrough, link, image, table cells, inline code, footnote references, task list markers, and inline html.
+    /// 7. Block elements suce as: paragraphs, code blocks, footnote definitions, metadata. Also, a block quote or row/item within a table or list that can contain other "block" type elements, and a list or table that contains items.
+    /// 8. Thematic breaks or horizontal rules.
+    /// 9. Headings by level
+    ///
+    /// Splitting doesn't occur below the character level, otherwise you could get partial bytes of a char, which may not be a valid unicode str.
+    ///
+    /// Markdown is parsed according to the Commonmark spec, along with some optional features such as GitHub Flavored Markdown.
+    ///
+    /// ```text
+    /// let splitter = MarkdownSplitter::new(10);
+    /// let chunks = splitter.chunks("# Header\n\nfrom a\ndocument").collect::<Vec<_>>();
+    /// ```
+    pub fn chunks<'splitter, 'text: 'splitter>(
+        &'splitter self,
+        text: &'text str,
+    ) -> impl Iterator<Item = &'text str> + 'splitter {
+        Splitter::<_>::chunks(self, text)
+    }
+}
+
+impl<Sizer> Splitter<Sizer> for MarkdownSplitter<Sizer>
+where
+    Sizer: ChunkSizer,
+{
+    type Level = Element;
+
+    const TRIM: Trim = Trim::PreserveIndentation;
+
+    fn chunk_config(&self) -> &ChunkConfig<Sizer> {
+        &self.chunk_config
+    }
+
+    fn parse(&self, text: &str) -> Vec<(Self::Level, Range<usize>)> {
+        Parser::new_ext(text, Options::all())
+            .into_offset_iter()
+            .filter_map(|(event, range)| match event {
+                Event::Start(
+                    Tag::Emphasis
+                    | Tag::Strong
+                    | Tag::Strikethrough
+                    | Tag::Link { .. }
+                    | Tag::Image { .. }
+                    | Tag::Subscript
+                    | Tag::Superscript
+                    | Tag::TableCell,
+                )
+                | Event::Text(_)
+                | Event::HardBreak
+                | Event::Code(_)
+                | Event::InlineHtml(_)
+                | Event::InlineMath(_)
+                | Event::FootnoteReference(_)
+                | Event::TaskListMarker(_) => Some((Element::Inline, range)),
+                Event::SoftBreak => Some((Element::SoftBreak, range)),
+                Event::Html(_)
+                | Event::DisplayMath(_)
+                | Event::Start(
+                    Tag::Paragraph
+                    | Tag::CodeBlock(_)
+                    | Tag::FootnoteDefinition(_)
+                    | Tag::MetadataBlock(_)
+                    | Tag::TableHead
+                    | Tag::BlockQuote(_)
+                    | Tag::TableRow
+                    | Tag::Item
+                    | Tag::HtmlBlock
+                    | Tag::List(_)
+                    | Tag::Table(_)
+                    | Tag::DefinitionList
+                    | Tag::DefinitionListTitle
+                    | Tag::DefinitionListDefinition,
+                ) => Some((Element::Block, range)),
+                Event::Rule => Some((Element::Rule, range)),
+                Event::Start(Tag::Heading { level, .. }) => Some((Element::Heading(level.into()), range)),
+                // End events are identical to start, so no need to grab them.
+                Event::End(_) => None,
+            })
+            .collect()
+    }
+}
+
+/// Heading levels in markdown.
+/// Sorted in reverse order for sorting purposes.
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)]
+pub enum HeadingLevel {
+    H6,
+    H5,
+    H4,
+    H3,
+    H2,
+    H1,
+}
+
+impl From<pulldown_cmark::HeadingLevel> for HeadingLevel {
+    fn from(value: pulldown_cmark::HeadingLevel) -> Self {
+        match value {
+            pulldown_cmark::HeadingLevel::H1 => HeadingLevel::H1,
+            pulldown_cmark::HeadingLevel::H2 => HeadingLevel::H2,
+            pulldown_cmark::HeadingLevel::H3 => HeadingLevel::H3,
+            pulldown_cmark::HeadingLevel::H4 => HeadingLevel::H4,
+            pulldown_cmark::HeadingLevel::H5 => HeadingLevel::H5,
+            pulldown_cmark::HeadingLevel::H6 => HeadingLevel::H6,
+        }
+    }
+}
+
+/// How a particular semantic level relates to surrounding text elements.
+#[derive(Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
+enum SemanticSplitPosition {
+    /// The semantic level should be treated as its own chunk.
+    Own,
+    /// The semantic level should be included in the next chunk.
+    Next,
+}
+
+/// Different semantic levels that text can be split by.
+/// Each level provides a method of splitting text into chunks of a given level
+/// as well as a fallback in case a given fallback is too large.
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)]
+pub enum Element {
+    /// Single line break, which isn't necessarily a new element in Markdown
+    SoftBreak,
+    /// An inline element that is within a larger element such as a paragraph, but
+    /// more specific than a sentence.
+    Inline,
+    /// Paragraph, code block, metadata, a row/item within a table or list, block quote, that can contain other "block" type elements, List or table that contains items
+    Block,
+    /// thematic break/horizontal rule
+    Rule,
+    /// Heading levels in markdown
+    Heading(HeadingLevel),
+}
+
+impl Element {
+    fn split_position(self) -> SemanticSplitPosition {
+        match self {
+            Self::SoftBreak | Self::Block | Self::Rule | Self::Inline => SemanticSplitPosition::Own,
+            // Attach it to the next text
+            Self::Heading(_) => SemanticSplitPosition::Next,
+        }
+    }
+
+    fn treat_whitespace_as_previous(self) -> bool {
+        match self {
+            Self::SoftBreak | Self::Inline | Self::Rule | Self::Heading(_) => false,
+            Self::Block => true,
+        }
+    }
+}
+
+impl SemanticLevel for Element {
+    fn sections(
+        text: &str,
+        level_ranges: impl Iterator<Item = (Self, Range<usize>)>,
+    ) -> impl Iterator<Item = (usize, &str)> {
+        let mut cursor = 0;
+        let mut final_match = false;
+        level_ranges
+            .batching(move |it| {
+                loop {
+                    match it.next() {
+                        // If we've hit the end, actually return None
+                        None if final_match => return None,
+                        // First time we hit None, return the final section of the text
+                        None => {
+                            final_match = true;
+                            return text.get(cursor..).map(|t| Either::Left(once((cursor, t))));
+                        }
+                        // Return text preceding match + the match
+                        Some((level, range)) => {
+                            let offset = cursor;
+                            match level.split_position() {
+                                SemanticSplitPosition::Own => {
+                                    if range.start < cursor {
+                                        continue;
+                                    }
+                                    let prev_section =
+                                        text.get(cursor..range.start).expect("invalid character sequence");
+                                    if level.treat_whitespace_as_previous()
+                                        && prev_section.chars().all(char::is_whitespace)
+                                    {
+                                        let section = text.get(cursor..range.end).expect("invalid character sequence");
+                                        cursor = range.end;
+                                        return Some(Either::Left(once((offset, section))));
+                                    }
+                                    let separator =
+                                        text.get(range.start..range.end).expect("invalid character sequence");
+                                    cursor = range.end;
+                                    return Some(Either::Right(
+                                        [(offset, prev_section), (range.start, separator)].into_iter(),
+                                    ));
+                                }
+                                SemanticSplitPosition::Next => {
+                                    if range.start < cursor {
+                                        continue;
+                                    }
+                                    let prev_section =
+                                        text.get(cursor..range.start).expect("invalid character sequence");
+                                    // Separator will be part of the next chunk
+                                    cursor = range.start;
+                                    return Some(Either::Left(once((offset, prev_section))));
+                                }
+                            }
+                        }
+                    }
+                }
+            })
+            .flatten()
+            .filter(|(_, s)| !s.is_empty())
+    }
+}

crates/kreuzberg/src/chunking/text_splitter/splitter/text.rs

@@ -0,0 +1,126 @@
+//! Vendored from text-splitter v0.30.1 (MIT, © 2023 Benjamin Brandt). See ATTRIBUTIONS.md.
+/*!
+# [`TextSplitter`]
+Semantic splitting of text documents.
+*/
+
+use std::ops::Range;
+
+use itertools::Itertools;
+use memchr::memchr2_iter;
+
+use super::super::{
+    ChunkConfig, ChunkSizer,
+    splitter::{SemanticLevel, Splitter},
+};
+
+use super::fallback::GRAPHEME_SEGMENTER;
+
+/// Default plain-text splitter. Recursively splits chunks into the largest
+/// semantic units that fit within the chunk size. Also will attempt to merge
+/// neighboring chunks if they can fit within the given chunk size.
+#[derive(Debug)]
+pub struct TextSplitter<Sizer>
+where
+    Sizer: ChunkSizer,
+{
+    /// Method of determining chunk sizes.
+    chunk_config: ChunkConfig<Sizer>,
+}
+
+impl<Sizer> TextSplitter<Sizer>
+where
+    Sizer: ChunkSizer,
+{
+    /// Creates a new [`TextSplitter`].
+    ///
+    /// ```text
+    /// let splitter = TextSplitter::new(512);
+    /// ```
+    #[must_use]
+    pub fn new(chunk_config: impl Into<ChunkConfig<Sizer>>) -> Self {
+        Self {
+            chunk_config: chunk_config.into(),
+        }
+    }
+
+    /// Generate a list of chunks from a given text. Each chunk will be up to the `chunk_capacity`.
+    ///
+    /// ## Method
+    ///
+    /// To preserve as much semantic meaning within a chunk as possible, each chunk is composed of the largest semantic units that can fit in the next given chunk. For each splitter type, there is a defined set of semantic levels. Here is an example of the steps used:
+    ///
+    /// 1. Split the text by a increasing semantic levels.
+    /// 2. Check the first item for each level and select the highest level whose first item still fits within the chunk size.
+    /// 3. Merge as many of these neighboring sections of this level or above into a chunk to maximize chunk length.
+    ///    Boundaries of higher semantic levels are always included when merging, so that the chunk doesn't inadvertently cross semantic boundaries.
+    ///
+    /// The boundaries used to split the text if using the `chunks` method, in ascending order:
+    ///
+    /// 1. Characters
+    /// 2. [Unicode Grapheme Cluster Boundaries](https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries)
+    /// 3. [Unicode Word Boundaries](https://www.unicode.org/reports/tr29/#Word_Boundaries)
+    /// 4. [Unicode Sentence Boundaries](https://www.unicode.org/reports/tr29/#Sentence_Boundaries)
+    /// 5. Ascending sequence length of newlines. (Newline is `\r\n`, `\n`, or `\r`)
+    ///    Each unique length of consecutive newline sequences is treated as its own semantic level. So a sequence of 2 newlines is a higher level than a sequence of 1 newline, and so on.
+    ///
+    /// Splitting doesn't occur below the character level, otherwise you could get partial bytes of a char, which may not be a valid unicode str.
+    ///
+    /// ```text
+    /// let splitter = TextSplitter::new(10);
+    /// let chunks = splitter.chunks("Some text\n\nfrom a\ndocument").collect::<Vec<_>>();
+    /// ```
+    pub fn chunks<'splitter, 'text: 'splitter>(
+        &'splitter self,
+        text: &'text str,
+    ) -> impl Iterator<Item = &'text str> + 'splitter {
+        Splitter::<_>::chunks(self, text)
+    }
+}
+
+impl<Sizer> Splitter<Sizer> for TextSplitter<Sizer>
+where
+    Sizer: ChunkSizer,
+{
+    type Level = LineBreaks;
+
+    fn chunk_config(&self) -> &ChunkConfig<Sizer> {
+        &self.chunk_config
+    }
+
+    fn parse(&self, text: &str) -> Vec<(Self::Level, Range<usize>)> {
+        memchr2_iter(b'\n', b'\r', text.as_bytes())
+            .map(|i| i..i + 1)
+            .coalesce(|a, b| {
+                if a.end == b.start {
+                    Ok(a.start..b.end)
+                } else {
+                    Err((a, b))
+                }
+            })
+            .map(|range| {
+                let level = GRAPHEME_SEGMENTER
+                    .segment_str(text.get(range.start..range.end).unwrap())
+                    .tuple_windows::<(usize, usize)>()
+                    .count();
+                (
+                    match level {
+                        0 => unreachable!("regex should always match at least one newline"),
+                        n => LineBreaks(n),
+                    },
+                    range,
+                )
+            })
+            .collect()
+    }
+}
+
+/// Different semantic levels that text can be split by.
+/// Each level provides a method of splitting text into chunks of a given level
+/// as well as a fallback in case a given fallback is too large.
+///
+/// Split by given number of linebreaks, either `\n`, `\r`, or `\r\n`.
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)]
+pub struct LineBreaks(usize);
+
+impl SemanticLevel for LineBreaks {}

crates/kreuzberg/src/chunking/text_splitter/trim.rs

@@ -0,0 +1,74 @@
+//! Vendored from text-splitter v0.30.1 (MIT, © 2023 Benjamin Brandt). See ATTRIBUTIONS.md.
+/*!
+Different trimming behaviors for different splitter types.
+*/
+
+/// Out-of-the-box trim options.
+/// If you need a custom trim behavior, you can implement the `Trim` trait.
+#[derive(Clone, Copy, Debug)]
+pub enum Trim {
+    /// Will remove all leading and trailing whitespaces.
+    All,
+    /// Will remove all leading newlines and all trailing whitespace.
+    /// If there are newlines within the text, then indentation will be preserved
+    /// (leading spaces or tabs at the beginning of the text). If not, then all
+    /// leading whitespace will be trimmed.
+    /// Useful for text like Markdown or code, where indentation is important to
+    /// the meaning of the text.
+    PreserveIndentation,
+    /// Apply no trimming
+    None,
+}
+
+const NEWLINES: [char; 2] = ['\n', '\r'];
+
+impl Trim {
+    pub fn trim(self, offset: usize, chunk: &str) -> (usize, &str) {
+        match self {
+            Self::All => {
+                // Figure out how many bytes we lose trimming the beginning
+                let diff = chunk.len() - chunk.trim_start().len();
+                (offset + diff, chunk.trim())
+            }
+            Self::PreserveIndentation => {
+                // Preserve indentation if we have newlines inside the element
+                if chunk.trim().contains(NEWLINES) {
+                    let diff = chunk.len() - chunk.trim_start_matches(NEWLINES).len();
+                    (offset + diff, chunk.trim_start_matches(NEWLINES).trim_end())
+                } else {
+                    Self::All.trim(offset, chunk)
+                }
+            }
+            Self::None => (offset, chunk),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn trim_all() {
+        let chunk = "  hello world  ";
+        let (offset, chunk) = Trim::All.trim(0, chunk);
+        assert_eq!(offset, 2);
+        assert_eq!(chunk, "hello world");
+    }
+
+    #[test]
+    fn trim_indentation_fallback() {
+        let chunk = "  hello world  ";
+        let (offset, chunk) = Trim::PreserveIndentation.trim(0, chunk);
+        assert_eq!(offset, 2);
+        assert_eq!(chunk, "hello world");
+    }
+
+    #[test]
+    fn trim_indentation_preserved() {
+        let chunk = "\n  hello\n  world  ";
+        let (offset, chunk) = Trim::PreserveIndentation.trim(0, chunk);
+        assert_eq!(offset, 1);
+        assert_eq!(chunk, "  hello\n  world");
+    }
+}

crates/kreuzberg/src/chunking/tokenizer_cache.rs

@@ -0,0 +1,81 @@
+//! In-memory cache for HuggingFace tokenizers.
+//!
+//! Tokenizers are downloaded from HuggingFace Hub on first use and cached in-memory
+//! for subsequent calls. File-level caching is handled by the `hf-hub` crate
+//! (defaults to `~/.cache/huggingface/`, configurable via `HF_HOME` env var).
+
+use ahash::AHashMap;
+use std::sync::{Arc, RwLock};
+
+use std::sync::LazyLock;
+
+use crate::KreuzbergError;
+
+/// Global in-memory cache for loaded tokenizers.
+///
+/// Keyed by model ID string. Once a tokenizer is loaded and parsed,
+/// it's stored here to avoid re-downloading and re-parsing on subsequent calls.
+static TOKENIZER_CACHE: LazyLock<RwLock<AHashMap<String, Arc<tokenizers::Tokenizer>>>> =
+    LazyLock::new(|| RwLock::new(AHashMap::new()));
+
+/// Get a cached tokenizer or initialize one from HuggingFace Hub.
+///
+/// Uses a two-phase locking strategy (read lock first, write lock on miss)
+/// following the same pattern as the embeddings model cache in `embeddings.rs`.
+///
+/// # Arguments
+///
+/// * `model` - HuggingFace model ID (e.g., "Xenova/gpt-4o", "bert-base-uncased")
+///
+/// # Errors
+///
+/// Returns an error if the tokenizer cannot be downloaded or parsed.
+pub(crate) fn get_or_init_tokenizer(model: &str) -> crate::Result<Arc<tokenizers::Tokenizer>> {
+    // Phase 1: try read lock (fast path for cache hits)
+    {
+        let cache = TOKENIZER_CACHE
+            .read()
+            .map_err(|e| KreuzbergError::Other(format!("Tokenizer cache read lock poisoned: {}", e)))?;
+        if let Some(tok) = cache.get(model) {
+            return Ok(Arc::clone(tok));
+        }
+    }
+
+    // Phase 2: write lock, double-check, then initialize
+    let mut cache = TOKENIZER_CACHE
+        .write()
+        .map_err(|e| KreuzbergError::Other(format!("Tokenizer cache write lock poisoned: {}", e)))?;
+
+    // Double-check after acquiring write lock (another thread may have initialized)
+    if let Some(tok) = cache.get(model) {
+        return Ok(Arc::clone(tok));
+    }
+
+    let tokenizer = tokenizers::Tokenizer::from_pretrained(model, None)
+        .map_err(|e| KreuzbergError::validation(format!("Failed to load tokenizer '{}': {}", model, e)))?;
+
+    let arc = Arc::new(tokenizer);
+    cache.insert(model.to_string(), Arc::clone(&arc));
+    Ok(arc)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_cache_returns_same_instance() {
+        // This test requires network access to download a tokenizer.
+        // Skip in CI by checking for a specific env var.
+        if std::env::var("CI").is_ok() {
+            return;
+        }
+
+        let model = "bert-base-uncased";
+        let tok1 = get_or_init_tokenizer(model).unwrap();
+        let tok2 = get_or_init_tokenizer(model).unwrap();
+
+        // Same Arc instance (pointer equality)
+        assert!(Arc::ptr_eq(&tok1, &tok2));
+    }
+}

crates/kreuzberg/src/chunking/validation.rs

@@ -0,0 +1,686 @@
+//! UTF-8 boundary validation for text chunking.
+//!
+//! This module provides validation functions to ensure that page boundaries fall
+//! on valid UTF-8 character boundaries. This is critical to prevent text corruption
+//! when boundaries are created from language bindings or external sources, particularly
+//! with multibyte UTF-8 characters (emoji, CJK characters, combining marks, etc.).
+
+use crate::error::{KreuzbergError, Result};
+use crate::types::PageBoundary;
+use bitvec::prelude::*;
+
+/// Threshold below which we use O(1) direct validation instead of precomputing a BitVec.
+///
+/// When there are 10 or fewer boundaries, the overhead of creating a BitVec (which is O(n)
+/// where n is the text length) exceeds the cost of calling `is_char_boundary()` directly
+/// for each boundary position. This threshold balances performance across different scenarios:
+/// - Small documents with few boundaries: fast path dominates
+/// - Large documents with many boundaries: batch path leverages the precomputed BitVec
+pub const ADAPTIVE_VALIDATION_THRESHOLD: usize = 10;
+
+/// Pre-computes valid UTF-8 character boundaries for a text string.
+///
+/// This function performs a single O(n) pass through the text to identify all valid
+/// UTF-8 character boundaries, storing them in a BitVec for O(1) lookups.
+///
+/// # Arguments
+///
+/// * `text` - The text to analyze
+///
+/// # Returns
+///
+/// A BitVec where each bit represents whether a byte offset is a valid UTF-8 character boundary.
+/// The BitVec has length `text.len() + 1` (includes the end position).
+///
+/// # Examples
+///
+/// ```ignore
+/// let text = "Hello 👋";
+/// let boundaries = precompute_utf8_boundaries(text);
+/// assert!(boundaries[0]);      // Start is always valid
+/// assert!(boundaries[6]);      // 'H' + "ello " = 6 bytes
+/// assert!(!boundaries[7]);     // Middle of emoji (first byte of 4-byte sequence)
+/// assert!(boundaries[10]);     // After emoji (valid boundary)
+/// ```
+pub(crate) fn precompute_utf8_boundaries(text: &str) -> BitVec {
+    let text_len = text.len();
+    let mut boundaries = bitvec![0; text_len + 1];
+
+    boundaries.set(0, true);
+
+    for (i, _) in text.char_indices() {
+        if i <= text_len {
+            boundaries.set(i, true);
+        }
+    }
+
+    if text_len > 0 {
+        boundaries.set(text_len, true);
+    }
+
+    boundaries
+}
+
+/// Validates that byte offsets in page boundaries fall on valid UTF-8 character boundaries.
+///
+/// This function ensures that all page boundary positions are at valid UTF-8 character
+/// boundaries within the text. This is CRITICAL to prevent text corruption when boundaries
+/// are created from language bindings or external sources, particularly with multibyte
+/// UTF-8 characters (emoji, CJK characters, combining marks, etc.).
+///
+/// **Performance Strategy**: Uses adaptive validation to optimize for different boundary counts:
+/// - **Small sets (≤10 boundaries)**: O(k) approach using Rust's native `is_char_boundary()` for each position
+/// - **Large sets (>10 boundaries)**: O(n) precomputation with O(1) lookups via BitVec
+///
+/// For typical PDF documents with 1-10 page boundaries, the fast path provides 30-50% faster
+/// validation than always precomputing. For documents with 100+ boundaries, batch precomputation
+/// is 2-4% faster overall due to amortized costs. This gives ~2-4% improvement across all scenarios.
+///
+/// # Arguments
+///
+/// * `text` - The text being chunked
+/// * `boundaries` - Page boundary markers to validate
+///
+/// # Returns
+///
+/// Returns `Ok(())` if all boundaries are at valid UTF-8 character boundaries.
+/// Returns `KreuzbergError::Validation` if any boundary is at an invalid position.
+///
+/// # UTF-8 Boundary Safety
+///
+/// Rust strings use UTF-8 encoding where characters can be 1-4 bytes. For example:
+/// - ASCII letters: 1 byte each
+/// - Emoji (🌍): 4 bytes but 1 character
+/// - CJK characters (中): 3 bytes but 1 character
+///
+/// This function checks that all byte_start and byte_end values are at character boundaries
+/// using an adaptive strategy: direct calls for small boundary sets, or precomputed BitVec
+/// for large sets.
+pub(crate) fn validate_utf8_boundaries(text: &str, boundaries: &[PageBoundary]) -> Result<()> {
+    if boundaries.is_empty() {
+        return Ok(());
+    }
+
+    let text_len = text.len();
+
+    if boundaries.len() <= ADAPTIVE_VALIDATION_THRESHOLD {
+        validate_utf8_boundaries_fast_path(text, boundaries, text_len)
+    } else {
+        validate_utf8_boundaries_batch_path(text, boundaries, text_len)
+    }
+}
+
+/// Fast path: direct UTF-8 boundary validation for small boundary counts (≤10).
+///
+/// Uses Rust's native `str::is_char_boundary()` for O(1) checks on each boundary position.
+/// This avoids the O(n) overhead of BitVec precomputation, making it ideal for typical
+/// PDF documents with few page boundaries.
+///
+/// # Arguments
+///
+/// * `text` - The text being validated
+/// * `boundaries` - Page boundary markers to validate
+/// * `text_len` - Pre-computed text length (avoids recomputation)
+///
+/// # Returns
+///
+/// Returns `Ok(())` if all boundaries are at valid UTF-8 character boundaries.
+/// Returns `KreuzbergError::Validation` if any boundary is invalid.
+fn validate_utf8_boundaries_fast_path(text: &str, boundaries: &[PageBoundary], text_len: usize) -> Result<()> {
+    for (idx, boundary) in boundaries.iter().enumerate() {
+        if boundary.byte_start > text_len {
+            return Err(KreuzbergError::validation(format!(
+                "Page boundary {} has byte_start={} which exceeds text length {}",
+                idx, boundary.byte_start, text_len
+            )));
+        }
+
+        if boundary.byte_end > text_len {
+            return Err(KreuzbergError::validation(format!(
+                "Page boundary {} has byte_end={} which exceeds text length {}",
+                idx, boundary.byte_end, text_len
+            )));
+        }
+
+        if boundary.byte_start > 0 && boundary.byte_start < text_len && !text.is_char_boundary(boundary.byte_start) {
+            return Err(KreuzbergError::validation(format!(
+                "Page boundary {} has byte_start={} which is not a valid UTF-8 character boundary (text length={}). This may indicate corrupted multibyte characters (emoji, CJK, etc.)",
+                idx, boundary.byte_start, text_len
+            )));
+        }
+
+        if boundary.byte_end > 0 && boundary.byte_end < text_len && !text.is_char_boundary(boundary.byte_end) {
+            return Err(KreuzbergError::validation(format!(
+                "Page boundary {} has byte_end={} which is not a valid UTF-8 character boundary (text length={}). This may indicate corrupted multibyte characters (emoji, CJK, etc.)",
+                idx, boundary.byte_end, text_len
+            )));
+        }
+    }
+
+    Ok(())
+}
+
+/// Batch path: precomputed BitVec validation for large boundary counts (>10).
+///
+/// Precomputes all valid UTF-8 boundaries in a single O(n) pass, then performs O(1)
+/// lookups for each boundary position. This is more efficient than O(k*1) direct checks
+/// when k is large or when the repeated `is_char_boundary()` calls have measurable overhead.
+///
+/// # Arguments
+///
+/// * `text` - The text being validated
+/// * `boundaries` - Page boundary markers to validate
+/// * `text_len` - Pre-computed text length (avoids recomputation)
+///
+/// # Returns
+///
+/// Returns `Ok(())` if all boundaries are at valid UTF-8 character boundaries.
+/// Returns `KreuzbergError::Validation` if any boundary is invalid.
+fn validate_utf8_boundaries_batch_path(text: &str, boundaries: &[PageBoundary], text_len: usize) -> Result<()> {
+    let valid_boundaries = precompute_utf8_boundaries(text);
+
+    for (idx, boundary) in boundaries.iter().enumerate() {
+        if boundary.byte_start > text_len {
+            return Err(KreuzbergError::validation(format!(
+                "Page boundary {} has byte_start={} which exceeds text length {}",
+                idx, boundary.byte_start, text_len
+            )));
+        }
+
+        if boundary.byte_end > text_len {
+            return Err(KreuzbergError::validation(format!(
+                "Page boundary {} has byte_end={} which exceeds text length {}",
+                idx, boundary.byte_end, text_len
+            )));
+        }
+
+        if boundary.byte_start > 0 && boundary.byte_start <= text_len && !valid_boundaries[boundary.byte_start] {
+            return Err(KreuzbergError::validation(format!(
+                "Page boundary {} has byte_start={} which is not a valid UTF-8 character boundary (text length={}). This may indicate corrupted multibyte characters (emoji, CJK, etc.)",
+                idx, boundary.byte_start, text_len
+            )));
+        }
+
+        if boundary.byte_end > 0 && boundary.byte_end <= text_len && !valid_boundaries[boundary.byte_end] {
+            return Err(KreuzbergError::validation(format!(
+                "Page boundary {} has byte_end={} which is not a valid UTF-8 character boundary (text length={}). This may indicate corrupted multibyte characters (emoji, CJK, etc.)",
+                idx, boundary.byte_end, text_len
+            )));
+        }
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_validate_utf8_boundaries_valid_ascii() {
+        let text = "This is ASCII text.";
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 10,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 10,
+                byte_end: 19,
+                page_number: 2,
+            },
+        ];
+
+        let result = validate_utf8_boundaries(text, &boundaries);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_validate_utf8_boundaries_valid_emoji() {
+        let text = "Hello 👋 World 🌍 End";
+
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 11,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 11,
+                byte_end: 25,
+                page_number: 2,
+            },
+        ];
+
+        let result = validate_utf8_boundaries(text, &boundaries);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_validate_utf8_boundaries_valid_cjk() {
+        let text = "你好世界 こんにちは 안녕하세요";
+
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 13,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 13,
+                byte_end: 44,
+                page_number: 2,
+            },
+        ];
+
+        let result = validate_utf8_boundaries(text, &boundaries);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_validate_utf8_boundaries_invalid_mid_emoji() {
+        let text = "Hello 👋 World";
+        let boundaries = vec![PageBoundary {
+            byte_start: 0,
+            byte_end: 7,
+            page_number: 1,
+        }];
+
+        let result = validate_utf8_boundaries(text, &boundaries);
+        assert!(result.is_err());
+        let err = result.unwrap_err();
+        assert!(err.to_string().contains("UTF-8 character boundary"));
+        assert!(err.to_string().contains("byte_end=7"));
+    }
+
+    #[test]
+    fn test_validate_utf8_boundaries_invalid_mid_multibyte_cjk() {
+        let text = "中文文本";
+        let boundaries = vec![PageBoundary {
+            byte_start: 0,
+            byte_end: 1,
+            page_number: 1,
+        }];
+
+        let result = validate_utf8_boundaries(text, &boundaries);
+        assert!(result.is_err());
+        let err = result.unwrap_err();
+        assert!(err.to_string().contains("UTF-8 character boundary"));
+    }
+
+    #[test]
+    fn test_validate_utf8_boundaries_byte_start_exceeds_length() {
+        let text = "Short";
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 3,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 10,
+                byte_end: 15,
+                page_number: 2,
+            },
+        ];
+
+        let result = validate_utf8_boundaries(text, &boundaries);
+        assert!(result.is_err());
+        let err = result.unwrap_err();
+        assert!(err.to_string().contains("exceeds text length"));
+    }
+
+    #[test]
+    fn test_validate_utf8_boundaries_byte_end_exceeds_length() {
+        let text = "Short";
+        let boundaries = vec![PageBoundary {
+            byte_start: 0,
+            byte_end: 100,
+            page_number: 1,
+        }];
+
+        let result = validate_utf8_boundaries(text, &boundaries);
+        assert!(result.is_err());
+        let err = result.unwrap_err();
+        assert!(err.to_string().contains("exceeds text length"));
+    }
+
+    #[test]
+    fn test_validate_utf8_boundaries_empty_boundaries() {
+        let text = "Some text";
+        let boundaries: Vec<PageBoundary> = vec![];
+
+        let result = validate_utf8_boundaries(text, &boundaries);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_validate_utf8_boundaries_at_text_boundaries() {
+        let text = "Exact boundary test";
+        let text_len = text.len();
+        let boundaries = vec![PageBoundary {
+            byte_start: 0,
+            byte_end: text_len,
+            page_number: 1,
+        }];
+
+        let result = validate_utf8_boundaries(text, &boundaries);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_validate_utf8_boundaries_mixed_languages() {
+        let text = "English text mixed with 中文 and français";
+
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 24,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 24,
+                byte_end: text.len(),
+                page_number: 2,
+            },
+        ];
+
+        let result = validate_utf8_boundaries(text, &boundaries);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_validate_utf8_boundaries_error_messages_are_clear() {
+        let text = "Test 👋 text";
+
+        let boundaries = vec![PageBoundary {
+            byte_start: 0,
+            byte_end: 6,
+            page_number: 1,
+        }];
+
+        let result = validate_utf8_boundaries(text, &boundaries);
+        assert!(result.is_err());
+        let err = result.unwrap_err();
+        let err_msg = err.to_string();
+        assert!(err_msg.contains("UTF-8"));
+        assert!(err_msg.contains("boundary"));
+        assert!(err_msg.contains("6"));
+    }
+
+    #[test]
+    fn test_validate_utf8_boundaries_multiple_valid_boundaries() {
+        let text = "First👋Second🌍Third";
+
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 5,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 5,
+                byte_end: 9,
+                page_number: 2,
+            },
+            PageBoundary {
+                byte_start: 9,
+                byte_end: 15,
+                page_number: 3,
+            },
+            PageBoundary {
+                byte_start: 15,
+                byte_end: 19,
+                page_number: 4,
+            },
+            PageBoundary {
+                byte_start: 19,
+                byte_end: text.len(),
+                page_number: 5,
+            },
+        ];
+
+        let result = validate_utf8_boundaries(text, &boundaries);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_validate_utf8_boundaries_zero_start_and_end() {
+        let text = "Text";
+
+        // Zero-length ranges are allowed as they represent valid UTF-8 boundaries
+        // (e.g., cursor positions, empty pages, etc.)
+        let boundaries = vec![PageBoundary {
+            byte_start: 0,
+            byte_end: 0,
+            page_number: 1,
+        }];
+
+        let result = validate_utf8_boundaries(text, &boundaries);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_utf8_boundaries_caching_with_many_boundaries() {
+        let text = "🌍 Hello World ".repeat(200);
+        let text_len = text.len();
+
+        let mut boundaries = vec![];
+        let boundary_count = 10;
+        let step = text_len / boundary_count;
+
+        for i in 0..boundary_count {
+            let start = i * step;
+            let end = if i == boundary_count - 1 {
+                text_len
+            } else {
+                (i + 1) * step
+            };
+
+            if start < end
+                && start <= text_len
+                && end <= text_len
+                && let Some(boundary_start) = text[..start].char_indices().last().map(|(idx, _)| idx)
+                && let Some(boundary_end) = text[..end].char_indices().last().map(|(idx, _)| idx)
+            {
+                boundaries.push(PageBoundary {
+                    byte_start: boundary_start,
+                    byte_end: boundary_end,
+                    page_number: (i + 1) as u32,
+                });
+            }
+        }
+
+        if !boundaries.is_empty() {
+            let result = validate_utf8_boundaries(&text, &boundaries);
+            assert!(result.is_ok());
+        }
+    }
+
+    #[test]
+    fn test_utf8_boundaries_caching_large_document_with_emojis() {
+        let large_text = "This is a large document with lots of emoji: 🌍 🚀 💻 🎉 🔥 ✨ 🎨 🌟 ".repeat(100);
+
+        let all_indices: Vec<usize> = large_text.char_indices().map(|(idx, _)| idx).collect();
+
+        let third_idx = all_indices.len() / 3;
+        let two_thirds_idx = (2 * all_indices.len()) / 3;
+
+        let boundary_start_1 = if third_idx < all_indices.len() {
+            all_indices[third_idx]
+        } else {
+            large_text.len()
+        };
+
+        let boundary_start_2 = if two_thirds_idx < all_indices.len() {
+            all_indices[two_thirds_idx]
+        } else {
+            large_text.len()
+        };
+
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: boundary_start_1,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: boundary_start_1,
+                byte_end: boundary_start_2,
+                page_number: 2,
+            },
+            PageBoundary {
+                byte_start: boundary_start_2,
+                byte_end: large_text.len(),
+                page_number: 3,
+            },
+        ];
+
+        let result = validate_utf8_boundaries(&large_text, &boundaries);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_adaptive_validation_small_boundary_set() {
+        let text = "Hello 👋 World 🌍 End";
+
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 6,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 6,
+                byte_end: 15,
+                page_number: 2,
+            },
+            PageBoundary {
+                byte_start: 15,
+                byte_end: text.len(),
+                page_number: 3,
+            },
+        ];
+
+        let result = validate_utf8_boundaries(text, &boundaries);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_adaptive_validation_threshold_boundary() {
+        let text = "Test text ".repeat(50);
+        let text_len = text.len();
+
+        let mut boundaries = vec![];
+        let step = text_len / ADAPTIVE_VALIDATION_THRESHOLD;
+
+        for i in 0..ADAPTIVE_VALIDATION_THRESHOLD {
+            let start = i * step;
+            let end = if i == ADAPTIVE_VALIDATION_THRESHOLD - 1 {
+                text_len
+            } else {
+                (i + 1) * step
+            };
+
+            if start < end
+                && start <= text_len
+                && end <= text_len
+                && let Some(boundary_start) = text[..start.min(text_len - 1)]
+                    .char_indices()
+                    .last()
+                    .map(|(idx, _)| idx)
+                && let Some(boundary_end) = text[..end.min(text_len)].char_indices().last().map(|(idx, _)| idx)
+                && boundary_start < boundary_end
+            {
+                boundaries.push(PageBoundary {
+                    byte_start: boundary_start,
+                    byte_end: boundary_end,
+                    page_number: (i + 1) as u32,
+                });
+            }
+        }
+
+        if !boundaries.is_empty() {
+            let result = validate_utf8_boundaries(&text, &boundaries);
+            assert!(result.is_ok());
+        }
+    }
+
+    #[test]
+    fn test_adaptive_validation_large_boundary_set() {
+        let text = "Lorem ipsum dolor sit amet ".repeat(100);
+        let text_len = text.len();
+
+        let mut boundaries = vec![];
+        let boundary_count = 50;
+        let step = text_len / boundary_count;
+
+        for i in 0..boundary_count {
+            let start = i * step;
+            let end = if i == boundary_count - 1 {
+                text_len
+            } else {
+                (i + 1) * step
+            };
+
+            if start < end
+                && start <= text_len
+                && end <= text_len
+                && let Some(boundary_start) = text[..start.min(text_len - 1)]
+                    .char_indices()
+                    .last()
+                    .map(|(idx, _)| idx)
+                && let Some(boundary_end) = text[..end.min(text_len)].char_indices().last().map(|(idx, _)| idx)
+                && boundary_start < boundary_end
+            {
+                boundaries.push(PageBoundary {
+                    byte_start: boundary_start,
+                    byte_end: boundary_end,
+                    page_number: (i + 1) as u32,
+                });
+            }
+        }
+
+        if !boundaries.is_empty() {
+            let result = validate_utf8_boundaries(&text, &boundaries);
+            assert!(result.is_ok());
+        }
+    }
+
+    #[test]
+    fn test_adaptive_validation_consistency() {
+        let text = "Mixed language: 你好 مرحبا Здравствуй ".repeat(50);
+
+        let boundaries = vec![
+            PageBoundary {
+                byte_start: 0,
+                byte_end: 50,
+                page_number: 1,
+            },
+            PageBoundary {
+                byte_start: 50,
+                byte_end: 100,
+                page_number: 2,
+            },
+            PageBoundary {
+                byte_start: 100,
+                byte_end: 150,
+                page_number: 3,
+            },
+            PageBoundary {
+                byte_start: 150,
+                byte_end: 200,
+                page_number: 4,
+            },
+            PageBoundary {
+                byte_start: 200,
+                byte_end: text.len(),
+                page_number: 5,
+            },
+        ];
+
+        let result = validate_utf8_boundaries(&text, &boundaries);
+        let _ = result;
+    }
+}

crates/kreuzberg/src/chunking/yaml_section.rs

@@ -0,0 +1,702 @@
+//! Structured data section-aware chunking (YAML and JSON).
+//!
+//! Splits YAML and JSON documents by nested keys, creating one chunk per
+//! leaf section. Each chunk includes the full key path for context, making it
+//! possible for RAG systems to index configuration files at the section level
+//! rather than as a single monolithic document.
+//!
+//! Both formats are parsed into a value tree (`serde_yaml_ng` for YAML,
+//! `serde_json` for JSON) and walked with BFS to avoid stack overflow on
+//! deeply nested untrusted input (see RUSTSEC-2024-0012).
+
+use std::collections::VecDeque;
+
+use crate::error::Result;
+use crate::types::{Chunk, ChunkMetadata, PageBoundary};
+
+use super::config::{ChunkingConfig, ChunkingResult};
+
+/// Split YAML or JSON text into per-section chunks.
+///
+/// Detects JSON (starts with `{` or `[`) and parses with `serde_json`.
+/// Otherwise parses with `serde_yaml_ng`. Both are walked with BFS.
+///
+/// Falls back to plain text chunking if the input cannot be parsed
+/// or has no extractable mapping keys.
+pub(crate) fn chunk_yaml_by_sections(
+    text: &str,
+    config: &ChunkingConfig,
+    page_boundaries: Option<&[PageBoundary]>,
+) -> Result<ChunkingResult> {
+    if text.is_empty() {
+        return Ok(ChunkingResult {
+            chunks: vec![],
+            chunk_count: 0,
+        });
+    }
+
+    let trimmed = text.trim();
+    let sections = if trimmed.starts_with('{') || trimmed.starts_with('[') {
+        flatten_json(text)?
+    } else {
+        flatten_yaml(text)?
+    };
+
+    if sections.is_empty() {
+        return fallback_to_text(text, config);
+    }
+
+    build_chunks_from_sections(&sections, config, page_boundaries)
+}
+
+/// Parse JSON and flatten into leaf sections via BFS.
+///
+/// Uses serde_json for strict JSON parsing, then converts to serde_yaml_ng::Value
+/// so both paths share the same BFS walker.
+fn flatten_json(text: &str) -> Result<Vec<Section>> {
+    let json_val: serde_json::Value = match serde_json::from_str(text) {
+        Ok(v) => v,
+        Err(_) => return Ok(Vec::new()),
+    };
+    if !json_val.is_object() {
+        return Ok(Vec::new());
+    }
+    // Convert JSON → YAML value tree (YAML is a superset of JSON)
+    let yaml_str = serde_json::to_string(&json_val).unwrap_or_default();
+    let yaml_val: serde_yaml_ng::Value = match serde_yaml_ng::from_str(&yaml_str) {
+        Ok(v) => v,
+        Err(_) => return Ok(Vec::new()),
+    };
+    let mapping = match yaml_val.as_mapping() {
+        Some(m) => m,
+        None => return Ok(Vec::new()),
+    };
+    Ok(flatten_value_tree(
+        mapping
+            .iter()
+            .filter_map(|(k, v)| Some((k.as_str()?.to_string(), v.clone()))),
+    ))
+}
+
+/// Parse YAML and flatten into leaf sections via BFS.
+fn flatten_yaml(text: &str) -> Result<Vec<Section>> {
+    let parsed: serde_yaml_ng::Value = match serde_yaml_ng::from_str(text) {
+        Ok(v) => v,
+        Err(_) => return Ok(Vec::new()),
+    };
+    let mapping = match parsed.as_mapping() {
+        Some(m) => m,
+        None => return Ok(Vec::new()),
+    };
+    Ok(flatten_value_tree(
+        mapping
+            .iter()
+            .filter_map(|(k, v)| Some((k.as_str()?.to_string(), v.clone()))),
+    ))
+}
+
+/// BFS walk of a key-value tree, collecting leaf sections.
+///
+/// Uses a VecDeque work queue instead of recursion to avoid stack overflow
+/// on deeply nested input (RUSTSEC-2024-0012).
+///
+/// A node is a leaf if it is not a mapping/object, or is an empty mapping/object.
+fn flatten_value_tree(roots: impl Iterator<Item = (String, serde_yaml_ng::Value)>) -> Vec<Section> {
+    let mut sections: Vec<Section> = Vec::new();
+    let mut queue: VecDeque<(Vec<String>, serde_yaml_ng::Value)> = VecDeque::new();
+
+    for (key, val) in roots {
+        queue.push_back((vec![key], val));
+    }
+
+    while let Some((path, value)) = queue.pop_front() {
+        match value {
+            serde_yaml_ng::Value::Mapping(map) if !map.is_empty() => {
+                // Path is cloned per sibling; cost is O(depth) per node which
+                // is acceptable for typical config files.
+                for (k, v) in map {
+                    if let Some(key_str) = k.as_str() {
+                        let mut child_path = path.clone();
+                        child_path.push(key_str.to_string());
+                        queue.push_back((child_path, v));
+                    }
+                }
+            }
+            other => {
+                sections.push(Section {
+                    key: path.join(" > "),
+                    value: format_yaml_value(&other),
+                    // Byte offsets not tracked; parsed values lack source positions
+                    byte_start: 0,
+                    byte_end: 0,
+                });
+            }
+        }
+    }
+
+    sections
+}
+
+/// Format a YAML value as a readable string for chunk content.
+fn format_yaml_value(value: &serde_yaml_ng::Value) -> String {
+    match value {
+        serde_yaml_ng::Value::String(s) => s.clone(),
+        serde_yaml_ng::Value::Null => "null".to_string(),
+        serde_yaml_ng::Value::Bool(b) => b.to_string(),
+        serde_yaml_ng::Value::Number(n) => n.to_string(),
+        other => serde_yaml_ng::to_string(other).unwrap_or_default().trim().to_string(),
+    }
+}
+
+/// Fall back to the plain text chunker.
+fn fallback_to_text(text: &str, config: &ChunkingConfig) -> Result<ChunkingResult> {
+    let fallback_config = ChunkingConfig {
+        chunker_type: super::config::ChunkerType::Text,
+        ..config.clone()
+    };
+    super::core::chunk_text(text, &fallback_config, None)
+}
+
+/// Return the page number for a byte offset, or `None` if no boundary covers it.
+fn page_for_offset(offset: usize, boundaries: &[PageBoundary]) -> Option<u32> {
+    boundaries
+        .iter()
+        .find(|b| offset >= b.byte_start && offset < b.byte_end)
+        .map(|b| b.page_number)
+        // If not inside any boundary, use the last boundary when the offset equals
+        // the final byte_end (common when byte_start == byte_end == 0 on a single page).
+        .or_else(|| boundaries.last().map(|b| b.page_number))
+}
+
+/// Shared logic: convert Sections into Chunks, handling oversized splitting.
+fn build_chunks_from_sections(
+    sections: &[Section],
+    config: &ChunkingConfig,
+    page_boundaries: Option<&[PageBoundary]>,
+) -> Result<ChunkingResult> {
+    let mut chunks: Vec<Chunk> = Vec::new();
+
+    for section in sections {
+        let prefix = format!("# {}", section.key);
+        let content = format!("{}\n\n{}", prefix, section.value.trim());
+
+        // Derive page provenance from byte offsets against page_boundaries.
+        // YAML parsed values don't carry source positions, so byte_start/byte_end
+        // are both 0; page_for_offset returns the first (and usually only) page.
+        let (first_page, last_page) = match page_boundaries {
+            Some(b) if !b.is_empty() => (
+                page_for_offset(section.byte_start, b),
+                page_for_offset(section.byte_start, b),
+            ),
+            _ => (None, None),
+        };
+
+        if config.max_characters == 0 || content.len() <= config.max_characters {
+            chunks.push(Chunk {
+                content,
+                chunk_type: Default::default(),
+                embedding: None,
+                metadata: ChunkMetadata {
+                    byte_start: section.byte_start,
+                    byte_end: section.byte_end,
+                    token_count: None,
+                    chunk_index: 0,
+                    total_chunks: 0,
+                    first_page,
+                    last_page,
+                    heading_context: None,
+                    image_indices: Vec::new(),
+                },
+            });
+        } else {
+            let sub_result = super::core::chunk_text(
+                section.value.trim(),
+                &ChunkingConfig {
+                    chunker_type: super::config::ChunkerType::Text,
+                    ..config.clone()
+                },
+                None,
+            )?;
+            for sub_chunk in sub_result.chunks {
+                chunks.push(Chunk {
+                    content: format!("{}\n\n{}", prefix, sub_chunk.content),
+                    chunk_type: Default::default(),
+                    embedding: None,
+                    metadata: ChunkMetadata {
+                        byte_start: section.byte_start + sub_chunk.metadata.byte_start,
+                        byte_end: section.byte_start + sub_chunk.metadata.byte_end,
+                        token_count: None,
+                        chunk_index: 0,
+                        total_chunks: 0,
+                        first_page,
+                        last_page,
+                        heading_context: None,
+                        image_indices: Vec::new(),
+                    },
+                });
+            }
+        }
+    }
+
+    let total = chunks.len();
+    for (i, chunk) in chunks.iter_mut().enumerate() {
+        chunk.metadata.chunk_index = i;
+        chunk.metadata.total_chunks = total;
+    }
+
+    Ok(ChunkingResult {
+        chunk_count: total,
+        chunks,
+    })
+}
+
+struct Section {
+    key: String,
+    value: String,
+    byte_start: usize,
+    byte_end: usize,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::chunking::config::ChunkerType;
+
+    fn make_config() -> ChunkingConfig {
+        ChunkingConfig {
+            max_characters: 10000,
+            overlap: 0,
+            trim: true,
+            chunker_type: ChunkerType::Yaml,
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn test_simple_yaml() {
+        let yaml = "server:\n  host: localhost\n  port: 8080\ndb:\n  name: mydb\n  user: admin";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 4);
+        assert!(result.chunks[0].content.contains("# server > host"));
+        assert!(result.chunks[0].content.contains("localhost"));
+        assert!(result.chunks[1].content.contains("# server > port"));
+        assert!(result.chunks[2].content.contains("# db > name"));
+        assert!(result.chunks[2].content.contains("mydb"));
+        assert!(result.chunks[3].content.contains("# db > user"));
+    }
+
+    #[test]
+    fn test_nested_yaml() {
+        let yaml = "app:\n  name: test\n  config:\n    debug: true\n    log_level: info";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        // Leaf sections: app > name, app > config > debug, app > config > log_level
+        assert_eq!(result.chunk_count, 3);
+        assert!(result.chunks[0].content.contains("# app > name"));
+        assert!(result.chunks[0].content.contains("test"));
+        assert!(result.chunks[1].content.contains("# app > config > debug"));
+        assert!(result.chunks[1].content.contains("true"));
+        assert!(result.chunks[2].content.contains("# app > config > log_level"));
+        assert!(result.chunks[2].content.contains("info"));
+    }
+
+    #[test]
+    fn test_inline_values() {
+        let yaml = "name: myapp\nversion: 1.0\ndescription: A test app";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 3);
+        assert!(result.chunks[0].content.contains("# name"));
+        assert!(result.chunks[0].content.contains("myapp"));
+    }
+
+    #[test]
+    fn test_empty_yaml() {
+        let result = chunk_yaml_by_sections("", &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 0);
+    }
+
+    #[test]
+    fn test_comments_and_blanks() {
+        let yaml = "# Top comment\n\nserver:\n  port: 8080\n\n# Another comment\nclient:\n  timeout: 30";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 2);
+    }
+
+    #[test]
+    fn test_chunk_indices() {
+        let yaml = "a: 1\nb: 2\nc: 3";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        for (i, chunk) in result.chunks.iter().enumerate() {
+            assert_eq!(chunk.metadata.chunk_index, i);
+            assert_eq!(chunk.metadata.total_chunks, 3);
+        }
+    }
+
+    #[test]
+    fn test_produces_chunks_for_flat_yaml() {
+        let yaml = "first: value\nsecond: value";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 2);
+        assert!(result.chunks[0].content.contains("# first"));
+        assert!(result.chunks[1].content.contains("# second"));
+    }
+
+    #[test]
+    fn test_single_scalar() {
+        let yaml = "a: 1";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 1);
+        assert!(result.chunks[0].content.contains("# a"));
+    }
+
+    #[test]
+    fn test_array_values() {
+        let yaml = "items:\n  - one\n  - two\n  - three\nother: stuff";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 2);
+        assert!(result.chunks[0].content.contains("- one"));
+    }
+
+    #[test]
+    fn test_no_top_level_keys_falls_back() {
+        let yaml = "  indented: value\n  another: value";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert!(result.chunk_count >= 1);
+    }
+
+    #[test]
+    fn test_unicode_keys() {
+        let yaml = "\u{540d}\u{524d}: test\n\u{8a2d}\u{5b9a}:\n  key: value";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 2);
+    }
+
+    #[test]
+    fn test_multiline_string_values() {
+        let yaml = "description: |\n  This is a\n  multiline string\nother: value";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 2);
+        assert!(result.chunks[0].content.contains("multiline string"));
+    }
+
+    #[test]
+    fn test_colon_in_value() {
+        let yaml = "url: http://example.com:8080\nname: test";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 2);
+        assert!(result.chunks[0].content.contains("# url"));
+    }
+
+    #[test]
+    fn test_windows_line_endings() {
+        let yaml = "a: 1\r\nb: 2\r\nc: 3";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 3);
+        assert!(result.chunks[0].content.contains("# a"));
+        assert!(result.chunks[1].content.contains("# b"));
+        assert!(result.chunks[2].content.contains("# c"));
+    }
+
+    #[test]
+    fn test_oversized_section_split() {
+        // A single leaf with a very long value should get sub-split
+        let long_text = "word ".repeat(500);
+        let yaml = format!("description: |\n  {}\nother: ok", long_text.trim());
+        let config = ChunkingConfig {
+            max_characters: 100,
+            ..make_config()
+        };
+        let result = chunk_yaml_by_sections(&yaml, &config, None).unwrap();
+        assert!(result.chunk_count > 2);
+    }
+
+    #[test]
+    fn test_document_separator() {
+        let yaml = "---\nname: test\nversion: 1.0";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert!(result.chunk_count >= 1);
+    }
+
+    #[test]
+    fn test_single_key() {
+        let yaml = "only_key:\n  a: 1\n  b: 2";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 2);
+        assert!(result.chunks[0].content.contains("# only_key > a"));
+        assert!(result.chunks[1].content.contains("# only_key > b"));
+    }
+
+    #[test]
+    fn test_flow_style_value() {
+        let yaml = "ports: [80, 443]\nname: test";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        // ports is a leaf (array value), name is a leaf (scalar)
+        assert_eq!(result.chunk_count, 2);
+        assert!(result.chunks.iter().any(|c| c.content.contains("# ports")));
+        assert!(result.chunks.iter().any(|c| c.content.contains("# name")));
+    }
+
+    #[test]
+    fn test_chunk_indices_after_split() {
+        let yaml = "a: 1\nb: 2\nc: 3";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        for (i, chunk) in result.chunks.iter().enumerate() {
+            assert_eq!(chunk.metadata.chunk_index, i);
+            assert_eq!(chunk.metadata.total_chunks, result.chunk_count);
+        }
+    }
+
+    #[test]
+    fn test_anchor_alias() {
+        let yaml = "defaults: &defaults\n  adapter: postgres\nproduction:\n  <<: *defaults\n  host: prod-db";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        // defaults > adapter is a leaf, production has << and host as leaves
+        assert_eq!(result.chunk_count, 3);
+    }
+
+    #[test]
+    fn test_chunk_metadata_valid() {
+        let yaml = "server:\n  host: localhost\n  port: 8080\ndb:\n  name: mydb";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        for chunk in &result.chunks {
+            assert!(chunk.metadata.byte_start <= chunk.metadata.byte_end);
+            assert!(!chunk.content.is_empty());
+        }
+    }
+
+    #[test]
+    fn test_max_characters_zero_disables_splitting() {
+        let long_text = "word ".repeat(500);
+        let yaml = format!("big: |\n  {}\nsmall: ok", long_text.trim());
+        let config = ChunkingConfig {
+            max_characters: 0,
+            ..make_config()
+        };
+        let result = chunk_yaml_by_sections(&yaml, &config, None).unwrap();
+        assert_eq!(result.chunk_count, 2);
+        assert!(result.chunks.iter().any(|c| c.content.contains("# big")));
+        assert!(result.chunks.iter().any(|c| c.content.contains("# small")));
+    }
+
+    // --- New tests for nested YAML ---
+
+    #[test]
+    fn test_nested_yaml_creates_leaf_chunks() {
+        let yaml = "database:\n  primary:\n    host: db1\n  replica:\n    host: db2";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 2);
+        assert!(result.chunks[0].content.contains("# database > primary > host"));
+        assert!(result.chunks[0].content.contains("db1"));
+        assert!(result.chunks[1].content.contains("# database > replica > host"));
+        assert!(result.chunks[1].content.contains("db2"));
+    }
+
+    #[test]
+    fn test_deeply_nested_4_levels() {
+        let yaml = "a:\n  b:\n    c:\n      d: deep_value";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 1);
+        assert!(result.chunks[0].content.contains("# a > b > c > d"));
+        assert!(result.chunks[0].content.contains("deep_value"));
+    }
+
+    #[test]
+    fn test_nested_with_sibling_reset() {
+        let yaml = "parent1:\n  child_a: 1\n  child_b: 2\nparent2:\n  child_c: 3\n  child_d: 4";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 4);
+        assert!(result.chunks[0].content.contains("# parent1 > child_a"));
+        assert!(result.chunks[1].content.contains("# parent1 > child_b"));
+        assert!(result.chunks[2].content.contains("# parent2 > child_c"));
+        assert!(result.chunks[3].content.contains("# parent2 > child_d"));
+        assert!(!result.chunks[2].content.contains("parent1"));
+        assert!(!result.chunks[3].content.contains("parent1"));
+    }
+
+    // --- JSON tests ---
+
+    #[test]
+    fn test_json_object_by_keys() {
+        let json = r#"{"server": "localhost", "port": 8080, "debug": true}"#;
+        let result = chunk_yaml_by_sections(json, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 3);
+        // Key order may vary depending on JSON/YAML internal map ordering
+        let all_content: String = result
+            .chunks
+            .iter()
+            .map(|c| c.content.as_str())
+            .collect::<Vec<_>>()
+            .join("\n");
+        assert!(all_content.contains("# debug"), "should contain debug key");
+        assert!(all_content.contains("# port"), "should contain port key");
+        assert!(all_content.contains("# server"), "should contain server key");
+        assert!(all_content.contains("localhost"), "should contain localhost value");
+    }
+
+    #[test]
+    fn test_json_nested_objects() {
+        let json = r#"{"database": {"primary": {"host": "db1"}, "replica": {"host": "db2"}}}"#;
+        let result = chunk_yaml_by_sections(json, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 2);
+        assert!(result.chunks[0].content.contains("# database > primary > host"));
+        assert!(result.chunks[0].content.contains("db1"));
+        assert!(result.chunks[1].content.contains("# database > replica > host"));
+        assert!(result.chunks[1].content.contains("db2"));
+    }
+
+    #[test]
+    fn test_json_array_root_fallback() {
+        let json = r#"[1, 2, 3, 4, 5]"#;
+        let result = chunk_yaml_by_sections(json, &make_config(), None).unwrap();
+        assert!(result.chunk_count >= 1);
+        assert!(!result.chunks[0].content.starts_with("# "));
+    }
+
+    #[test]
+    fn test_json_minified() {
+        let json = r#"{"a":1,"b":{"c":2,"d":3},"e":"hello"}"#;
+        let result = chunk_yaml_by_sections(json, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 4);
+        // BFS order: a (leaf), e (leaf), then b's children (b>c, b>d)
+        assert!(result.chunks[0].content.contains("# a"));
+        assert!(result.chunks[1].content.contains("# e"));
+        assert!(result.chunks[1].content.contains("hello"));
+        assert!(result.chunks[2].content.contains("# b > c"));
+        assert!(result.chunks[3].content.contains("# b > d"));
+    }
+
+    #[test]
+    fn test_json_empty_object() {
+        let json = r#"{}"#;
+        let result = chunk_yaml_by_sections(json, &make_config(), None).unwrap();
+        // Empty object has no keys to split — falls back to text chunker
+        assert!(result.chunk_count <= 1);
+    }
+
+    #[test]
+    fn test_json_deeply_nested() {
+        let json = r#"{"a": {"b": {"c": {"d": {"e": "deep"}}}}}"#;
+        let result = chunk_yaml_by_sections(json, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 1);
+        assert!(result.chunks[0].content.contains("# a > b > c > d > e"));
+        assert!(result.chunks[0].content.contains("deep"));
+    }
+
+    #[test]
+    fn test_invalid_yaml_falls_back() {
+        let yaml = ":\n  - :\n  invalid:: yaml::: [unterminated";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        // Should not panic; falls back to text chunking
+        assert!(result.chunk_count > 0);
+    }
+
+    #[test]
+    fn test_malformed_json_falls_back() {
+        let json = r#"{"key": "unterminated"#;
+        let result = chunk_yaml_by_sections(json, &make_config(), None).unwrap();
+        // Should not panic; falls back to text chunking
+        assert!(result.chunk_count > 0);
+    }
+
+    #[test]
+    fn test_null_values_handled() {
+        let yaml = "present: value\nmissing:\nempty: \"\"";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 3);
+        assert!(result.chunks[0].content.contains("# present"));
+        assert!(result.chunks[1].content.contains("# missing"));
+        assert!(result.chunks[2].content.contains("# empty"));
+    }
+
+    #[test]
+    fn test_yaml_scalar_root_falls_back() {
+        // Root is a scalar, not a mapping — should fall back
+        let yaml = "just a plain string";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert!(result.chunk_count >= 1);
+        assert!(!result.chunks[0].content.starts_with("# "));
+    }
+
+    #[test]
+    fn test_yaml_list_root_falls_back() {
+        let yaml = "- item1\n- item2\n- item3";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert!(result.chunk_count >= 1);
+        assert!(!result.chunks[0].content.starts_with("# "));
+    }
+
+    #[test]
+    fn test_json_null_value() {
+        let json = r#"{"key": null, "other": "value"}"#;
+        let result = chunk_yaml_by_sections(json, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 2);
+        assert!(result.chunks.iter().any(|c| c.content.contains("# key")));
+        assert!(result.chunks.iter().any(|c| c.content.contains("# other")));
+    }
+
+    #[test]
+    fn test_boolean_leaf_values() {
+        let yaml = "debug: true\nverbose: false";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        assert_eq!(result.chunk_count, 2);
+        assert!(result.chunks[0].content.contains("true"));
+        assert!(result.chunks[1].content.contains("false"));
+    }
+
+    #[test]
+    fn test_page_bounds_none_without_boundaries() {
+        let yaml = "server:\n  host: localhost\n  port: 8080";
+        let result = chunk_yaml_by_sections(yaml, &make_config(), None).unwrap();
+        for chunk in &result.chunks {
+            assert_eq!(chunk.metadata.first_page, None);
+            assert_eq!(chunk.metadata.last_page, None);
+        }
+    }
+
+    #[test]
+    fn test_page_bounds_set_from_single_boundary() {
+        let yaml = "server:\n  host: localhost\n  port: 8080";
+        let boundaries = vec![PageBoundary {
+            byte_start: 0,
+            byte_end: yaml.len(),
+            page_number: 3,
+        }];
+        let result = chunk_yaml_by_sections(yaml, &make_config(), Some(&boundaries)).unwrap();
+        assert_eq!(result.chunk_count, 2);
+        for chunk in &result.chunks {
+            assert_eq!(chunk.metadata.first_page, Some(3), "first_page should be page 3");
+            assert_eq!(chunk.metadata.last_page, Some(3), "last_page should be page 3");
+        }
+    }
+
+    #[test]
+    fn test_page_bounds_set_for_oversized_sub_chunks() {
+        let long_text = "word ".repeat(500);
+        let yaml = format!("description: |\n  {}\nother: ok", long_text.trim());
+        let config = ChunkingConfig {
+            max_characters: 100,
+            ..make_config()
+        };
+        let boundaries = vec![PageBoundary {
+            byte_start: 0,
+            byte_end: yaml.len(),
+            page_number: 7,
+        }];
+        let result = chunk_yaml_by_sections(&yaml, &config, Some(&boundaries)).unwrap();
+        assert!(result.chunk_count > 1);
+        for chunk in &result.chunks {
+            assert_eq!(chunk.metadata.first_page, Some(7));
+            assert_eq!(chunk.metadata.last_page, Some(7));
+        }
+    }
+
+    #[test]
+    fn test_empty_boundaries_slice_leaves_pages_none() {
+        let yaml = "a: 1\nb: 2";
+        let boundaries: Vec<PageBoundary> = vec![];
+        let result = chunk_yaml_by_sections(yaml, &make_config(), Some(&boundaries)).unwrap();
+        for chunk in &result.chunks {
+            assert_eq!(chunk.metadata.first_page, None);
+            assert_eq!(chunk.metadata.last_page, None);
+        }
+    }
+}

crates/kreuzberg/src/core/batch_mode.rs

@@ -0,0 +1,110 @@
+//! Internal batch mode tracking using tokio task-local storage.
+//!
+//! This module provides a way to track whether we're in batch processing mode
+//! without exposing it in the public API. Extractors check this flag to decide
+//! whether to use `spawn_blocking` for CPU-intensive work.
+
+use std::cell::Cell;
+use tokio::task_local;
+
+task_local! {
+    /// Task-local flag indicating batch processing mode.
+    ///
+    /// When true, extractors use `spawn_blocking` for CPU-intensive work to enable
+    /// parallelism. When false (single-file mode), extractors run directly to avoid
+    /// spawn overhead.
+    static BATCH_MODE: Cell<bool>;
+}
+
+/// Check if we're currently in batch processing mode.
+///
+/// Returns `false` if the task-local is not set (single-file mode).
+#[cfg(any(
+    feature = "pdf",
+    feature = "office",
+    feature = "excel",
+    feature = "excel-wasm",
+    feature = "archives"
+))]
+pub(crate) fn is_batch_mode() -> bool {
+    BATCH_MODE.try_with(|cell| cell.get()).unwrap_or(false)
+}
+
+/// Run a future with batch mode enabled.
+///
+/// This sets the task-local BATCH_MODE flag for the duration of the future.
+pub(crate) async fn with_batch_mode<F, T>(future: F) -> T
+where
+    F: std::future::Future<Output = T>,
+{
+    BATCH_MODE.scope(Cell::new(true), future).await
+}
+
+#[cfg(all(
+    test,
+    any(
+        feature = "pdf",
+        feature = "office",
+        feature = "excel",
+        feature = "excel-wasm",
+        feature = "archives"
+    )
+))]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_batch_mode_not_set_by_default() {
+        let result = is_batch_mode();
+        assert!(!result, "batch mode should be false by default");
+    }
+
+    #[tokio::test]
+    async fn test_with_batch_mode_sets_flag() {
+        let result = with_batch_mode(async { is_batch_mode() }).await;
+
+        assert!(result, "batch mode should be true inside with_batch_mode");
+    }
+
+    #[tokio::test]
+    async fn test_batch_mode_scoped_to_future() {
+        assert!(!is_batch_mode(), "batch mode should be false before");
+
+        with_batch_mode(async {
+            assert!(is_batch_mode(), "batch mode should be true inside");
+        })
+        .await;
+
+        assert!(!is_batch_mode(), "batch mode should be false after future completes");
+    }
+
+    #[tokio::test]
+    async fn test_nested_batch_mode_calls() {
+        let result = with_batch_mode(async {
+            let outer = is_batch_mode();
+            let inner = with_batch_mode(async { is_batch_mode() }).await;
+            (outer, inner)
+        })
+        .await;
+
+        assert!(result.0, "outer batch mode should be true");
+        assert!(result.1, "inner batch mode should be true");
+    }
+
+    #[tokio::test]
+    async fn test_batch_mode_unaffected_after_with_batch_mode() {
+        with_batch_mode(async {
+            assert!(is_batch_mode(), "first call should set batch mode");
+        })
+        .await;
+
+        assert!(!is_batch_mode(), "batch mode should be false between calls");
+
+        with_batch_mode(async {
+            assert!(is_batch_mode(), "second call should set batch mode");
+        })
+        .await;
+
+        assert!(!is_batch_mode(), "batch mode should be false after all calls");
+    }
+}

crates/kreuzberg/src/core/batch_optimizations.rs

@@ -0,0 +1,315 @@
+//! Batch extraction optimizations using object pooling.
+//!
+//! This module provides optimized batch processing utilities that leverage
+//! object pooling to reduce allocations during concurrent extraction of
+//! multiple documents.
+//!
+//! # Performance Impact
+//!
+//! - Reuses temporary string/buffer allocations across documents
+//! - Reduces garbage collection pressure by ~5-10%
+//! - Overall throughput improvement of 5-10% for batch operations
+//!
+//! # Usage
+//!
+//! The batch extraction functions automatically use pooling internally.
+//! For manual control, use `BatchProcessor` to create pools and manage
+//! extraction with custom pool sizes.
+
+use crate::utils::pool::{ByteBufferPool, StringBufferPool, create_byte_buffer_pool, create_string_buffer_pool};
+use crate::utils::pool_sizing::PoolSizeHint;
+use parking_lot::Mutex;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+/// Configuration for batch processing with pooling optimizations.
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone)]
+pub struct BatchProcessorConfig {
+    /// Maximum number of string buffers to maintain in the pool
+    pub string_pool_size: usize,
+
+    /// Initial capacity for pooled string buffers in bytes
+    pub string_buffer_capacity: usize,
+
+    /// Maximum number of byte buffers to maintain in the pool
+    pub byte_pool_size: usize,
+
+    /// Initial capacity for pooled byte buffers in bytes
+    pub byte_buffer_capacity: usize,
+
+    /// Maximum concurrent extractions (for concurrency control)
+    pub max_concurrent: Option<usize>,
+}
+
+impl Default for BatchProcessorConfig {
+    fn default() -> Self {
+        BatchProcessorConfig {
+            string_pool_size: 10,
+            string_buffer_capacity: 8192,
+            byte_pool_size: 10,
+            byte_buffer_capacity: 65536,
+            max_concurrent: None,
+        }
+    }
+}
+
+/// Batch processor that manages object pools for optimized extraction.
+///
+/// This struct manages the lifecycle of reusable object pools used during
+/// batch extraction. Pools are created lazily on first use and reused across
+/// all documents processed by this batch processor.
+///
+/// # Lazy Initialization
+///
+/// Pools are initialized on demand to reduce memory usage for applications
+/// that may not use batch processing immediately or at all.
+#[cfg_attr(alef, alef(skip))]
+pub struct BatchProcessor {
+    string_pool: Mutex<Option<Arc<StringBufferPool>>>,
+    byte_pool: Mutex<Option<Arc<ByteBufferPool>>>,
+    config: BatchProcessorConfig,
+    string_pool_initialized: AtomicBool,
+    byte_pool_initialized: AtomicBool,
+}
+
+impl BatchProcessor {
+    /// Create a new batch processor with default pool configuration.
+    ///
+    /// # Returns
+    ///
+    /// A new `BatchProcessor` ready to process documents.
+    ///
+    /// # Example
+    ///
+    /// ```rust,no_run
+    /// use kreuzberg::core::batch_optimizations::BatchProcessor;
+    ///
+    /// let processor = BatchProcessor::new();
+    /// ```
+    pub fn new() -> Self {
+        Self::with_config(BatchProcessorConfig::default())
+    }
+
+    /// Create a new batch processor with custom pool configuration.
+    ///
+    /// Pools are not created immediately but lazily on first access.
+    ///
+    /// # Arguments
+    ///
+    /// * `config` - Custom batch processor configuration
+    ///
+    /// # Returns
+    ///
+    /// A new `BatchProcessor` configured with the provided settings.
+    ///
+    /// # Example
+    ///
+    /// ```rust,no_run
+    /// use kreuzberg::core::batch_optimizations::{BatchProcessor, BatchProcessorConfig};
+    ///
+    /// let mut config = BatchProcessorConfig::default();
+    /// config.string_pool_size = 20;
+    /// config.string_buffer_capacity = 16384;
+    /// let processor = BatchProcessor::with_config(config);
+    /// ```
+    pub fn with_config(config: BatchProcessorConfig) -> Self {
+        BatchProcessor {
+            string_pool: Mutex::new(None),
+            byte_pool: Mutex::new(None),
+            config,
+            string_pool_initialized: AtomicBool::new(false),
+            byte_pool_initialized: AtomicBool::new(false),
+        }
+    }
+
+    /// Create a batch processor with pool sizes optimized for a specific document.
+    ///
+    /// This method uses a `PoolSizeHint` (derived from file size and MIME type)
+    /// to create a batch processor with appropriately sized pools. This reduces
+    /// memory waste by tailoring pool allocation to actual document complexity.
+    ///
+    /// # Arguments
+    ///
+    /// * `hint` - Pool sizing hint containing recommended buffer counts and capacities
+    ///
+    /// # Returns
+    ///
+    /// A new `BatchProcessor` configured with the hint-based pool sizes
+    ///
+    /// # Example
+    ///
+    /// ```ignore
+    /// use kreuzberg::core::batch_optimizations::BatchProcessor;
+    /// use kreuzberg::utils::pool_sizing::estimate_pool_size;
+    ///
+    /// let hint = estimate_pool_size(5_000_000, "application/pdf");
+    /// let processor = BatchProcessor::with_pool_hint(&hint);
+    /// ```
+    pub fn with_pool_hint(hint: &PoolSizeHint) -> Self {
+        let config = BatchProcessorConfig {
+            string_pool_size: hint.string_buffer_count,
+            string_buffer_capacity: hint.string_buffer_capacity,
+            byte_pool_size: hint.byte_buffer_count,
+            byte_buffer_capacity: hint.byte_buffer_capacity,
+            max_concurrent: None,
+        };
+        Self::with_config(config)
+    }
+
+    /// Get a reference to the string buffer pool.
+    ///
+    /// Creates the pool lazily on first access.
+    /// Useful for custom pooling implementations that need direct pool access.
+    pub fn string_pool(&self) -> Arc<StringBufferPool> {
+        if self.string_pool_initialized.load(Ordering::Acquire) {
+            return Arc::clone(self.string_pool.lock().as_ref().unwrap());
+        }
+
+        let mut pool_opt = self.string_pool.lock();
+        if pool_opt.is_none() {
+            let pool = Arc::new(create_string_buffer_pool(
+                self.config.string_pool_size,
+                self.config.string_buffer_capacity,
+            ));
+            *pool_opt = Some(pool);
+            self.string_pool_initialized.store(true, Ordering::Release);
+        }
+
+        Arc::clone(pool_opt.as_ref().unwrap())
+    }
+
+    /// Get a reference to the byte buffer pool.
+    ///
+    /// Creates the pool lazily on first access.
+    /// Useful for custom pooling implementations that need direct pool access.
+    pub fn byte_pool(&self) -> Arc<ByteBufferPool> {
+        if self.byte_pool_initialized.load(Ordering::Acquire) {
+            return Arc::clone(self.byte_pool.lock().as_ref().unwrap());
+        }
+
+        let mut pool_opt = self.byte_pool.lock();
+        if pool_opt.is_none() {
+            let pool = Arc::new(create_byte_buffer_pool(
+                self.config.byte_pool_size,
+                self.config.byte_buffer_capacity,
+            ));
+            *pool_opt = Some(pool);
+            self.byte_pool_initialized.store(true, Ordering::Release);
+        }
+
+        Arc::clone(pool_opt.as_ref().unwrap())
+    }
+
+    /// Get the current configuration.
+    pub fn config(&self) -> &BatchProcessorConfig {
+        &self.config
+    }
+
+    /// Get the number of pooled string buffers currently available.
+    pub fn string_pool_size(&self) -> usize {
+        self.string_pool.lock().as_ref().map(|p| p.size()).unwrap_or(0)
+    }
+
+    /// Get the number of pooled byte buffers currently available.
+    pub fn byte_pool_size(&self) -> usize {
+        self.byte_pool.lock().as_ref().map(|p| p.size()).unwrap_or(0)
+    }
+
+    /// Clear all pooled objects, forcing new allocations on next acquire.
+    ///
+    /// Useful for memory-constrained environments or to reclaim memory
+    /// after processing large batches.
+    pub fn clear_pools(&self) {
+        if let Some(pool) = self.string_pool.lock().as_ref() {
+            pool.clear();
+        }
+        if let Some(pool) = self.byte_pool.lock().as_ref() {
+            pool.clear();
+        }
+    }
+}
+
+impl Default for BatchProcessor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_batch_processor_creation() {
+        let processor = BatchProcessor::new();
+        assert_eq!(processor.string_pool_size(), 0);
+        assert_eq!(processor.byte_pool_size(), 0);
+    }
+
+    #[test]
+    fn test_batch_processor_with_config() {
+        let config = BatchProcessorConfig {
+            string_pool_size: 5,
+            string_buffer_capacity: 1024,
+            byte_pool_size: 3,
+            byte_buffer_capacity: 4096,
+            max_concurrent: None,
+        };
+
+        let processor = BatchProcessor::with_config(config);
+        assert_eq!(processor.config().string_pool_size, 5);
+        assert_eq!(processor.config().byte_pool_size, 3);
+    }
+
+    #[test]
+    fn test_batch_processor_string_pool_usage() {
+        let processor = BatchProcessor::new();
+        let pool = processor.string_pool();
+
+        {
+            let mut s = pool.acquire();
+            s.push_str("test");
+        }
+
+        {
+            let s = pool.acquire();
+            assert_eq!(s.len(), 0);
+        }
+    }
+
+    #[test]
+    fn test_batch_processor_byte_pool_usage() {
+        let processor = BatchProcessor::new();
+        let pool = processor.byte_pool();
+
+        {
+            let mut buf = pool.acquire();
+            buf.extend_from_slice(b"test");
+        }
+
+        {
+            let buf = pool.acquire();
+            assert_eq!(buf.len(), 0);
+        }
+    }
+
+    #[test]
+    fn test_batch_processor_clear_pools() {
+        let processor = BatchProcessor::new();
+
+        let s1 = processor.string_pool().acquire();
+        let s2 = processor.byte_pool().acquire();
+
+        drop(s1);
+        drop(s2);
+
+        assert!(processor.string_pool_size() > 0);
+        assert!(processor.byte_pool_size() > 0);
+
+        processor.clear_pools();
+
+        assert_eq!(processor.string_pool_size(), 0);
+        assert_eq!(processor.byte_pool_size(), 0);
+    }
+}

crates/kreuzberg/src/core/config/acceleration.rs

@@ -0,0 +1,55 @@
+//! Acceleration configuration for ONNX Runtime execution providers.
+
+use serde::{Deserialize, Serialize};
+
+/// Hardware acceleration configuration for ONNX Runtime models.
+///
+/// Controls which execution provider (CPU, CoreML, CUDA, TensorRT) is used
+/// for inference in layout detection and embedding generation.
+///
+/// # Example
+///
+/// ```rust
+/// use kreuzberg::AccelerationConfig;
+///
+/// // Auto-select: CoreML on macOS, CUDA on Linux, CPU elsewhere
+/// let config = AccelerationConfig::default();
+///
+/// // Force CPU only
+/// let config = AccelerationConfig {
+///     provider: kreuzberg::ExecutionProviderType::Cpu,
+///     ..Default::default()
+/// };
+/// ```
+#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)]
+pub struct AccelerationConfig {
+    /// Execution provider to use for ONNX inference.
+    #[serde(default)]
+    pub provider: ExecutionProviderType,
+
+    /// GPU device ID (for CUDA/TensorRT). Ignored for CPU/CoreML/Auto.
+    #[serde(default)]
+    pub device_id: u32,
+}
+
+/// ONNX Runtime execution provider type.
+///
+/// Determines which hardware backend is used for model inference.
+/// `Auto` (default) selects the best available provider per platform.
+#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)]
+#[serde(rename_all = "lowercase")]
+pub enum ExecutionProviderType {
+    /// Auto-select: CoreML on macOS, CUDA on Linux, CPU elsewhere.
+    #[default]
+    Auto,
+    /// CPU execution provider (always available).
+    Cpu,
+    /// Apple CoreML (macOS/iOS Neural Engine + GPU).
+    #[serde(alias = "coreml")]
+    CoreMl,
+    /// NVIDIA CUDA GPU acceleration.
+    Cuda,
+    /// NVIDIA TensorRT (optimized CUDA inference).
+    #[serde(alias = "tensorrt")]
+    TensorRt,
+}

crates/kreuzberg/src/core/config/concurrency.rs

@@ -0,0 +1,140 @@
+//! Concurrency and thread pool configuration.
+
+use std::sync::Once;
+
+use serde::{Deserialize, Serialize};
+
+/// Controls thread usage for constrained environments.
+///
+/// Set `max_threads` to cap all internal thread pools (Rayon, ONNX Runtime
+/// intra-op) and batch concurrency to a single limit.
+///
+/// # Example
+///
+/// ```rust
+/// use kreuzberg::core::config::ConcurrencyConfig;
+///
+/// let config = ConcurrencyConfig {
+///     max_threads: Some(2),
+/// };
+/// ```
+#[cfg_attr(alef, alef(skip))]
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[serde(default)]
+pub struct ConcurrencyConfig {
+    /// Maximum number of threads for all internal thread pools.
+    ///
+    /// Caps Rayon global pool size, ONNX Runtime intra-op threads, and
+    /// (when `max_concurrent_extractions` is unset) the batch concurrency
+    /// semaphore. When `None`, system defaults are used.
+    pub max_threads: Option<usize>,
+}
+
+static POOL_INIT: Once = Once::new();
+
+/// Resolve the effective thread budget from config or auto-detection.
+///
+/// User-set `max_threads` takes priority. Otherwise auto-detects from `num_cpus`,
+/// capped at 8 for sane defaults in serverless environments.
+///
+/// # Example
+///
+/// ```ignore
+/// use kreuzberg::core::config::ConcurrencyConfig;
+/// use kreuzberg::core::config::concurrency::resolve_thread_budget;
+///
+/// let config = ConcurrencyConfig { max_threads: Some(4) };
+/// assert_eq!(resolve_thread_budget(Some(&config)), 4);
+/// assert!(resolve_thread_budget(None) >= 1);
+/// ```
+pub(crate) fn resolve_thread_budget(config: Option<&ConcurrencyConfig>) -> usize {
+    if let Some(n) = config.and_then(|c| c.max_threads) {
+        return n.max(1);
+    }
+    num_cpus::get().min(8)
+}
+
+/// Initialize the global Rayon thread pool with the given budget.
+///
+/// Safe to call multiple times — only the first call takes effect (subsequent
+/// calls are silently ignored).
+///
+/// # Example
+///
+/// ```ignore
+/// use kreuzberg::core::config::concurrency::init_thread_pools;
+///
+/// init_thread_pools(4);
+/// init_thread_pools(2); // no-op: pool already initialized
+/// ```
+pub(crate) fn init_thread_pools(budget: usize) {
+    POOL_INIT.call_once(|| {
+        #[cfg(not(target_arch = "wasm32"))]
+        rayon::ThreadPoolBuilder::new().num_threads(budget).build_global().ok();
+        #[cfg(target_arch = "wasm32")]
+        let _ = budget;
+    });
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_resolve_thread_budget_none() {
+        let budget = resolve_thread_budget(None);
+        assert!(budget >= 1);
+        assert!(budget <= 8);
+    }
+
+    #[test]
+    fn test_resolve_thread_budget_with_config() {
+        let config = ConcurrencyConfig { max_threads: Some(4) };
+        assert_eq!(resolve_thread_budget(Some(&config)), 4);
+    }
+
+    #[test]
+    fn test_resolve_thread_budget_clamps_to_one() {
+        let config = ConcurrencyConfig { max_threads: Some(0) };
+        assert_eq!(resolve_thread_budget(Some(&config)), 1);
+    }
+
+    #[test]
+    fn test_resolve_thread_budget_no_max() {
+        let config = ConcurrencyConfig { max_threads: None };
+        let budget = resolve_thread_budget(Some(&config));
+        assert!(budget >= 1);
+        assert!(budget <= 8);
+    }
+
+    #[test]
+    fn test_init_thread_pools_idempotent() {
+        // Should not panic when called multiple times.
+        init_thread_pools(2);
+        init_thread_pools(4);
+    }
+
+    #[test]
+    fn test_default() {
+        let config = ConcurrencyConfig::default();
+        assert!(config.max_threads.is_none());
+    }
+
+    #[test]
+    fn test_serde_roundtrip() {
+        let json = r#"{"max_threads": 2}"#;
+        let config: ConcurrencyConfig = serde_json::from_str(json).unwrap();
+        assert_eq!(config.max_threads, Some(2));
+
+        let serialized = serde_json::to_string(&config).unwrap();
+        let roundtripped: ConcurrencyConfig = serde_json::from_str(&serialized).unwrap();
+        assert_eq!(roundtripped.max_threads, Some(2));
+    }
+
+    #[test]
+    fn test_serde_empty() {
+        let json = r#"{}"#;
+        let config: ConcurrencyConfig = serde_json::from_str(json).unwrap();
+        assert!(config.max_threads.is_none());
+    }
+}

crates/kreuzberg/src/core/config/content_filter.rs

@@ -0,0 +1,80 @@
+//! Cross-extractor content filtering configuration.
+
+use serde::{Deserialize, Serialize};
+
+fn default_true() -> bool {
+    true
+}
+
+/// Cross-extractor content filtering configuration.
+///
+/// Controls whether "furniture" content (headers, footers, page numbers,
+/// watermarks, repeating text) is included in or stripped from extraction
+/// results. Applies across all extractors (PDF, DOCX, RTF, ODT, HTML, etc.)
+/// with format-specific implementation.
+///
+/// When `None` on `ExtractionConfig`, each extractor uses its current
+/// default behavior unchanged.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ContentFilterConfig {
+    /// Include running headers in extraction output.
+    ///
+    /// - PDF: Disables top-margin furniture stripping and prevents the layout
+    ///   model from treating `PageHeader`-classified regions as furniture.
+    /// - DOCX: Includes document headers in text output.
+    /// - RTF/ODT: Headers already included; this is a no-op when true.
+    /// - HTML/EPUB: Keeps `<header>` element content.
+    ///
+    /// Default: `false` (headers are stripped or excluded).
+    #[serde(default)]
+    pub include_headers: bool,
+
+    /// Include running footers in extraction output.
+    ///
+    /// - PDF: Disables bottom-margin furniture stripping and prevents the layout
+    ///   model from treating `PageFooter`-classified regions as furniture.
+    /// - DOCX: Includes document footers in text output.
+    /// - RTF/ODT: Footers already included; this is a no-op when true.
+    /// - HTML/EPUB: Keeps `<footer>` element content.
+    ///
+    /// Default: `false` (footers are stripped or excluded).
+    #[serde(default)]
+    pub include_footers: bool,
+
+    /// Enable the heuristic cross-page repeating text detector.
+    ///
+    /// When `true` (default), text that repeats verbatim across a supermajority
+    /// of pages is classified as furniture and stripped.  Disable this if brand
+    /// names or repeated headings are being incorrectly removed by the heuristic.
+    ///
+    /// Note: when a layout-detection model is active, the model may independently
+    /// classify page-header / page-footer regions as furniture on a per-page basis.
+    /// To preserve those regions, set `include_headers = true`, `include_footers = true`,
+    /// or both, in addition to disabling this flag.
+    ///
+    /// Primarily affects PDF extraction.
+    ///
+    /// Default: `true`.
+    #[serde(default = "default_true")]
+    pub strip_repeating_text: bool,
+
+    /// Include watermark text in extraction output.
+    ///
+    /// - PDF: Keeps watermark artifacts and arXiv identifiers.
+    /// - Other formats: No effect currently.
+    ///
+    /// Default: `false` (watermarks are stripped).
+    #[serde(default)]
+    pub include_watermarks: bool,
+}
+
+impl Default for ContentFilterConfig {
+    fn default() -> Self {
+        Self {
+            include_headers: false,
+            include_footers: false,
+            strip_repeating_text: true,
+            include_watermarks: false,
+        }
+    }
+}

crates/kreuzberg/src/core/config/email.rs

@@ -0,0 +1,58 @@
+//! Email extraction configuration.
+
+use serde::{Deserialize, Serialize};
+
+/// Configuration for email extraction.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default)]
+#[derive(Default)]
+pub struct EmailConfig {
+    /// Windows codepage number to use when an MSG file contains no codepage property.
+    /// Defaults to `None`, which falls back to windows-1252.
+    ///
+    /// If an unrecognized or invalid codepage number is supplied (including 0),
+    /// the behavior silently falls back to windows-1252 — the same as when the
+    /// MSG file itself contains an unrecognized codepage. No error or warning is
+    /// emitted. Users should verify output when supplying unusual values.
+    ///
+    /// Common values:
+    /// - 1250: Central European (Polish, Czech, Hungarian, etc.)
+    /// - 1251: Cyrillic (Russian, Ukrainian, Bulgarian, etc.)
+    /// - 1252: Western European (default)
+    /// - 1253: Greek
+    /// - 1254: Turkish
+    /// - 1255: Hebrew
+    /// - 1256: Arabic
+    /// - 932:  Japanese (Shift-JIS)
+    /// - 936:  Simplified Chinese (GBK)
+    pub msg_fallback_codepage: Option<u32>,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_email_config_default() {
+        let config = EmailConfig::default();
+        assert!(config.msg_fallback_codepage.is_none());
+    }
+
+    #[test]
+    fn test_email_config_serde_roundtrip() {
+        let json = r#"{"msg_fallback_codepage": 1251}"#;
+        let config: EmailConfig = serde_json::from_str(json).unwrap();
+        assert_eq!(config.msg_fallback_codepage, Some(1251));
+
+        let serialized = serde_json::to_string(&config).unwrap();
+        let roundtripped: EmailConfig = serde_json::from_str(&serialized).unwrap();
+        assert_eq!(roundtripped.msg_fallback_codepage, Some(1251));
+    }
+
+    #[test]
+    fn test_email_config_serde_default_omitted() {
+        let json = r#"{}"#;
+        let config: EmailConfig = serde_json::from_str(json).unwrap();
+        assert!(config.msg_fallback_codepage.is_none());
+    }
+}

crates/kreuzberg/src/core/config/extraction/core.rs

@@ -0,0 +1,728 @@
+//! Main extraction configuration struct.
+//!
+//! This module contains the main `ExtractionConfig` struct that aggregates all
+//! configuration options for the extraction process.
+
+use serde::{Deserialize, Serialize};
+
+use super::super::acceleration::AccelerationConfig;
+use super::super::content_filter::ContentFilterConfig;
+use super::super::formats::OutputFormat;
+use super::super::ocr::OcrConfig;
+use super::super::page::PageConfig;
+use super::super::processing::{ChunkingConfig, PostProcessorConfig};
+use super::file_config::FileExtractionConfig;
+use super::types::{ImageExtractionConfig, LanguageDetectionConfig, TokenReductionOptions};
+
+/// Main extraction configuration.
+///
+/// This struct contains all configuration options for the extraction process.
+/// It can be loaded from TOML, YAML, or JSON files, or created programmatically.
+///
+/// # Example
+///
+/// ```rust
+/// use kreuzberg::core::config::ExtractionConfig;
+///
+/// // Create with defaults
+/// let config = ExtractionConfig::default();
+///
+/// // Load from TOML file
+/// // let config = ExtractionConfig::from_toml_file("kreuzberg.toml")?;
+/// ```
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct ExtractionConfig {
+    /// Enable caching of extraction results
+    #[serde(default = "default_true")]
+    pub use_cache: bool,
+
+    /// Enable quality post-processing
+    #[serde(default = "default_true")]
+    pub enable_quality_processing: bool,
+
+    /// OCR configuration (None = OCR disabled)
+    #[serde(default)]
+    pub ocr: Option<OcrConfig>,
+
+    /// Force OCR even for searchable PDFs
+    #[serde(default)]
+    pub force_ocr: bool,
+
+    /// Force OCR on specific pages only (1-indexed page numbers, must be >= 1).
+    ///
+    /// When set, only the listed pages are OCR'd regardless of text layer quality.
+    /// Unlisted pages use native text extraction. Ignored when `force_ocr` is `true`.
+    /// Only applies to PDF documents. Duplicates are automatically deduplicated.
+    /// An `ocr` config is recommended for backend/language selection; defaults are used if absent.
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub force_ocr_pages: Option<Vec<u32>>,
+
+    /// Disable OCR entirely, even for images.
+    ///
+    /// When `true`, OCR is skipped for all document types. Images return metadata
+    /// only (dimensions, format, EXIF) without text extraction. PDFs use only
+    /// native text extraction without OCR fallback.
+    ///
+    /// Cannot be `true` simultaneously with `force_ocr`.
+    ///
+    /// *Added in v4.7.0.*
+    #[serde(default)]
+    pub disable_ocr: bool,
+
+    /// Text chunking configuration (None = chunking disabled)
+    #[serde(default)]
+    pub chunking: Option<ChunkingConfig>,
+
+    /// Content filtering configuration (None = use extractor defaults).
+    ///
+    /// Controls whether document "furniture" (headers, footers, watermarks,
+    /// repeating text) is included in or stripped from extraction results.
+    /// See [`ContentFilterConfig`] for per-field documentation.
+    #[serde(default)]
+    pub content_filter: Option<ContentFilterConfig>,
+
+    /// Image extraction configuration (None = no image extraction)
+    #[serde(default)]
+    pub images: Option<ImageExtractionConfig>,
+
+    /// PDF-specific options (None = use defaults)
+    #[cfg(feature = "pdf")]
+    #[serde(default)]
+    pub pdf_options: Option<super::super::pdf::PdfConfig>,
+
+    /// Token reduction configuration (None = no token reduction)
+    #[serde(default)]
+    pub token_reduction: Option<TokenReductionOptions>,
+
+    /// Language detection configuration (None = no language detection)
+    #[serde(default)]
+    pub language_detection: Option<LanguageDetectionConfig>,
+
+    /// Page extraction configuration (None = no page tracking)
+    #[serde(default)]
+    pub pages: Option<PageConfig>,
+
+    /// Keyword extraction configuration (None = no keyword extraction)
+    #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
+    #[serde(default)]
+    pub keywords: Option<crate::keywords::KeywordConfig>,
+
+    /// Post-processor configuration (None = use defaults)
+    #[serde(default)]
+    pub postprocessor: Option<PostProcessorConfig>,
+
+    /// HTML to Markdown conversion options (None = use defaults)
+    ///
+    /// Configure how HTML documents are converted to Markdown, including heading styles,
+    /// list formatting, code block styles, and preprocessing options.
+    #[cfg(feature = "html")]
+    #[serde(default)]
+    pub html_options: Option<html_to_markdown_rs::ConversionOptions>,
+
+    /// Styled HTML output configuration.
+    ///
+    /// When set alongside `output_format = OutputFormat::Html`, the extraction
+    /// pipeline uses [`StyledHtmlRenderer`](crate::rendering::StyledHtmlRenderer)
+    /// which emits stable `kb-*` CSS class hooks on every structural element
+    /// and optionally embeds theme CSS or user-supplied CSS in a `<style>` block.
+    ///
+    /// When `None`, the existing plain comrak-based HTML renderer is used.
+    #[cfg(feature = "html")]
+    #[serde(default)]
+    pub html_output: Option<crate::core::config::html_output::HtmlOutputConfig>,
+
+    /// Default per-file timeout in seconds for batch extraction.
+    ///
+    /// When set, each file in a batch will be canceled after this duration
+    /// unless overridden by [`FileExtractionConfig::timeout_secs`].
+    ///
+    /// Defaults to `Some(60)` to prevent pathological files (e.g. deeply
+    /// nested archives, documents with millions of cells) from running
+    /// indefinitely and exhausting caller resources. Set to `None` to
+    /// disable the timeout for trusted input or long-running workloads.
+    #[serde(default = "default_extraction_timeout")]
+    pub extraction_timeout_secs: Option<u64>,
+
+    /// Maximum concurrent extractions in batch operations (None = (num_cpus × 1.5).ceil()).
+    ///
+    /// Limits parallelism to prevent resource exhaustion when processing
+    /// large batches. Defaults to (num_cpus × 1.5).ceil() when not set.
+    #[serde(default)]
+    pub max_concurrent_extractions: Option<usize>,
+
+    /// Result structure format
+    ///
+    /// Controls whether results are returned in unified format (default) with all
+    /// content in the `content` field, or element-based format with semantic
+    /// elements (for Unstructured-compatible output).
+    #[serde(default)]
+    pub result_format: crate::types::ResultFormat,
+
+    /// Security limits for archive extraction.
+    ///
+    /// Controls maximum archive size, compression ratio, file count, and other
+    /// security thresholds to prevent decompression bomb attacks. Also caps
+    /// nesting depth, iteration count, entity / token length, total
+    /// content size, and table cell count for every extraction path that
+    /// ingests user-controlled bytes.
+    /// When `None`, default limits are used.
+    #[serde(default)]
+    pub security_limits: Option<crate::extractors::security::SecurityLimits>,
+
+    /// Maximum uncompressed size in bytes for a single embedded file before
+    /// recursive extraction is attempted (default: 50 MiB).
+    ///
+    /// Applies to embedded objects inside OOXML containers (DOCX, PPTX) and
+    /// to email attachments processed via recursive extraction. Files that
+    /// exceed this limit are skipped with a `ProcessingWarning` rather than
+    /// passed to the extraction pipeline, preventing a single oversized
+    /// embedded object from consuming unbounded memory or time.
+    ///
+    /// Set to `None` to disable the per-embedded-file cap (falls back to
+    /// `security_limits.max_archive_size` as the only guard).
+    #[serde(default = "default_max_embedded_file_bytes")]
+    pub max_embedded_file_bytes: Option<u64>,
+
+    /// Content text format (default: Plain).
+    ///
+    /// Controls the format of the extracted content:
+    /// - `Plain`: Raw extracted text (default)
+    /// - `Markdown`: Markdown formatted output
+    /// - `Djot`: Djot markup format (requires djot feature)
+    /// - `Html`: HTML formatted output
+    ///
+    /// When set to a structured format, extraction results will include
+    /// formatted output. The `formatted_content` field may be populated
+    /// when format conversion is applied.
+    #[serde(default)]
+    pub output_format: OutputFormat,
+
+    /// Layout detection configuration (None = layout detection disabled).
+    ///
+    /// When set, PDF pages and images are analyzed for document structure
+    /// (headings, code, formulas, tables, figures, etc.) using RT-DETR models
+    /// via ONNX Runtime. For PDFs, layout hints override paragraph classification
+    /// in the markdown pipeline. For images, per-region OCR is performed with
+    /// markdown formatting based on detected layout classes.
+    /// Requires the `layout-detection` feature to run inference; the field is
+    /// present whenever the `layout-types` feature is active (which includes
+    /// `layout-detection` as well as the no-ORT target groups).
+    #[cfg(feature = "layout-types")]
+    #[serde(default)]
+    pub layout: Option<super::super::layout::LayoutDetectionConfig>,
+
+    /// Run layout detection on the non-OCR PDF markdown path.
+    ///
+    /// When `true` and `layout` is `Some(_)`, layout regions inform heading,
+    /// table, list, and figure detection in the structure pipeline that would
+    /// otherwise rely on font-clustering heuristics alone. Significantly
+    /// improves SF1 (structural F1) at the cost of inference latency
+    /// (~150-300ms/page CPU, ~20-50ms/page GPU). Default: `false`.
+    /// Requires the `layout-detection` feature.
+    #[serde(default)]
+    pub use_layout_for_markdown: bool,
+
+    /// Enable structured document tree output.
+    ///
+    /// When true, populates the `document` field on `ExtractionResult` with a
+    /// hierarchical `DocumentStructure` containing heading-driven section nesting,
+    /// table grids, content layer classification, and inline annotations.
+    ///
+    /// Independent of `result_format` — can be combined with Unified or ElementBased.
+    #[serde(default)]
+    pub include_document_structure: bool,
+
+    /// Hardware acceleration configuration for ONNX Runtime models.
+    ///
+    /// Controls execution provider selection for layout detection and embedding
+    /// models. When `None`, uses platform defaults (CoreML on macOS, CUDA on
+    /// Linux, CPU on Windows).
+    #[serde(default)]
+    pub acceleration: Option<AccelerationConfig>,
+
+    /// Cache namespace for tenant isolation.
+    ///
+    /// When set, cache entries are stored under `{cache_dir}/{namespace}/`.
+    /// Must be alphanumeric, hyphens, or underscores only (max 64 chars).
+    /// Different namespaces have isolated cache spaces on the same filesystem.
+    #[serde(default)]
+    pub cache_namespace: Option<String>,
+
+    /// Per-request cache TTL in seconds.
+    ///
+    /// Overrides the global `max_age_days` for this specific extraction.
+    /// When `0`, caching is completely skipped (no read or write).
+    /// When `None`, the global TTL applies.
+    #[serde(default)]
+    pub cache_ttl_secs: Option<u64>,
+
+    /// Email extraction configuration (None = use defaults).
+    ///
+    /// Currently supports configuring the fallback codepage for MSG files
+    /// that do not specify one. See [`crate::core::config::EmailConfig`] for details.
+    #[serde(default)]
+    pub email: Option<super::super::email::EmailConfig>,
+
+    /// Concurrency limits for constrained environments (None = use defaults).
+    ///
+    /// Controls Rayon thread pool size, ONNX Runtime intra-op threads, and
+    /// (when `max_concurrent_extractions` is unset) the batch concurrency
+    /// semaphore. See [`crate::core::config::ConcurrencyConfig`] for details.
+    #[serde(default)]
+    pub concurrency: Option<super::super::concurrency::ConcurrencyConfig>,
+
+    /// Maximum recursion depth for archive extraction (default: 3).
+    /// Set to 0 to disable recursive extraction (legacy behavior).
+    #[serde(default = "default_archive_depth")]
+    pub max_archive_depth: usize,
+
+    /// Tree-sitter language pack configuration (None = tree-sitter disabled).
+    ///
+    /// When set, enables code file extraction using tree-sitter parsers.
+    /// Controls grammar download behavior and code analysis options.
+    #[cfg(feature = "tree-sitter")]
+    #[serde(default)]
+    pub tree_sitter: Option<super::super::tree_sitter::TreeSitterConfig>,
+
+    /// Structured extraction via LLM (None = disabled).
+    ///
+    /// When set, the extracted document content is sent to an LLM with the
+    /// provided JSON schema. The structured response is stored in
+    /// `ExtractionResult::structured_output`.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub structured_extraction: Option<super::super::llm::StructuredExtractionConfig>,
+
+    /// Cancellation token for this extraction (None = no external cancellation).
+    ///
+    /// Pass a [`CancellationToken`] clone here and call [`CancellationToken::cancel`]
+    /// from another thread / task to abort the extraction in progress. The extractor
+    /// checks the token at safe checkpoints (before lock acquisition, between pages,
+    /// between batch items) and returns [`KreuzbergError::Cancelled`] when set.
+    ///
+    /// The field is excluded from serialization because `CancellationToken` is a
+    /// runtime handle, not a configuration value.
+    #[serde(skip)]
+    pub cancel_token: Option<crate::cancellation::CancellationToken>,
+}
+
+impl Default for ExtractionConfig {
+    fn default() -> Self {
+        Self {
+            use_cache: true,
+            enable_quality_processing: true,
+            ocr: None,
+            force_ocr: false,
+            force_ocr_pages: None,
+            disable_ocr: false,
+            chunking: None,
+            content_filter: None,
+            images: None,
+            #[cfg(feature = "pdf")]
+            pdf_options: None,
+            token_reduction: None,
+            language_detection: None,
+            pages: None,
+            #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
+            keywords: None,
+            postprocessor: None,
+            #[cfg(feature = "html")]
+            html_options: None,
+            #[cfg(feature = "html")]
+            html_output: None,
+            extraction_timeout_secs: default_extraction_timeout(),
+            max_concurrent_extractions: None,
+            security_limits: None,
+            max_embedded_file_bytes: default_max_embedded_file_bytes(),
+            #[cfg(feature = "layout-types")]
+            layout: None,
+            use_layout_for_markdown: false,
+            result_format: crate::types::ResultFormat::Unified,
+            output_format: OutputFormat::Plain,
+            include_document_structure: false,
+            acceleration: None,
+            cache_namespace: None,
+            cache_ttl_secs: None,
+            email: None,
+            concurrency: None,
+            max_archive_depth: default_archive_depth(),
+            #[cfg(feature = "tree-sitter")]
+            tree_sitter: None,
+            structured_extraction: None,
+            cancel_token: None,
+        }
+    }
+}
+
+impl ExtractionConfig {
+    /// Create a new `ExtractionConfig` by applying per-file overrides from a
+    /// [`FileExtractionConfig`]. Fields that are `Some` in the override replace the
+    /// corresponding field in `self`; `None` fields keep the original value.
+    ///
+    /// Batch-level fields (`max_concurrent_extractions`, `use_cache`, `acceleration`,
+    /// `security_limits`) are never affected by overrides.
+    ///
+    /// # Example
+    ///
+    /// ```ignore
+    /// use kreuzberg::{ExtractionConfig, FileExtractionConfig};
+    ///
+    /// let base = ExtractionConfig::default();
+    /// let override_config = FileExtractionConfig {
+    ///     force_ocr: Some(true),
+    ///     ..Default::default()
+    /// };
+    /// let resolved = base.with_file_overrides(&override_config);
+    /// assert!(resolved.force_ocr);
+    /// ```
+    pub(crate) fn with_file_overrides(&self, overrides: &FileExtractionConfig) -> Self {
+        // Destructure to ensure compile-time exhaustiveness: adding a field to
+        // FileExtractionConfig without handling it here will produce a compile error.
+        let FileExtractionConfig {
+            ref enable_quality_processing,
+            ref ocr,
+            ref force_ocr,
+            ref force_ocr_pages,
+            ref disable_ocr,
+            ref chunking,
+            ref content_filter,
+            ref images,
+            #[cfg(feature = "pdf")]
+            ref pdf_options,
+            ref token_reduction,
+            ref language_detection,
+            ref pages,
+            #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
+            ref keywords,
+            ref postprocessor,
+            #[cfg(feature = "html")]
+            ref html_options,
+            ref result_format,
+            ref output_format,
+            ref include_document_structure,
+            #[cfg(feature = "layout-types")]
+            ref layout,
+            ref timeout_secs,
+            #[cfg(feature = "tree-sitter")]
+            ref tree_sitter,
+            ref structured_extraction,
+        } = *overrides;
+
+        let mut config = self.clone();
+
+        if let Some(v) = enable_quality_processing {
+            config.enable_quality_processing = *v;
+        }
+        if let Some(v) = ocr {
+            config.ocr = Some(v.clone());
+        }
+        if let Some(v) = force_ocr {
+            config.force_ocr = *v;
+        }
+        if let Some(v) = force_ocr_pages {
+            config.force_ocr_pages = Some(v.clone());
+        }
+        if let Some(v) = disable_ocr {
+            config.disable_ocr = *v;
+        }
+        if let Some(v) = chunking {
+            config.chunking = Some(v.clone());
+        }
+        if let Some(v) = content_filter {
+            config.content_filter = Some(v.clone());
+        }
+        if let Some(v) = images {
+            config.images = Some(v.clone());
+        }
+        #[cfg(feature = "pdf")]
+        if let Some(v) = pdf_options {
+            config.pdf_options = Some(v.clone());
+        }
+        if let Some(v) = token_reduction {
+            config.token_reduction = Some(v.clone());
+        }
+        if let Some(v) = language_detection {
+            config.language_detection = Some(v.clone());
+        }
+        if let Some(v) = pages {
+            config.pages = Some(v.clone());
+        }
+        #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
+        if let Some(v) = keywords {
+            config.keywords = Some(v.clone());
+        }
+        if let Some(v) = postprocessor {
+            config.postprocessor = Some(v.clone());
+        }
+        #[cfg(feature = "html")]
+        if let Some(v) = html_options {
+            config.html_options = Some(v.clone());
+        }
+        if let Some(v) = result_format {
+            config.result_format = *v;
+        }
+        if let Some(v) = output_format {
+            config.output_format = v.clone();
+        }
+        if let Some(v) = include_document_structure {
+            config.include_document_structure = *v;
+        }
+        #[cfg(feature = "layout-types")]
+        if let Some(v) = layout {
+            config.layout = Some(v.clone());
+        }
+        if let Some(v) = timeout_secs {
+            config.extraction_timeout_secs = Some(*v);
+        }
+        #[cfg(feature = "tree-sitter")]
+        if let Some(v) = tree_sitter {
+            config.tree_sitter = Some(v.clone());
+        }
+        if let Some(v) = structured_extraction {
+            config.structured_extraction = Some(v.clone());
+        }
+
+        config
+    }
+
+    /// Normalize configuration for implicit requirements.
+    ///
+    /// Currently handles:
+    /// - Auto-enabling `extract_pages` when `result_format` is `ElementBased`, because
+    ///   the element transformation requires per-page data to assign correct page numbers.
+    ///   Without this, all elements would incorrectly get `page_number=1`.
+    /// - Auto-enabling `extract_pages` when chunking is configured, because the chunker
+    ///   needs page boundaries to assign correct page numbers to chunks.
+    pub(crate) fn normalized(&self) -> std::borrow::Cow<'_, Self> {
+        let needs_pages = |cfg: &Self| -> bool {
+            match &cfg.pages {
+                Some(page_config) => !page_config.extract_pages,
+                None => true,
+            }
+        };
+
+        let needs_pages_for_elements =
+            self.result_format == crate::types::ResultFormat::ElementBased && needs_pages(self);
+        let needs_pages_for_chunking = self.chunking.is_some() && needs_pages(self);
+
+        if needs_pages_for_elements || needs_pages_for_chunking {
+            let mut config = self.clone();
+            let page_config = config.pages.get_or_insert_with(super::super::page::PageConfig::default);
+            page_config.extract_pages = true;
+            return std::borrow::Cow::Owned(config);
+        }
+        std::borrow::Cow::Borrowed(self)
+    }
+
+    /// Validate the configuration, returning an error if any settings are invalid.
+    ///
+    /// Checks:
+    /// Returns the effective disable-OCR value, accounting for both the top-level
+    /// `disable_ocr` flag and the `ocr.enabled` shorthand on [`OcrConfig`].
+    ///
+    /// Setting `ocr.enabled = false` in configuration is treated as equivalent to
+    /// `disable_ocr = true`. This method is the single source of truth for whether
+    /// OCR should be skipped.
+    pub(crate) fn effective_disable_ocr(&self) -> bool {
+        self.disable_ocr || self.ocr.as_ref().is_some_and(|o| !o.enabled)
+    }
+
+    /// Check if image processing is needed by examining OCR and image extraction settings.
+    ///
+    /// Returns `true` if either OCR is enabled or image extraction is configured,
+    /// indicating that image decompression and processing should occur.
+    /// Returns `false` if both are disabled, allowing optimization to skip unnecessary
+    /// image decompression for text-only extraction workflows.
+    ///
+    /// # Optimization Impact
+    /// For text-only extractions (no OCR, no image extraction), skipping image
+    /// decompression can improve CPU utilization by 5-10% by avoiding wasteful
+    /// image I/O and processing when results won't be used.
+    pub fn needs_image_processing(&self) -> bool {
+        let ocr_enabled = !self.effective_disable_ocr() && (self.ocr.is_some() || self.force_ocr);
+
+        let image_extraction_enabled = self.images.as_ref().map(|i| i.extract_images).unwrap_or(false);
+
+        #[cfg(feature = "layout-detection")]
+        let layout_enabled = self.layout.is_some();
+        #[cfg(not(feature = "layout-detection"))]
+        let layout_enabled = false;
+
+        ocr_enabled || image_extraction_enabled || layout_enabled
+    }
+}
+
+fn default_true() -> bool {
+    true
+}
+
+fn default_archive_depth() -> usize {
+    3
+}
+
+/// Default per-embedded-file cap: 50 MiB.
+///
+/// A single embedded object larger than this can consume significant memory
+/// when the recursive extractor materialises it. 50 MiB is generous for
+/// real-world embedded documents while still bounding worst-case allocation.
+fn default_max_embedded_file_bytes() -> Option<u64> {
+    Some(50 * 1024 * 1024)
+}
+
+/// Default extraction timeout: 60 seconds.
+///
+/// Pathological files (deeply nested archives, sheets with millions of cells,
+/// adversarial PDFs) can otherwise run indefinitely and exhaust caller
+/// resources. 60 s is generous for legitimate documents while bounding the
+/// worst-case cost of a single untrusted input.
+fn default_extraction_timeout() -> Option<u64> {
+    Some(60)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::core::config::OcrConfig;
+
+    #[test]
+    fn test_effective_disable_ocr_from_top_level_flag() {
+        let config = ExtractionConfig {
+            disable_ocr: true,
+            ..Default::default()
+        };
+        assert!(config.effective_disable_ocr());
+    }
+
+    #[test]
+    fn test_effective_disable_ocr_from_ocr_enabled_false() {
+        let config = ExtractionConfig {
+            ocr: Some(OcrConfig {
+                enabled: false,
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        assert!(
+            config.effective_disable_ocr(),
+            "ocr.enabled = false should be treated as disable_ocr = true"
+        );
+    }
+
+    #[test]
+    fn test_effective_disable_ocr_default_is_false() {
+        let config = ExtractionConfig::default();
+        assert!(!config.effective_disable_ocr());
+    }
+
+    #[test]
+    fn test_effective_disable_ocr_ocr_enabled_true_does_not_disable() {
+        let config = ExtractionConfig {
+            ocr: Some(OcrConfig {
+                enabled: true,
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        assert!(!config.effective_disable_ocr());
+    }
+
+    #[test]
+    fn test_ocr_enabled_false_deserialized_from_json() {
+        let json = r#"{"ocr": {"enabled": false}}"#;
+        let config: ExtractionConfig = serde_json::from_str(json).unwrap();
+        assert!(
+            config.effective_disable_ocr(),
+            "JSON ocr.enabled=false should disable OCR"
+        );
+    }
+
+    #[test]
+    fn test_ocr_enabled_defaults_to_true() {
+        let json = r#"{"ocr": {"backend": "tesseract"}}"#;
+        let config: ExtractionConfig = serde_json::from_str(json).unwrap();
+        assert!(!config.effective_disable_ocr(), "OCR should be enabled by default");
+    }
+
+    #[cfg(feature = "layout-detection")]
+    #[test]
+    fn test_use_layout_for_markdown_defaults_to_false() {
+        let config = ExtractionConfig::default();
+        assert!(!config.use_layout_for_markdown);
+    }
+
+    #[cfg(feature = "layout-detection")]
+    #[test]
+    fn test_use_layout_for_markdown_can_be_set_true() {
+        let config = ExtractionConfig {
+            use_layout_for_markdown: true,
+            ..Default::default()
+        };
+        assert!(config.use_layout_for_markdown);
+    }
+
+    #[cfg(feature = "layout-detection")]
+    #[test]
+    fn test_use_layout_for_markdown_serde_round_trip() {
+        let config = ExtractionConfig {
+            use_layout_for_markdown: true,
+            ..Default::default()
+        };
+        let json = serde_json::to_string(&config).unwrap();
+        let deserialized: ExtractionConfig = serde_json::from_str(&json).unwrap();
+        assert!(deserialized.use_layout_for_markdown);
+    }
+
+    #[cfg(feature = "layout-detection")]
+    #[test]
+    fn test_use_layout_for_markdown_serde_default_false() {
+        // Field absent in JSON → should default to false.
+        let json = r#"{}"#;
+        let config: ExtractionConfig = serde_json::from_str(json).unwrap();
+        assert!(!config.use_layout_for_markdown);
+    }
+
+    // --- extraction_timeout_secs defaults ----------------------------------
+
+    #[test]
+    fn test_default_extraction_timeout_is_sixty_seconds() {
+        let config = ExtractionConfig::default();
+        assert_eq!(
+            config.extraction_timeout_secs,
+            Some(60),
+            "default timeout must be Some(60) to prevent unbounded extraction"
+        );
+    }
+
+    #[test]
+    fn test_extraction_timeout_can_be_disabled_by_setting_none() {
+        let config = ExtractionConfig {
+            extraction_timeout_secs: None,
+            ..Default::default()
+        };
+        assert_eq!(config.extraction_timeout_secs, None);
+    }
+
+    #[test]
+    fn test_extraction_timeout_serde_round_trip() {
+        let config = ExtractionConfig {
+            extraction_timeout_secs: Some(120),
+            ..Default::default()
+        };
+        let json = serde_json::to_string(&config).unwrap();
+        let deserialized: ExtractionConfig = serde_json::from_str(&json).unwrap();
+        assert_eq!(deserialized.extraction_timeout_secs, Some(120));
+    }
+
+    #[test]
+    fn test_extraction_timeout_serde_absent_field_defaults_to_sixty() {
+        // When the JSON field is absent the serde default function must fire.
+        let json = r#"{}"#;
+        let config: ExtractionConfig = serde_json::from_str(json).unwrap();
+        assert_eq!(
+            config.extraction_timeout_secs,
+            Some(60),
+            "absent field must use default_extraction_timeout() -> Some(60)"
+        );
+    }
+}

crates/kreuzberg/src/core/config/extraction/env.rs

@@ -0,0 +1,493 @@
+//! Environment variable override support for extraction configuration.
+//!
+//! This module provides functionality to apply environment variable overrides
+//! to extraction configuration, allowing runtime configuration changes.
+
+use crate::{KreuzbergError, Result};
+
+use super::super::ocr::OcrConfig;
+use super::super::processing::ChunkingConfig;
+use super::core::ExtractionConfig;
+use super::types::TokenReductionOptions;
+
+impl ExtractionConfig {
+    /// Apply environment variable overrides to configuration.
+    ///
+    /// Environment variables have the highest precedence and will override any values
+    /// loaded from configuration files. This method supports the following environment variables:
+    ///
+    /// - `KREUZBERG_OCR_LANGUAGE`: OCR language (ISO 639-1 or 639-3 code, e.g., "eng", "fra", "deu")
+    /// - `KREUZBERG_OCR_BACKEND`: OCR backend ("tesseract", "easyocr", or "paddleocr")
+    /// - `KREUZBERG_CHUNKING_MAX_CHARS`: Maximum characters per chunk (positive integer)
+    /// - `KREUZBERG_CHUNKING_MAX_OVERLAP`: Maximum overlap between chunks (non-negative integer)
+    /// - `KREUZBERG_CACHE_ENABLED`: Cache enabled flag ("true" or "false")
+    /// - `KREUZBERG_TOKEN_REDUCTION_MODE`: Token reduction mode ("off", "light", "moderate", "aggressive", or "maximum")
+    /// - `KREUZBERG_CHUNKING_TOKENIZER`: HuggingFace tokenizer model ID for token-based chunk sizing (requires `chunking-tokenizers` feature)
+    /// - `KREUZBERG_DISABLE_OCR`: Disable OCR entirely ("true" or "false")
+    /// - `KREUZBERG_LLM_MODEL`: LLM model for structured extraction (e.g., "openai/gpt-4o")
+    /// - `KREUZBERG_LLM_API_KEY`: API key for the structured extraction LLM provider
+    /// - `KREUZBERG_LLM_BASE_URL`: Custom base URL for the structured extraction LLM provider
+    /// - `KREUZBERG_VLM_OCR_MODEL`: VLM model for vision-based OCR (e.g., "openai/gpt-4o")
+    /// - `KREUZBERG_VLM_EMBEDDING_MODEL`: LLM model for embedding generation (e.g., "openai/text-embedding-3-small")
+    /// - `KREUZBERG_EMBEDDING_PLUGIN_NAME`: Name of an in-process embedding backend registered via `plugins::register_embedding_backend`
+    /// - `KREUZBERG_MSG_FALLBACK_CODEPAGE`: (deferred) Windows codepage for MSG PT_STRING8 fallback
+    ///
+    /// # Behavior
+    ///
+    /// - If an environment variable is set and valid, it overrides the current configuration value
+    /// - If a required parent config is `None` (e.g., `self.ocr` is None), it's created with defaults before applying the override
+    /// - Invalid values return a `KreuzbergError::Validation` with helpful error messages
+    /// - Missing or unset environment variables are silently ignored
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// # use kreuzberg::core::config::ExtractionConfig;
+    /// # fn example() -> kreuzberg::Result<()> {
+    /// let mut config = ExtractionConfig::from_file("config.toml")?;
+    /// // Set KREUZBERG_OCR_LANGUAGE=fra before calling
+    /// config.apply_env_overrides()?; // OCR language is now "fra"
+    /// # Ok(())
+    /// # }
+    /// ```
+    ///
+    /// # Errors
+    ///
+    /// Returns `KreuzbergError::Validation` if:
+    /// - An environment variable contains an invalid value
+    /// - A number cannot be parsed as the expected type
+    /// - A boolean is not "true" or "false"
+    pub fn apply_env_overrides(&mut self) -> Result<()> {
+        use crate::core::config_validation::{
+            validate_chunking_params, validate_language_code, validate_ocr_backend, validate_token_reduction_level,
+        };
+
+        // KREUZBERG_OCR_LANGUAGE override
+        if let Ok(lang) = std::env::var("KREUZBERG_OCR_LANGUAGE") {
+            validate_language_code(&lang)?;
+            if self.ocr.is_none() {
+                self.ocr = Some(OcrConfig::default());
+            }
+            if let Some(ref mut ocr) = self.ocr {
+                ocr.language = lang;
+            }
+        }
+
+        // KREUZBERG_OCR_BACKEND override
+        if let Ok(backend) = std::env::var("KREUZBERG_OCR_BACKEND") {
+            validate_ocr_backend(&backend)?;
+            if self.ocr.is_none() {
+                self.ocr = Some(OcrConfig::default());
+            }
+            if let Some(ref mut ocr) = self.ocr {
+                ocr.backend = backend;
+            }
+        }
+
+        // KREUZBERG_CHUNKING_MAX_CHARS override
+        if let Ok(max_chars_str) = std::env::var("KREUZBERG_CHUNKING_MAX_CHARS") {
+            let max_chars: usize = max_chars_str.parse().map_err(|_| KreuzbergError::Validation {
+                message: format!(
+                    "Invalid value for KREUZBERG_CHUNKING_MAX_CHARS: '{}'. Must be a positive integer.",
+                    max_chars_str
+                ),
+                source: None,
+            })?;
+
+            if max_chars == 0 {
+                return Err(KreuzbergError::Validation {
+                    message: "KREUZBERG_CHUNKING_MAX_CHARS must be greater than 0".to_string(),
+                    source: None,
+                });
+            }
+
+            if self.chunking.is_none() {
+                self.chunking = Some(ChunkingConfig::default());
+            }
+
+            if let Some(ref mut chunking) = self.chunking {
+                // Validate against current overlap before updating
+                validate_chunking_params(max_chars, chunking.overlap)?;
+                chunking.max_characters = max_chars;
+            }
+        }
+
+        // KREUZBERG_CHUNKING_MAX_OVERLAP override
+        if let Ok(max_overlap_str) = std::env::var("KREUZBERG_CHUNKING_MAX_OVERLAP") {
+            let max_overlap: usize = max_overlap_str.parse().map_err(|_| KreuzbergError::Validation {
+                message: format!(
+                    "Invalid value for KREUZBERG_CHUNKING_MAX_OVERLAP: '{}'. Must be a non-negative integer.",
+                    max_overlap_str
+                ),
+                source: None,
+            })?;
+
+            if self.chunking.is_none() {
+                self.chunking = Some(ChunkingConfig::default());
+            }
+
+            if let Some(ref mut chunking) = self.chunking {
+                // Validate against current max_characters before updating
+                validate_chunking_params(chunking.max_characters, max_overlap)?;
+                chunking.overlap = max_overlap;
+            }
+        }
+
+        // KREUZBERG_CACHE_ENABLED override
+        if let Ok(cache_str) = std::env::var("KREUZBERG_CACHE_ENABLED") {
+            let cache_enabled = match cache_str.to_lowercase().as_str() {
+                "true" => true,
+                "false" => false,
+                _ => {
+                    return Err(KreuzbergError::Validation {
+                        message: format!(
+                            "Invalid value for KREUZBERG_CACHE_ENABLED: '{}'. Must be 'true' or 'false'.",
+                            cache_str
+                        ),
+                        source: None,
+                    });
+                }
+            };
+            self.use_cache = cache_enabled;
+        }
+
+        // KREUZBERG_TOKEN_REDUCTION_MODE override
+        if let Ok(mode) = std::env::var("KREUZBERG_TOKEN_REDUCTION_MODE") {
+            validate_token_reduction_level(&mode)?;
+            if self.token_reduction.is_none() {
+                self.token_reduction = Some(TokenReductionOptions {
+                    mode: "off".to_string(),
+                    preserve_important_words: true,
+                });
+            }
+            if let Some(ref mut token_reduction) = self.token_reduction {
+                token_reduction.mode = mode;
+            }
+        }
+
+        // KREUZBERG_OUTPUT_FORMAT override
+        if let Ok(val) = std::env::var("KREUZBERG_OUTPUT_FORMAT") {
+            self.output_format = val.parse().map_err(|e: String| KreuzbergError::Validation {
+                message: format!("Invalid value for KREUZBERG_OUTPUT_FORMAT: {}", e),
+                source: None,
+            })?;
+        }
+
+        // KREUZBERG_CHUNKING_TOKENIZER override
+        #[cfg(feature = "chunking-tokenizers")]
+        if let Ok(model) = std::env::var("KREUZBERG_CHUNKING_TOKENIZER") {
+            if model.is_empty() {
+                return Err(KreuzbergError::Validation {
+                    message: "KREUZBERG_CHUNKING_TOKENIZER must not be empty".to_string(),
+                    source: None,
+                });
+            }
+
+            if self.chunking.is_none() {
+                self.chunking = Some(ChunkingConfig::default());
+            }
+
+            if let Some(ref mut chunking) = self.chunking {
+                chunking.sizing = crate::core::config::processing::ChunkSizing::Tokenizer { model, cache_dir: None };
+            }
+        }
+
+        // KREUZBERG_LAYOUT_PRESET override (backward compat: enables layout detection).
+        // Only one model (RT-DETR) exists, so the specific preset value is ignored.
+        #[cfg(feature = "layout-detection")]
+        if let Ok(preset) = std::env::var("KREUZBERG_LAYOUT_PRESET") {
+            let lower = preset.to_lowercase();
+            if !["fast", "accurate", "yolo", "rtdetr", "rt-detr"].contains(&lower.as_str()) {
+                return Err(KreuzbergError::Validation {
+                    message: format!(
+                        "Invalid value for KREUZBERG_LAYOUT_PRESET: '{}'. Valid presets: fast, accurate",
+                        preset
+                    ),
+                    source: None,
+                });
+            }
+            if self.layout.is_none() {
+                self.layout = Some(super::super::layout::LayoutDetectionConfig::default());
+            }
+            // preset value is accepted but ignored -- only RT-DETR is available
+            let _ = lower;
+        }
+
+        // KREUZBERG_DISABLE_OCR override
+        if let Ok(val) = std::env::var("KREUZBERG_DISABLE_OCR") {
+            self.disable_ocr = match val.to_lowercase().as_str() {
+                "true" | "1" => true,
+                "false" | "0" => false,
+                _ => {
+                    return Err(KreuzbergError::Validation {
+                        message: format!(
+                            "Invalid value for KREUZBERG_DISABLE_OCR: '{}'. Must be 'true' or 'false'.",
+                            val
+                        ),
+                        source: None,
+                    });
+                }
+            };
+        }
+
+        // KREUZBERG_LLM_MODEL override
+        if let Ok(value) = std::env::var("KREUZBERG_LLM_MODEL") {
+            if value.is_empty() {
+                return Err(KreuzbergError::Validation {
+                    message: "KREUZBERG_LLM_MODEL must not be empty".to_string(),
+                    source: None,
+                });
+            }
+            if self.structured_extraction.is_none() {
+                self.structured_extraction = Some(super::super::llm::StructuredExtractionConfig {
+                    schema: serde_json::Value::Object(Default::default()),
+                    schema_name: "extraction".to_string(),
+                    schema_description: None,
+                    strict: false,
+                    prompt: None,
+                    llm: super::super::llm::LlmConfig {
+                        model: value,
+                        api_key: None,
+                        base_url: None,
+                        timeout_secs: None,
+                        max_retries: None,
+                        temperature: None,
+                        max_tokens: None,
+                    },
+                });
+            } else if let Some(ref mut config) = self.structured_extraction {
+                config.llm.model = value;
+            }
+        }
+
+        // KREUZBERG_LLM_API_KEY override
+        if let Ok(value) = std::env::var("KREUZBERG_LLM_API_KEY") {
+            if value.is_empty() {
+                return Err(KreuzbergError::Validation {
+                    message: "KREUZBERG_LLM_API_KEY must not be empty".to_string(),
+                    source: None,
+                });
+            }
+            if self.structured_extraction.is_none() {
+                self.structured_extraction = Some(super::super::llm::StructuredExtractionConfig {
+                    schema: serde_json::Value::Object(Default::default()),
+                    schema_name: "extraction".to_string(),
+                    schema_description: None,
+                    strict: false,
+                    prompt: None,
+                    llm: super::super::llm::LlmConfig {
+                        model: String::new(),
+                        api_key: Some(value),
+                        base_url: None,
+                        timeout_secs: None,
+                        max_retries: None,
+                        temperature: None,
+                        max_tokens: None,
+                    },
+                });
+            } else if let Some(ref mut config) = self.structured_extraction {
+                config.llm.api_key = Some(value);
+            }
+        }
+
+        // KREUZBERG_LLM_BASE_URL override
+        if let Ok(value) = std::env::var("KREUZBERG_LLM_BASE_URL") {
+            if value.is_empty() {
+                return Err(KreuzbergError::Validation {
+                    message: "KREUZBERG_LLM_BASE_URL must not be empty".to_string(),
+                    source: None,
+                });
+            }
+            if self.structured_extraction.is_none() {
+                self.structured_extraction = Some(super::super::llm::StructuredExtractionConfig {
+                    schema: serde_json::Value::Object(Default::default()),
+                    schema_name: "extraction".to_string(),
+                    schema_description: None,
+                    strict: false,
+                    prompt: None,
+                    llm: super::super::llm::LlmConfig {
+                        model: String::new(),
+                        api_key: None,
+                        base_url: Some(value),
+                        timeout_secs: None,
+                        max_retries: None,
+                        temperature: None,
+                        max_tokens: None,
+                    },
+                });
+            } else if let Some(ref mut config) = self.structured_extraction {
+                config.llm.base_url = Some(value);
+            }
+        }
+
+        // KREUZBERG_VLM_OCR_MODEL override
+        if let Ok(value) = std::env::var("KREUZBERG_VLM_OCR_MODEL") {
+            if value.is_empty() {
+                return Err(KreuzbergError::Validation {
+                    message: "KREUZBERG_VLM_OCR_MODEL must not be empty".to_string(),
+                    source: None,
+                });
+            }
+            if self.ocr.is_none() {
+                self.ocr = Some(OcrConfig::default());
+            }
+            if let Some(ref mut ocr) = self.ocr {
+                if ocr.vlm_config.is_none() {
+                    ocr.vlm_config = Some(super::super::llm::LlmConfig {
+                        model: value,
+                        api_key: None,
+                        base_url: None,
+                        timeout_secs: None,
+                        max_retries: None,
+                        temperature: None,
+                        max_tokens: None,
+                    });
+                } else if let Some(ref mut vlm) = ocr.vlm_config {
+                    vlm.model = value;
+                }
+            }
+        }
+
+        // KREUZBERG_VLM_EMBEDDING_MODEL override
+        if let Ok(value) = std::env::var("KREUZBERG_VLM_EMBEDDING_MODEL") {
+            if value.is_empty() {
+                return Err(KreuzbergError::Validation {
+                    message: "KREUZBERG_VLM_EMBEDDING_MODEL must not be empty".to_string(),
+                    source: None,
+                });
+            }
+            if self.chunking.is_none() {
+                self.chunking = Some(ChunkingConfig::default());
+            }
+            if let Some(ref mut chunking) = self.chunking {
+                chunking.embedding = Some(super::super::processing::EmbeddingConfig {
+                    model: super::super::processing::EmbeddingModelType::Llm {
+                        llm: super::super::llm::LlmConfig {
+                            model: value,
+                            api_key: None,
+                            base_url: None,
+                            timeout_secs: None,
+                            max_retries: None,
+                            temperature: None,
+                            max_tokens: None,
+                        },
+                    },
+                    ..super::super::processing::EmbeddingConfig::default()
+                });
+            }
+        }
+
+        // KREUZBERG_EMBEDDING_PLUGIN_NAME override.
+        // Selects an already-registered in-process embedding backend by name.
+        // Setting this together with KREUZBERG_VLM_EMBEDDING_MODEL is rejected — they
+        // configure mutually-exclusive embedding sources and the result of "both set"
+        // would otherwise depend on source order in this function. Pick one.
+        let plugin_name = std::env::var("KREUZBERG_EMBEDDING_PLUGIN_NAME").ok();
+        if plugin_name.is_some() && std::env::var("KREUZBERG_VLM_EMBEDDING_MODEL").is_ok() {
+            return Err(KreuzbergError::Validation {
+                message:
+                    "KREUZBERG_EMBEDDING_PLUGIN_NAME and KREUZBERG_VLM_EMBEDDING_MODEL are mutually exclusive — set one or the other, not both."
+                        .to_string(),
+                source: None,
+            });
+        }
+        if let Some(value) = plugin_name {
+            if value.is_empty() {
+                return Err(KreuzbergError::Validation {
+                    message: "KREUZBERG_EMBEDDING_PLUGIN_NAME must not be empty".to_string(),
+                    source: None,
+                });
+            }
+            if self.chunking.is_none() {
+                self.chunking = Some(ChunkingConfig::default());
+            }
+            if let Some(ref mut chunking) = self.chunking {
+                chunking.embedding = Some(super::super::processing::EmbeddingConfig {
+                    model: super::super::processing::EmbeddingModelType::Plugin { name: value },
+                    ..super::super::processing::EmbeddingConfig::default()
+                });
+            }
+        }
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+#[allow(unsafe_code)] // env mutation in 2024 edition is unsafe; tests serialize via ENV_LOCK
+mod tests {
+    use super::*;
+    use crate::core::config::processing::EmbeddingModelType;
+
+    /// Lock guarding env-var mutation across tests in this module — `std::env::set_var`
+    /// is process-global and concurrent tests would race.
+    static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
+
+    fn clear_embedding_env() {
+        // SAFETY: callers hold ENV_LOCK so no other thread is reading these vars.
+        unsafe {
+            std::env::remove_var("KREUZBERG_EMBEDDING_PLUGIN_NAME");
+            std::env::remove_var("KREUZBERG_VLM_EMBEDDING_MODEL");
+        }
+    }
+
+    #[test]
+    fn embedding_plugin_and_vlm_embedding_model_are_mutually_exclusive() {
+        let _guard = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner());
+        clear_embedding_env();
+        // SAFETY: see clear_embedding_env.
+        unsafe {
+            std::env::set_var("KREUZBERG_EMBEDDING_PLUGIN_NAME", "my-embedder");
+            std::env::set_var("KREUZBERG_VLM_EMBEDDING_MODEL", "openai/text-embedding-3-small");
+        }
+        let mut config = ExtractionConfig::default();
+        let err = config
+            .apply_env_overrides()
+            .expect_err("should reject conflicting embedding env vars");
+        assert!(
+            matches!(err, KreuzbergError::Validation { .. }),
+            "expected Validation, got {err:?}"
+        );
+        let msg = err.to_string();
+        assert!(msg.contains("mutually exclusive"), "message: {msg}");
+        clear_embedding_env();
+    }
+
+    #[test]
+    fn empty_embedding_plugin_name_rejected() {
+        let _guard = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner());
+        clear_embedding_env();
+        // SAFETY: see clear_embedding_env.
+        unsafe { std::env::set_var("KREUZBERG_EMBEDDING_PLUGIN_NAME", "") };
+        let mut config = ExtractionConfig::default();
+        let err = config
+            .apply_env_overrides()
+            .expect_err("should reject empty plugin name");
+        assert!(
+            matches!(err, KreuzbergError::Validation { .. }),
+            "expected Validation, got {err:?}"
+        );
+        clear_embedding_env();
+    }
+
+    #[test]
+    fn embedding_plugin_env_sets_chunking_embedding_to_plugin_variant() {
+        let _guard = ENV_LOCK.lock().unwrap_or_else(|p| p.into_inner());
+        clear_embedding_env();
+        // SAFETY: see clear_embedding_env.
+        unsafe { std::env::set_var("KREUZBERG_EMBEDDING_PLUGIN_NAME", "my-embedder") };
+        let mut config = ExtractionConfig::default();
+        config
+            .apply_env_overrides()
+            .expect("should succeed with only plugin name set");
+        let chunking = config.chunking.as_ref().expect("chunking should be created");
+        let embedding = chunking.embedding.as_ref().expect("embedding should be set");
+        match &embedding.model {
+            EmbeddingModelType::Plugin { name } => {
+                assert_eq!(name, "my-embedder");
+            }
+            other => panic!("expected Plugin variant, got {other:?}"),
+        }
+        clear_embedding_env();
+    }
+}

crates/kreuzberg/src/core/config/extraction/file_config.rs

@@ -0,0 +1,148 @@
+//! Per-file extraction configuration overrides for batch processing.
+//!
+//! This module contains [`FileExtractionConfig`], a subset of [`super::ExtractionConfig`]
+//! where every field is optional. When used with batch extraction functions, each file
+//! can specify overrides that are merged with the batch-level default config.
+//!
+//! Fields that are batch-level concerns (concurrency, caching, acceleration, security)
+//! are intentionally excluded and can only be set on the batch-level [`super::ExtractionConfig`].
+
+use serde::{Deserialize, Serialize};
+
+use super::super::formats::OutputFormat;
+use super::super::ocr::OcrConfig;
+use super::super::page::PageConfig;
+use super::super::processing::{ChunkingConfig, PostProcessorConfig};
+use super::types::{ImageExtractionConfig, LanguageDetectionConfig, TokenReductionOptions};
+
+/// Per-file extraction configuration overrides for batch processing.
+///
+/// All fields are `Option<T>` — `None` means "use the batch-level default."
+/// This type is used with [`crate::batch_extract_files`] and
+/// [`crate::batch_extract_bytes`] to allow heterogeneous
+/// extraction settings within a single batch.
+///
+/// # Excluded Fields
+///
+/// The following [`super::ExtractionConfig`] fields are batch-level only and
+/// cannot be overridden per file:
+/// - `max_concurrent_extractions` — controls batch parallelism
+/// - `use_cache` — global caching policy
+/// - `acceleration` — shared ONNX execution provider
+/// - `security_limits` — global archive security policy
+///
+/// # Example
+///
+/// ```rust
+/// use kreuzberg::FileExtractionConfig;
+///
+/// // Override just OCR forcing for a specific file
+/// let config = FileExtractionConfig {
+///     force_ocr: Some(true),
+///     ..Default::default()
+/// };
+/// ```
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+#[serde(default)]
+pub struct FileExtractionConfig {
+    /// Override quality post-processing for this file.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub enable_quality_processing: Option<bool>,
+
+    /// Override OCR configuration for this file (None in the Option = use batch default).
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub ocr: Option<OcrConfig>,
+
+    /// Override force OCR for this file.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub force_ocr: Option<bool>,
+
+    /// Override force OCR pages for this file (1-indexed page numbers).
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub force_ocr_pages: Option<Vec<u32>>,
+
+    /// Override disable OCR for this file.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub disable_ocr: Option<bool>,
+
+    /// Override chunking configuration for this file.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub chunking: Option<ChunkingConfig>,
+
+    /// Override content filtering configuration for this file.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub content_filter: Option<super::super::content_filter::ContentFilterConfig>,
+
+    /// Override image extraction configuration for this file.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub images: Option<ImageExtractionConfig>,
+
+    /// Override PDF options for this file.
+    #[cfg(feature = "pdf")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub pdf_options: Option<super::super::pdf::PdfConfig>,
+
+    /// Override token reduction for this file.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub token_reduction: Option<TokenReductionOptions>,
+
+    /// Override language detection for this file.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub language_detection: Option<LanguageDetectionConfig>,
+
+    /// Override page extraction for this file.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub pages: Option<PageConfig>,
+
+    /// Override keyword extraction for this file.
+    #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub keywords: Option<crate::keywords::KeywordConfig>,
+
+    /// Override post-processor for this file.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub postprocessor: Option<PostProcessorConfig>,
+
+    /// Override HTML conversion options for this file.
+    #[cfg(feature = "html")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub html_options: Option<html_to_markdown_rs::ConversionOptions>,
+
+    /// Override result format for this file.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub result_format: Option<crate::types::ResultFormat>,
+
+    /// Override output content format for this file.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub output_format: Option<OutputFormat>,
+
+    /// Override document structure output for this file.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub include_document_structure: Option<bool>,
+
+    /// Override layout detection for this file.
+    #[cfg(feature = "layout-types")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub layout: Option<super::super::layout::LayoutDetectionConfig>,
+
+    /// Override per-file extraction timeout in seconds.
+    ///
+    /// When set, the extraction for this file will be canceled after the
+    /// specified duration. A timed-out file produces an error result without
+    /// affecting other files in the batch.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub timeout_secs: Option<u64>,
+
+    /// Override tree-sitter configuration for this file.
+    #[cfg(feature = "tree-sitter")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub tree_sitter: Option<super::super::tree_sitter::TreeSitterConfig>,
+
+    /// Override structured extraction configuration for this file.
+    ///
+    /// When set, enables LLM-based structured extraction with a JSON schema
+    /// for this specific file. The extracted content is sent to a VLM/LLM
+    /// and the response is parsed according to the provided schema.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub structured_extraction: Option<super::super::llm::StructuredExtractionConfig>,
+}

crates/kreuzberg/src/core/config/extraction/loaders.rs

@@ -0,0 +1,87 @@
+//! Configuration file loading.
+//!
+//! This module provides methods for loading extraction configuration from
+//! TOML, YAML, and JSON files.
+
+use crate::{KreuzbergError, Result};
+use std::path::Path;
+
+use super::core::ExtractionConfig;
+
+impl ExtractionConfig {
+    /// Load configuration from a TOML file.
+    ///
+    /// # Errors
+    ///
+    /// Returns `KreuzbergError::Validation` if file doesn't exist or is invalid TOML.
+    pub fn from_toml_file(path: impl AsRef<Path>) -> Result<Self> {
+        let path = path.as_ref();
+        let content = std::fs::read_to_string(path)
+            .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
+        toml::from_str(&content)
+            .map_err(|e| KreuzbergError::validation(format!("Invalid TOML in {}: {}", path.display(), e)))
+    }
+
+    /// Load configuration from a YAML file.
+    pub fn from_yaml_file(path: impl AsRef<Path>) -> Result<Self> {
+        let path = path.as_ref();
+        let content = std::fs::read_to_string(path)
+            .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
+        serde_yaml_ng::from_str(&content)
+            .map_err(|e| KreuzbergError::validation(format!("Invalid YAML in {}: {}", path.display(), e)))
+    }
+
+    /// Load configuration from a JSON file.
+    pub fn from_json_file(path: impl AsRef<Path>) -> Result<Self> {
+        let path = path.as_ref();
+        let content = std::fs::read_to_string(path)
+            .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
+        serde_json::from_str(&content)
+            .map_err(|e| KreuzbergError::validation(format!("Invalid JSON in {}: {}", path.display(), e)))
+    }
+
+    /// Load configuration from a file, auto-detecting format by extension.
+    ///
+    /// Supported formats: `.toml`, `.yaml`, `.yml`, `.json`.
+    pub fn from_file(path: impl AsRef<Path>) -> Result<Self> {
+        let path = path.as_ref();
+        let extension = path.extension().and_then(|ext| ext.to_str()).ok_or_else(|| {
+            KreuzbergError::validation(format!(
+                "Cannot determine file format: no extension found in {}",
+                path.display()
+            ))
+        })?;
+
+        match extension.to_lowercase().as_str() {
+            "toml" => Self::from_toml_file(path),
+            "yaml" | "yml" => Self::from_yaml_file(path),
+            "json" => Self::from_json_file(path),
+            other => Err(KreuzbergError::validation(format!(
+                "Unsupported config file format: .{}. Supported formats: .toml, .yaml, .json",
+                other
+            ))),
+        }
+    }
+
+    /// Discover configuration file in parent directories.
+    ///
+    /// Searches for `kreuzberg.toml` in current directory and parent directories.
+    pub fn discover() -> Result<Option<Self>> {
+        let mut current = std::env::current_dir().map_err(KreuzbergError::Io)?;
+
+        loop {
+            let kreuzberg_toml = current.join("kreuzberg.toml");
+            if kreuzberg_toml.exists() {
+                return Ok(Some(Self::from_toml_file(kreuzberg_toml)?));
+            }
+
+            if let Some(parent) = current.parent() {
+                current = parent.to_path_buf();
+            } else {
+                break;
+            }
+        }
+
+        Ok(None)
+    }
+}

crates/kreuzberg/src/core/config/extraction/mod.rs

@@ -0,0 +1,46 @@
+//! Main extraction configuration and environment variable handling.
+//!
+//! This module contains the main `ExtractionConfig` struct and related utilities
+//! for loading configuration from files and applying environment variable overrides.
+//!
+//! The module is organized into focused submodules:
+//! - `types`: Feature-specific configuration types (image, token reduction, language detection)
+//! - `core`: Main ExtractionConfig struct and implementation
+//! - `env`: Environment variable override support
+//! - `loaders`: Configuration file loading with caching
+
+mod core;
+mod env;
+mod file_config;
+mod loaders;
+mod types;
+
+// Re-export all public types for backward compatibility
+pub use self::core::ExtractionConfig;
+pub use self::file_config::FileExtractionConfig;
+pub use self::types::{
+    BatchBytesItem, BatchFileItem, ImageExtractionConfig, LanguageDetectionConfig, TokenReductionOptions,
+};
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::core::config::ocr::OcrConfig;
+
+    #[test]
+    fn test_default_config() {
+        let config = ExtractionConfig::default();
+        assert!(config.use_cache);
+        assert!(config.enable_quality_processing);
+        assert!(config.ocr.is_none());
+    }
+
+    #[test]
+    fn test_needs_image_processing() {
+        let mut config = ExtractionConfig::default();
+        assert!(!config.needs_image_processing());
+
+        config.ocr = Some(OcrConfig::default());
+        assert!(config.needs_image_processing());
+    }
+}

crates/kreuzberg/src/core/config/extraction/types.rs

@@ -0,0 +1,345 @@
+//! Feature-specific configuration types for extraction.
+//!
+//! This module contains configuration structs for specific extraction features:
+//! - Image extraction and processing
+//! - Token reduction
+//! - Language detection
+//! - Batch extraction items
+
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+
+/// Batch item for byte array extraction.
+///
+/// Used with [`crate::batch_extract_bytes`] and [`crate::batch_extract_bytes_sync`]
+/// to represent a single item in a batch extraction job.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct BatchBytesItem {
+    /// The content bytes to extract from
+    pub content: Vec<u8>,
+
+    /// MIME type of the content (e.g., "application/pdf", "text/html")
+    pub mime_type: String,
+
+    /// Per-item configuration overrides (None uses batch-level defaults)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub config: Option<super::FileExtractionConfig>,
+}
+
+/// Batch item for file extraction.
+///
+/// Used with [`crate::batch_extract_files`] and [`crate::batch_extract_files_sync`]
+/// to represent a single file in a batch extraction job.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct BatchFileItem {
+    /// Path to the file to extract from
+    pub path: PathBuf,
+
+    /// Per-file configuration overrides (None uses batch-level defaults)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub config: Option<super::FileExtractionConfig>,
+}
+
+/// Image extraction configuration.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ImageExtractionConfig {
+    /// Extract images from documents
+    #[serde(default = "default_true")]
+    pub extract_images: bool,
+
+    /// Target DPI for image normalization
+    #[serde(default = "default_target_dpi")]
+    pub target_dpi: i32,
+
+    /// Maximum dimension for images (width or height)
+    #[serde(default = "default_max_dimension")]
+    pub max_image_dimension: i32,
+
+    /// Whether to inject image reference placeholders into markdown output.
+    /// When `true` (default), image references like `![Image 1](embedded:p1_i0)`
+    /// are appended to the markdown. Set to `false` to extract images as data
+    /// without polluting the markdown output.
+    #[serde(default = "default_true")]
+    pub inject_placeholders: bool,
+
+    /// Automatically adjust DPI based on image content
+    #[serde(default = "default_true")]
+    pub auto_adjust_dpi: bool,
+
+    /// Minimum DPI threshold
+    #[serde(default = "default_min_dpi")]
+    pub min_dpi: i32,
+
+    /// Maximum DPI threshold
+    #[serde(default = "default_max_dpi")]
+    pub max_dpi: i32,
+
+    /// Maximum number of image objects to extract per PDF page.
+    ///
+    /// Some PDFs (e.g. technical diagrams stored as thousands of raster fragments)
+    /// can trigger extremely long or indefinite extraction times when every image
+    /// object on a dense page is decoded individually via the PDF extractor. Setting this
+    /// limit causes kreuzberg to stop collecting individual images once the count
+    /// per page reaches the cap and emit a warning instead.
+    ///
+    /// `None` (default) means no limit — all images are extracted.
+    #[serde(default)]
+    pub max_images_per_page: Option<u32>,
+
+    /// When `true` (default), extracted images are classified by kind and grouped
+    /// into clusters where they appear to belong to one figure.
+    #[serde(default = "default_true")]
+    pub classify: bool,
+
+    /// When `true`, full-page renders produced during OCR preprocessing are captured
+    /// and returned as `ImageKind::PageRaster` entries in `ExtractionResult.images`.
+    ///
+    /// **PDF + OCR only.** No rasters are captured for non-PDF inputs or when the
+    /// document-level OCR bypass is active (whole-document backend). When OCR is
+    /// enabled and this flag is set but the active backend skips per-page rendering,
+    /// a `ProcessingWarning` is emitted in `ExtractionResult.processing_warnings`.
+    ///
+    /// Defaults to `false`. Enable when downstream consumers need page thumbnails
+    /// (e.g. citation previews, visual grounding).
+    #[serde(default)]
+    pub include_page_rasters: bool,
+
+    /// Run OCR on extracted images and include the recognized text in the document content.
+    ///
+    /// When `true` (default) and `ExtractionConfig.ocr` is configured, extracted images
+    /// are processed with the configured OCR backend. Set to `false` to extract images
+    /// without OCR processing, even when OCR is enabled.
+    #[serde(default = "default_true")]
+    pub run_ocr_on_images: bool,
+
+    /// When `true`, image OCR results are rendered as plain text without the
+    /// `![...](...)` markdown placeholder. Only takes effect when `run_ocr_on_images`
+    /// is also `true`.
+    #[serde(default)]
+    pub ocr_text_only: bool,
+
+    /// When `true` and `ocr_text_only` is `false`, append the OCR text after
+    /// the image placeholder in the rendered output.
+    #[serde(default)]
+    pub append_ocr_text: bool,
+}
+
+/// Token reduction configuration.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TokenReductionOptions {
+    /// Reduction mode: "off", "light", "moderate", "aggressive", "maximum"
+    #[serde(default = "default_reduction_mode")]
+    pub mode: String,
+
+    /// Preserve important words (capitalized, technical terms)
+    #[serde(default = "default_true")]
+    pub preserve_important_words: bool,
+}
+
+/// Language detection configuration.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct LanguageDetectionConfig {
+    /// Enable language detection
+    #[serde(default = "default_true")]
+    pub enabled: bool,
+
+    /// Minimum confidence threshold (0.0-1.0)
+    #[serde(default = "default_confidence")]
+    pub min_confidence: f64,
+
+    /// Detect multiple languages in the document
+    #[serde(default)]
+    pub detect_multiple: bool,
+}
+
+impl Default for ImageExtractionConfig {
+    fn default() -> Self {
+        Self {
+            extract_images: true,
+            target_dpi: 300,
+            max_image_dimension: 4096,
+            inject_placeholders: true,
+            auto_adjust_dpi: true,
+            min_dpi: 72,
+            max_dpi: 600,
+            max_images_per_page: None,
+            classify: true,
+            include_page_rasters: false,
+            run_ocr_on_images: true,
+            ocr_text_only: false,
+            append_ocr_text: false,
+        }
+    }
+}
+
+impl Default for TokenReductionOptions {
+    fn default() -> Self {
+        Self {
+            mode: default_reduction_mode(),
+            preserve_important_words: true,
+        }
+    }
+}
+
+impl Default for LanguageDetectionConfig {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            min_confidence: 0.8,
+            detect_multiple: false,
+        }
+    }
+}
+
+// Default value functions
+fn default_true() -> bool {
+    true
+}
+
+fn default_target_dpi() -> i32 {
+    300
+}
+
+fn default_max_dimension() -> i32 {
+    4096
+}
+
+fn default_min_dpi() -> i32 {
+    72
+}
+
+fn default_max_dpi() -> i32 {
+    600
+}
+
+fn default_reduction_mode() -> String {
+    "off".to_string()
+}
+
+fn default_confidence() -> f64 {
+    0.8
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_image_extraction_config_default_booleans_are_true() {
+        let cfg = ImageExtractionConfig::default();
+        assert!(cfg.extract_images, "extract_images must default to true");
+        assert!(cfg.inject_placeholders, "inject_placeholders must default to true");
+        assert!(cfg.auto_adjust_dpi, "auto_adjust_dpi must default to true");
+        assert!(cfg.classify, "classify must default to true");
+        assert_eq!(cfg.target_dpi, 300);
+        assert_eq!(cfg.max_image_dimension, 4096);
+        assert_eq!(cfg.min_dpi, 72);
+        assert_eq!(cfg.max_dpi, 600);
+    }
+
+    #[test]
+    fn test_image_extraction_config_defaults() {
+        let cfg = ImageExtractionConfig::default();
+        assert!(cfg.run_ocr_on_images, "run_ocr_on_images must default to true");
+    }
+
+    #[test]
+    fn test_image_extraction_config_explicit_false_disables_placeholders() {
+        let cfg = ImageExtractionConfig {
+            inject_placeholders: false,
+            ..ImageExtractionConfig::default()
+        };
+        assert!(!cfg.inject_placeholders);
+        assert!(cfg.extract_images);
+    }
+
+    #[test]
+    fn test_image_extraction_config_explicit_false_disables_classify() {
+        let cfg = ImageExtractionConfig {
+            classify: false,
+            ..ImageExtractionConfig::default()
+        };
+        assert!(!cfg.classify);
+        assert!(cfg.extract_images);
+    }
+
+    #[test]
+    fn test_image_extraction_config_absent_json_fields_get_canonical_defaults() {
+        let json = r#"{"extract_images": true}"#;
+        let cfg: ImageExtractionConfig = serde_json::from_str(json).unwrap();
+        assert!(
+            cfg.inject_placeholders,
+            "absent inject_placeholders must deserialize to true"
+        );
+        assert!(cfg.auto_adjust_dpi, "absent auto_adjust_dpi must deserialize to true");
+        assert_eq!(cfg.target_dpi, 300);
+    }
+
+    #[test]
+    fn test_max_images_per_page_defaults_none() {
+        let config = ImageExtractionConfig::default();
+        assert_eq!(config.max_images_per_page, None);
+    }
+
+    #[test]
+    fn test_max_images_per_page_serializes_as_null_when_none() {
+        let config = ImageExtractionConfig::default();
+        let json = serde_json::to_string(&config).unwrap();
+        assert!(json.contains("\"max_images_per_page\":null"));
+    }
+
+    #[test]
+    fn test_max_images_per_page_roundtrips_via_json() {
+        let config = ImageExtractionConfig {
+            max_images_per_page: Some(50),
+            ..Default::default()
+        };
+        let json = serde_json::to_string(&config).unwrap();
+        let back: ImageExtractionConfig = serde_json::from_str(&json).unwrap();
+        assert_eq!(back.max_images_per_page, Some(50));
+    }
+
+    /// Regression test for issue #766: missing field in JSON must not break
+    /// deserialization (backwards-compat — existing configs without this key
+    /// must still deserialize cleanly).
+    #[test]
+    fn test_max_images_per_page_absent_in_json_deserializes_as_none() {
+        let json = r#"{"extract_images":true,"target_dpi":300,"max_image_dimension":4096,
+                       "inject_placeholders":true,"auto_adjust_dpi":true,
+                       "min_dpi":72,"max_dpi":600}"#;
+        let config: ImageExtractionConfig = serde_json::from_str(json).unwrap();
+        assert_eq!(config.max_images_per_page, None);
+    }
+
+    #[test]
+    fn test_include_page_rasters_defaults_false() {
+        let config = ImageExtractionConfig::default();
+        assert!(
+            !config.include_page_rasters,
+            "include_page_rasters must default to false"
+        );
+    }
+
+    #[test]
+    fn test_include_page_rasters_absent_in_json_deserializes_as_false() {
+        let json = r#"{"extract_images":true,"target_dpi":300,"max_image_dimension":4096,
+                       "inject_placeholders":true,"auto_adjust_dpi":true,
+                       "min_dpi":72,"max_dpi":600}"#;
+        let config: ImageExtractionConfig = serde_json::from_str(json).unwrap();
+        assert!(
+            !config.include_page_rasters,
+            "absent include_page_rasters must deserialize to false (backward compat)"
+        );
+    }
+
+    #[test]
+    fn test_include_page_rasters_roundtrips_via_json() {
+        let config = ImageExtractionConfig {
+            include_page_rasters: true,
+            ..Default::default()
+        };
+        let json = serde_json::to_string(&config).unwrap();
+        let back: ImageExtractionConfig = serde_json::from_str(&json).unwrap();
+        assert!(back.include_page_rasters);
+    }
+}

crates/kreuzberg/src/core/config/formats.rs

@@ -0,0 +1,201 @@
+//! Output format configuration and validation.
+//!
+//! This module defines the `OutputFormat` enum for controlling how extraction
+//! results are formatted (plain text, markdown, HTML, etc.) and provides
+//! serialization/deserialization support.
+
+use serde::{Deserialize, Serialize};
+use std::str::FromStr;
+
+/// Output format for extraction results.
+///
+/// Controls the format of the `content` field in `ExtractionResult`.
+/// When set to `Markdown`, `Djot`, or `Html`, the output uses that format.
+/// `Plain` returns the raw extracted text.
+/// `Structured` returns JSON with full OCR element data including bounding
+/// boxes and confidence scores.
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum OutputFormat {
+    /// Plain text content only (default)
+    #[default]
+    Plain,
+    /// Markdown format
+    Markdown,
+    /// Djot markup format
+    Djot,
+    /// HTML format
+    Html,
+    /// JSON tree format with heading-driven sections.
+    Json,
+    /// Structured JSON format with full OCR element metadata.
+    Structured,
+    /// Custom renderer registered via the RendererRegistry.
+    /// The string is the renderer name (e.g., "docx", "latex").
+    #[serde(untagged)]
+    Custom(String),
+}
+
+#[cfg(test)]
+impl OutputFormat {
+    /// Get the renderer name for this format.
+    /// Returns `None` for formats that don't use the renderer registry
+    /// (Plain, Structured, Toon — these are handled differently).
+    pub(crate) fn renderer_name(&self) -> Option<&str> {
+        match self {
+            OutputFormat::Plain | OutputFormat::Json | OutputFormat::Structured => None,
+            OutputFormat::Markdown => Some("markdown"),
+            OutputFormat::Djot => Some("djot"),
+            OutputFormat::Html => Some("html"),
+            OutputFormat::Custom(name) => Some(name.as_str()),
+        }
+    }
+}
+
+impl std::fmt::Display for OutputFormat {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            OutputFormat::Plain => write!(f, "plain"),
+            OutputFormat::Markdown => write!(f, "markdown"),
+            OutputFormat::Djot => write!(f, "djot"),
+            OutputFormat::Html => write!(f, "html"),
+            OutputFormat::Json => write!(f, "json"),
+            OutputFormat::Structured => write!(f, "structured"),
+            OutputFormat::Custom(name) => write!(f, "{}", name),
+        }
+    }
+}
+
+impl FromStr for OutputFormat {
+    type Err = String;
+
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        match s.to_lowercase().as_str() {
+            "plain" | "text" => Ok(OutputFormat::Plain),
+            "markdown" | "md" => Ok(OutputFormat::Markdown),
+            "djot" => Ok(OutputFormat::Djot),
+            "html" => Ok(OutputFormat::Html),
+            "json" => Ok(OutputFormat::Json),
+            "structured" | "structured-ocr" => Ok(OutputFormat::Structured),
+            other => Ok(OutputFormat::Custom(other.to_string())),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_output_format_from_str_plain() {
+        assert_eq!("plain".parse::<OutputFormat>().unwrap(), OutputFormat::Plain);
+        assert_eq!("PLAIN".parse::<OutputFormat>().unwrap(), OutputFormat::Plain);
+        assert_eq!("text".parse::<OutputFormat>().unwrap(), OutputFormat::Plain);
+        assert_eq!("TEXT".parse::<OutputFormat>().unwrap(), OutputFormat::Plain);
+    }
+
+    #[test]
+    fn test_output_format_from_str_markdown() {
+        assert_eq!("markdown".parse::<OutputFormat>().unwrap(), OutputFormat::Markdown);
+        assert_eq!("MARKDOWN".parse::<OutputFormat>().unwrap(), OutputFormat::Markdown);
+        assert_eq!("md".parse::<OutputFormat>().unwrap(), OutputFormat::Markdown);
+        assert_eq!("MD".parse::<OutputFormat>().unwrap(), OutputFormat::Markdown);
+    }
+
+    #[test]
+    fn test_output_format_from_str_djot() {
+        assert_eq!("djot".parse::<OutputFormat>().unwrap(), OutputFormat::Djot);
+        assert_eq!("DJOT".parse::<OutputFormat>().unwrap(), OutputFormat::Djot);
+        assert_eq!("Djot".parse::<OutputFormat>().unwrap(), OutputFormat::Djot);
+    }
+
+    #[test]
+    fn test_output_format_from_str_html() {
+        assert_eq!("html".parse::<OutputFormat>().unwrap(), OutputFormat::Html);
+        assert_eq!("HTML".parse::<OutputFormat>().unwrap(), OutputFormat::Html);
+        assert_eq!("Html".parse::<OutputFormat>().unwrap(), OutputFormat::Html);
+    }
+
+    #[test]
+    fn test_output_format_from_str_json() {
+        assert_eq!("json".parse::<OutputFormat>().unwrap(), OutputFormat::Json);
+        assert_eq!("JSON".parse::<OutputFormat>().unwrap(), OutputFormat::Json);
+    }
+
+    #[test]
+    fn test_output_format_from_str_structured() {
+        assert_eq!("structured".parse::<OutputFormat>().unwrap(), OutputFormat::Structured);
+        assert_eq!("STRUCTURED".parse::<OutputFormat>().unwrap(), OutputFormat::Structured);
+        assert_eq!(
+            "structured-ocr".parse::<OutputFormat>().unwrap(),
+            OutputFormat::Structured
+        );
+        assert_eq!(
+            "STRUCTURED-OCR".parse::<OutputFormat>().unwrap(),
+            OutputFormat::Structured
+        );
+    }
+
+    #[test]
+    fn test_output_format_from_str_custom() {
+        let result = "docx".parse::<OutputFormat>().unwrap();
+        assert_eq!(result, OutputFormat::Custom("docx".to_string()));
+    }
+
+    #[test]
+    fn test_output_format_to_string() {
+        assert_eq!(OutputFormat::Plain.to_string(), "plain");
+        assert_eq!(OutputFormat::Markdown.to_string(), "markdown");
+        assert_eq!(OutputFormat::Djot.to_string(), "djot");
+        assert_eq!(OutputFormat::Html.to_string(), "html");
+        assert_eq!(OutputFormat::Json.to_string(), "json");
+        assert_eq!(OutputFormat::Structured.to_string(), "structured");
+        assert_eq!(OutputFormat::Custom("docx".to_string()).to_string(), "docx");
+    }
+
+    #[test]
+    fn test_output_format_default() {
+        let format = OutputFormat::default();
+        assert_eq!(format, OutputFormat::Plain);
+    }
+
+    #[test]
+    fn test_output_format_serde_roundtrip() {
+        for format in [
+            OutputFormat::Plain,
+            OutputFormat::Markdown,
+            OutputFormat::Djot,
+            OutputFormat::Html,
+            OutputFormat::Json,
+            OutputFormat::Structured,
+        ] {
+            let json = serde_json::to_string(&format).unwrap();
+            let deserialized: OutputFormat = serde_json::from_str(&json).unwrap();
+            assert_eq!(format, deserialized);
+        }
+    }
+
+    #[test]
+    fn test_output_format_serde_values() {
+        assert_eq!(serde_json::to_string(&OutputFormat::Plain).unwrap(), "\"plain\"");
+        assert_eq!(serde_json::to_string(&OutputFormat::Markdown).unwrap(), "\"markdown\"");
+        assert_eq!(serde_json::to_string(&OutputFormat::Djot).unwrap(), "\"djot\"");
+        assert_eq!(serde_json::to_string(&OutputFormat::Html).unwrap(), "\"html\"");
+        assert_eq!(serde_json::to_string(&OutputFormat::Json).unwrap(), "\"json\"");
+        assert_eq!(
+            serde_json::to_string(&OutputFormat::Structured).unwrap(),
+            "\"structured\""
+        );
+    }
+
+    #[test]
+    fn test_output_format_renderer_name() {
+        assert_eq!(OutputFormat::Plain.renderer_name(), None);
+        assert_eq!(OutputFormat::Markdown.renderer_name(), Some("markdown"));
+        assert_eq!(OutputFormat::Html.renderer_name(), Some("html"));
+        assert_eq!(OutputFormat::Djot.renderer_name(), Some("djot"));
+        assert_eq!(OutputFormat::Json.renderer_name(), None);
+        assert_eq!(OutputFormat::Structured.renderer_name(), None);
+        assert_eq!(OutputFormat::Custom("docx".to_string()).renderer_name(), Some("docx"));
+    }
+}

crates/kreuzberg/src/core/config/html_output.rs

@@ -0,0 +1,136 @@
+//! HTML output configuration.
+//!
+//! Controls how `OutputFormat::Html` renders an `InternalDocument`:
+//! which built-in theme to use, whether to embed the CSS in a `<style>`
+//! block, and optional user-supplied CSS (inline string or file path).
+
+use std::path::PathBuf;
+
+use serde::{Deserialize, Serialize};
+
+fn default_class_prefix() -> String {
+    "kb-".to_string()
+}
+
+fn default_true() -> bool {
+    true
+}
+
+/// Configuration for styled HTML output.
+///
+/// When set on [`ExtractionConfig::html_output`] alongside
+/// `output_format = OutputFormat::Html`, the pipeline builds a
+/// [`StyledHtmlRenderer`](crate::rendering::StyledHtmlRenderer) instead of
+/// the plain comrak-based renderer.
+///
+/// # Example
+///
+/// ```rust
+/// use kreuzberg::core::config::{HtmlOutputConfig, HtmlTheme};
+///
+/// let config = HtmlOutputConfig {
+///     theme: HtmlTheme::GitHub,
+///     css: Some(".kb-p { font-size: 1.1rem; }".to_string()),
+///     ..Default::default()
+/// };
+/// ```
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct HtmlOutputConfig {
+    /// Inline CSS string injected into the output after the theme stylesheet.
+    /// Concatenated after `css_file` content when both are set.
+    #[serde(default)]
+    pub css: Option<String>,
+
+    /// Path to a CSS file loaded once at renderer construction time.
+    /// Concatenated before `css` when both are set.
+    #[serde(default)]
+    pub css_file: Option<PathBuf>,
+
+    /// Built-in colour/typography theme. Default: [`HtmlTheme::Unstyled`].
+    #[serde(default)]
+    pub theme: HtmlTheme,
+
+    /// CSS class prefix applied to every emitted class name.
+    ///
+    /// Default: `"kb-"`. Change this if your host application already uses
+    /// classes that start with `kb-`.
+    #[serde(default = "default_class_prefix")]
+    pub class_prefix: String,
+
+    /// When `true` (default), write the resolved CSS into a `<style>` block
+    /// immediately after the opening `<div class="{prefix}doc">`.
+    ///
+    /// Set to `false` to emit only the structural markup and wire up your
+    /// own stylesheet targeting the `kb-*` class names.
+    #[serde(default = "default_true")]
+    pub embed_css: bool,
+}
+
+impl Default for HtmlOutputConfig {
+    fn default() -> Self {
+        Self {
+            css: None,
+            css_file: None,
+            theme: HtmlTheme::Unstyled,
+            class_prefix: default_class_prefix(),
+            embed_css: true,
+        }
+    }
+}
+
+/// Built-in HTML theme selection.
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum HtmlTheme {
+    /// Sensible defaults: system font stack, neutral colours, readable line
+    /// measure. CSS custom properties (`--kb-*`) are all defined so user CSS
+    /// can override individual values.
+    Default,
+    /// GitHub Markdown-inspired palette and spacing.
+    GitHub,
+    /// Dark background, light text.
+    Dark,
+    /// Minimal light theme with generous whitespace.
+    Light,
+    /// No built-in stylesheet emitted. CSS custom properties are still defined
+    /// on `:root` so user stylesheets can reference `var(--kb-*)` tokens.
+    #[default]
+    Unstyled,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn default_config_values() {
+        let cfg = HtmlOutputConfig::default();
+        assert_eq!(cfg.class_prefix, "kb-");
+        assert!(cfg.embed_css);
+        assert!(cfg.css.is_none());
+        assert!(cfg.css_file.is_none());
+        assert_eq!(cfg.theme, HtmlTheme::Unstyled);
+    }
+
+    #[test]
+    fn serde_roundtrip() {
+        let cfg = HtmlOutputConfig {
+            css: Some(".kb-p { color: red; }".to_string()),
+            theme: HtmlTheme::GitHub,
+            embed_css: false,
+            ..Default::default()
+        };
+        let json = serde_json::to_string(&cfg).unwrap();
+        let back: HtmlOutputConfig = serde_json::from_str(&json).unwrap();
+        assert_eq!(back.css, cfg.css);
+        assert_eq!(back.theme, HtmlTheme::GitHub);
+        assert!(!back.embed_css);
+    }
+
+    #[test]
+    fn theme_serde() {
+        assert_eq!(serde_json::to_string(&HtmlTheme::GitHub).unwrap(), "\"github\"");
+        let t: HtmlTheme = serde_json::from_str("\"dark\"").unwrap();
+        assert_eq!(t, HtmlTheme::Dark);
+    }
+}

crates/kreuzberg/src/core/config/layout.rs

@@ -0,0 +1,180 @@
+//! Layout detection configuration.
+
+use std::fmt;
+
+use serde::{Deserialize, Serialize};
+
+/// Which table structure recognition model to use.
+///
+/// Controls the model used for table cell detection within layout-detected
+/// table regions. Wire format is snake_case in all serializers (JSON, TOML,
+/// YAML).
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TableModel {
+    /// TATR (Table Transformer) -- default, 30MB, DETR-based row/column detection.
+    #[default]
+    Tatr,
+    /// SLANeXT wired variant -- 365MB, optimized for bordered tables.
+    SlanetWired,
+    /// SLANeXT wireless variant -- 365MB, optimized for borderless tables.
+    SlanetWireless,
+    /// SLANet-plus -- 7.78MB, lightweight general-purpose.
+    SlanetPlus,
+    /// Classifier-routed SLANeXT: auto-select wired/wireless per table.
+    /// Uses PP-LCNet classifier (6.78MB) + both SLANeXT variants (730MB total).
+    SlanetAuto,
+    /// Disable table structure model inference entirely; use heuristic path only.
+    Disabled,
+}
+
+impl std::str::FromStr for TableModel {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "tatr" => Ok(Self::Tatr),
+            "slanet_wired" => Ok(Self::SlanetWired),
+            "slanet_wireless" => Ok(Self::SlanetWireless),
+            "slanet_plus" => Ok(Self::SlanetPlus),
+            "slanet_auto" => Ok(Self::SlanetAuto),
+            "disabled" => Ok(Self::Disabled),
+            other => Err(format!(
+                "unknown table model: '{other}'. Valid: tatr, slanet_wired, slanet_wireless, slanet_plus, slanet_auto, disabled"
+            )),
+        }
+    }
+}
+
+impl fmt::Display for TableModel {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            TableModel::Tatr => write!(f, "tatr"),
+            TableModel::SlanetWired => write!(f, "slanet_wired"),
+            TableModel::SlanetWireless => write!(f, "slanet_wireless"),
+            TableModel::SlanetPlus => write!(f, "slanet_plus"),
+            TableModel::SlanetAuto => write!(f, "slanet_auto"),
+            TableModel::Disabled => write!(f, "disabled"),
+        }
+    }
+}
+
+/// Layout detection configuration.
+///
+/// Controls layout detection behavior in the extraction pipeline.
+/// When set on [`ExtractionConfig`](super::ExtractionConfig), layout detection
+/// is enabled for PDF extraction.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct LayoutDetectionConfig {
+    /// Confidence threshold override (None = use model default).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub confidence_threshold: Option<f32>,
+
+    /// Whether to apply postprocessing heuristics (default: true).
+    #[serde(default = "default_true")]
+    pub apply_heuristics: bool,
+
+    /// Table structure recognition model.
+    ///
+    /// Controls which model is used for table cell detection within layout-detected
+    /// table regions. Defaults to [`TableModel::Tatr`].
+    #[serde(default)]
+    pub table_model: TableModel,
+
+    /// Hardware acceleration for ONNX models (layout detection + table structure).
+    ///
+    /// When set, controls which execution provider (CPU, CUDA, CoreML, TensorRT)
+    /// is used for inference. Defaults to `None` (auto-select per platform).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub acceleration: Option<super::acceleration::AccelerationConfig>,
+}
+
+impl Default for LayoutDetectionConfig {
+    fn default() -> Self {
+        Self {
+            confidence_threshold: None,
+            apply_heuristics: true,
+            table_model: TableModel::default(),
+            acceleration: None,
+        }
+    }
+}
+
+fn default_true() -> bool {
+    true
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_default_config() {
+        let config = LayoutDetectionConfig::default();
+        assert_eq!(config.table_model, TableModel::Tatr);
+        assert!(config.apply_heuristics);
+        assert!(config.confidence_threshold.is_none());
+    }
+
+    #[test]
+    fn test_table_model_deserialize() {
+        let json = r#""tatr""#;
+        let model: TableModel = serde_json::from_str(json).unwrap();
+        assert_eq!(model, TableModel::Tatr);
+
+        let json = r#""slanet_auto""#;
+        let model: TableModel = serde_json::from_str(json).unwrap();
+        assert_eq!(model, TableModel::SlanetAuto);
+
+        let json = r#""disabled""#;
+        let model: TableModel = serde_json::from_str(json).unwrap();
+        assert_eq!(model, TableModel::Disabled);
+    }
+
+    #[test]
+    fn test_table_model_serialize() {
+        let json = serde_json::to_string(&TableModel::SlanetWired).unwrap();
+        assert_eq!(json, r#""slanet_wired""#);
+    }
+
+    #[test]
+    fn test_table_model_round_trip() {
+        for model in [
+            TableModel::Tatr,
+            TableModel::SlanetWired,
+            TableModel::SlanetWireless,
+            TableModel::SlanetPlus,
+            TableModel::SlanetAuto,
+            TableModel::Disabled,
+        ] {
+            let serialized = serde_json::to_string(&model).unwrap();
+            let parsed: TableModel = serde_json::from_str(&serialized).unwrap();
+            assert_eq!(parsed, model, "round-trip failed for {model:?}");
+        }
+    }
+
+    #[test]
+    fn test_backward_compat_unknown_fields_ignored() {
+        // Old configs with "preset" field should still deserialize because
+        // serde ignores unknown fields by default.
+        let json = r#"{"preset": "accurate", "apply_heuristics": true}"#;
+        let config: LayoutDetectionConfig = serde_json::from_str(json).unwrap();
+        assert!(config.apply_heuristics);
+        assert_eq!(config.table_model, TableModel::Tatr);
+    }
+
+    #[test]
+    fn test_backward_compat_old_table_model_field() {
+        // Old configs with table_model as a string should still work
+        let json = r#"{"table_model": "slanet_wired"}"#;
+        let config: LayoutDetectionConfig = serde_json::from_str(json).unwrap();
+        assert_eq!(config.table_model, TableModel::SlanetWired);
+    }
+
+    #[test]
+    fn test_table_model_display() {
+        assert_eq!(TableModel::Tatr.to_string(), "tatr");
+        assert_eq!(TableModel::SlanetWired.to_string(), "slanet_wired");
+        assert_eq!(TableModel::Disabled.to_string(), "disabled");
+    }
+}

crates/kreuzberg/src/core/config/llm.rs

@@ -0,0 +1,154 @@
+//! LLM configuration types for liter-llm integration.
+//!
+//! These types are always available (not feature-gated) since they are
+//! pure configuration data with no runtime dependency on liter-llm.
+
+use serde::{Deserialize, Serialize};
+
+/// Configuration for an LLM provider/model via liter-llm.
+///
+/// Each feature (VLM OCR, VLM embeddings, structured extraction) carries
+/// its own `LlmConfig`, allowing different providers per feature.
+///
+/// # Example
+///
+/// ```toml
+/// [structured_extraction.llm]
+/// model = "openai/gpt-4o"
+/// api_key = "sk-..."  # or use KREUZBERG_LLM_API_KEY env var
+/// ```
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct LlmConfig {
+    /// Provider/model string using liter-llm routing format.
+    ///
+    /// Examples: `"openai/gpt-4o"`, `"anthropic/claude-sonnet-4-20250514"`,
+    /// `"groq/llama-3.1-70b-versatile"`.
+    pub model: String,
+
+    /// API key for the provider. When `None`, liter-llm falls back to
+    /// the provider's standard environment variable (e.g., `OPENAI_API_KEY`).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub api_key: Option<String>,
+
+    /// Custom base URL override for the provider endpoint.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub base_url: Option<String>,
+
+    /// Request timeout in seconds (default: 60).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub timeout_secs: Option<u64>,
+
+    /// Maximum retry attempts (default: 3).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub max_retries: Option<u32>,
+
+    /// Sampling temperature for generation tasks.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub temperature: Option<f64>,
+
+    /// Maximum tokens to generate.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub max_tokens: Option<u64>,
+}
+
+/// Configuration for LLM-based structured data extraction.
+///
+/// Sends extracted document content to a VLM with a JSON schema,
+/// returning structured data that conforms to the schema.
+///
+/// # Example
+///
+/// ```toml
+/// [structured_extraction]
+/// schema_name = "invoice_data"
+/// strict = true
+///
+/// [structured_extraction.schema]
+/// type = "object"
+/// properties.vendor = { type = "string" }
+/// properties.total = { type = "number" }
+/// required = ["vendor", "total"]
+///
+/// [structured_extraction.llm]
+/// model = "openai/gpt-4o"
+/// ```
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct StructuredExtractionConfig {
+    /// JSON Schema defining the desired output structure.
+    pub schema: serde_json::Value,
+
+    /// Schema name passed to the LLM's structured output mode.
+    #[serde(default = "default_schema_name")]
+    pub schema_name: String,
+
+    /// Optional schema description for the LLM.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub schema_description: Option<String>,
+
+    /// Enable strict mode — output must exactly match the schema.
+    #[serde(default)]
+    pub strict: bool,
+
+    /// Custom Jinja2 extraction prompt template. When `None`, a default template is used.
+    ///
+    /// Available template variables:
+    /// - `{{ content }}` — The extracted document text.
+    /// - `{{ schema }}` — The JSON schema as a formatted string.
+    /// - `{{ schema_name }}` — The schema name.
+    /// - `{{ schema_description }}` — The schema description (may be empty).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub prompt: Option<String>,
+
+    /// LLM configuration for the extraction.
+    pub llm: LlmConfig,
+}
+
+fn default_schema_name() -> String {
+    "extraction".to_string()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Regression test for https://github.com/kreuzberg-dev/kreuzberg/issues/716
+    ///
+    /// `LlmConfig` must implement `Default` so callers can use the struct-update
+    /// syntax documented in the VLM OCR guide:
+    ///
+    /// ```rust
+    /// use kreuzberg::core::config::LlmConfig;
+    /// let cfg = LlmConfig {
+    ///     model: "openai/gpt-4o-mini".to_string(),
+    ///     ..Default::default()
+    /// };
+    /// ```
+    #[test]
+    fn test_llm_config_default_trait_is_satisfied() {
+        let cfg = LlmConfig::default();
+        assert!(cfg.model.is_empty(), "default model should be empty string");
+        assert!(cfg.api_key.is_none());
+        assert!(cfg.base_url.is_none());
+        assert!(cfg.timeout_secs.is_none());
+        assert!(cfg.max_retries.is_none());
+        assert!(cfg.temperature.is_none());
+        assert!(cfg.max_tokens.is_none());
+    }
+
+    /// Verify the struct-update pattern from the issue compiles and produces
+    /// only the explicitly set field.
+    #[test]
+    fn test_llm_config_struct_update_syntax() {
+        let cfg = LlmConfig {
+            model: "openai/gpt-4o-mini".to_string(),
+            ..Default::default()
+        };
+        assert_eq!(cfg.model, "openai/gpt-4o-mini");
+        assert!(cfg.api_key.is_none());
+        assert!(cfg.base_url.is_none());
+        assert!(cfg.timeout_secs.is_none());
+        assert!(cfg.max_retries.is_none());
+        assert!(cfg.temperature.is_none());
+        assert!(cfg.max_tokens.is_none());
+    }
+}

crates/kreuzberg/src/core/config/merge.rs

@@ -0,0 +1,150 @@
+//! JSON-level configuration merging.
+//!
+//! Provides a unified merge function for combining a base `ExtractionConfig` with
+//! JSON overrides. Used by both the CLI (`--config-json`) and MCP server to apply
+//! partial configuration overrides without losing unspecified fields.
+
+use super::ExtractionConfig;
+
+/// Merge extraction configuration using JSON-level field override.
+///
+/// Serializes the base config to JSON, merges each field from the override JSON
+/// (top-level only), and deserializes back. This correctly handles boolean fields
+/// explicitly set to their default values — the override always wins for any field
+/// present in `override_json`.
+///
+/// Fields **not** present in `override_json` are preserved from `base`.
+///
+/// # Errors
+///
+/// Returns `Err` if the base config cannot be serialized, or if the merged JSON
+/// cannot be deserialized back into `ExtractionConfig` (e.g., wrong field types).
+///
+/// # Examples
+///
+/// ```rust,ignore
+/// use kreuzberg::ExtractionConfig;
+/// use serde_json::json;
+///
+/// let mut base = ExtractionConfig::default();
+/// base.use_cache = true;
+///
+/// let overrides = r#"{"force_ocr": true}"#;
+/// let merged = kreuzberg::core::config::merge::merge_config_json(&base, overrides).unwrap();
+/// assert!(merged.use_cache);   // preserved from base
+/// assert!(merged.force_ocr);   // applied from override
+/// ```
+#[cfg_attr(alef, alef(skip))]
+pub fn merge_config_json(base: &ExtractionConfig, override_json: &str) -> Result<ExtractionConfig, String> {
+    let override_value: serde_json::Value =
+        serde_json::from_str(override_json).map_err(|e| format!("Failed to parse override JSON: {e}"))?;
+
+    let mut config_json =
+        serde_json::to_value(base).map_err(|e| format!("Failed to serialize base config to JSON: {e}"))?;
+
+    if let serde_json::Value::Object(json_obj) = override_value
+        && let Some(config_obj) = config_json.as_object_mut()
+    {
+        for (key, value) in json_obj {
+            config_obj.insert(key, value);
+        }
+    }
+
+    serde_json::from_value(config_json).map_err(|e| format!("Failed to deserialize merged config: {e}"))
+}
+
+/// Build extraction config by optionally merging JSON overrides into a base config.
+///
+/// If `override_json` is `None`, returns a clone of `base`. Otherwise delegates
+/// to [`merge_config_json`].
+#[cfg_attr(alef, alef(skip))]
+pub fn build_config_from_json(
+    base: &ExtractionConfig,
+    override_json: Option<&str>,
+) -> Result<ExtractionConfig, String> {
+    match override_json {
+        Some(json) => merge_config_json(base, json),
+        None => Ok(base.clone()),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_merge_preserves_unspecified_fields() {
+        let base = ExtractionConfig {
+            use_cache: false,
+            enable_quality_processing: true,
+            force_ocr: false,
+            ..Default::default()
+        };
+
+        let merged = merge_config_json(&base, r#"{"force_ocr": true}"#).unwrap();
+
+        assert!(!merged.use_cache, "use_cache should be preserved from base");
+        assert!(
+            merged.enable_quality_processing,
+            "enable_quality_processing should be preserved"
+        );
+        assert!(merged.force_ocr, "force_ocr should be overridden");
+    }
+
+    #[test]
+    fn test_merge_override_to_default_value() {
+        let base = ExtractionConfig {
+            use_cache: false,
+            ..Default::default()
+        };
+
+        let merged = merge_config_json(&base, r#"{"use_cache": true}"#).unwrap();
+        assert!(
+            merged.use_cache,
+            "Should use explicit override even if it matches the struct default"
+        );
+    }
+
+    #[test]
+    fn test_merge_multiple_fields() {
+        let base = ExtractionConfig {
+            use_cache: true,
+            force_ocr: true,
+            ..Default::default()
+        };
+
+        let merged = merge_config_json(&base, r#"{"use_cache": false, "output_format": "markdown"}"#).unwrap();
+
+        assert!(!merged.use_cache);
+        assert!(merged.force_ocr, "force_ocr should be preserved");
+        assert_eq!(
+            merged.output_format,
+            crate::core::config::formats::OutputFormat::Markdown,
+        );
+    }
+
+    #[test]
+    fn test_merge_invalid_field_type_returns_error() {
+        let base = ExtractionConfig::default();
+        let result = merge_config_json(&base, r#"{"use_cache": "not_a_boolean"}"#);
+        assert!(result.is_err());
+        assert!(result.unwrap_err().contains("Failed to deserialize"));
+    }
+
+    #[test]
+    fn test_build_config_from_json_none_returns_clone() {
+        let base = ExtractionConfig {
+            use_cache: false,
+            ..Default::default()
+        };
+        let result = build_config_from_json(&base, None).unwrap();
+        assert!(!result.use_cache);
+    }
+
+    #[test]
+    fn test_build_config_from_json_some_merges() {
+        let base = ExtractionConfig::default();
+        let result = build_config_from_json(&base, Some(r#"{"force_ocr": true}"#)).unwrap();
+        assert!(result.force_ocr);
+    }
+}

crates/kreuzberg/src/core/config/mod.rs

@@ -0,0 +1,47 @@
+//! Configuration loading and management.
+//!
+//! This module provides utilities for loading extraction configuration from various
+//! sources (TOML, YAML, JSON) and discovering configuration files in the project hierarchy.
+
+pub mod acceleration;
+pub mod concurrency;
+pub mod content_filter;
+pub mod email;
+pub mod extraction;
+pub mod formats;
+#[cfg(feature = "html")]
+pub mod html_output;
+pub mod layout;
+pub mod llm;
+pub mod merge;
+pub mod ocr;
+pub mod page;
+pub mod pdf;
+pub mod processing;
+#[cfg(feature = "tree-sitter")]
+pub mod tree_sitter;
+
+// Re-export main types for backward compatibility
+pub use acceleration::{AccelerationConfig, ExecutionProviderType};
+pub use concurrency::ConcurrencyConfig;
+pub use content_filter::ContentFilterConfig;
+pub use email::EmailConfig;
+pub use extraction::{
+    BatchBytesItem, BatchFileItem, ExtractionConfig, FileExtractionConfig, ImageExtractionConfig,
+    LanguageDetectionConfig, TokenReductionOptions,
+};
+pub use formats::OutputFormat;
+#[cfg(feature = "html")]
+pub use html_output::{HtmlOutputConfig, HtmlTheme};
+#[cfg(feature = "layout-types")]
+pub use layout::{LayoutDetectionConfig, TableModel};
+pub use llm::{LlmConfig, StructuredExtractionConfig};
+pub use ocr::{OcrConfig, OcrPipelineConfig, OcrPipelineStage, OcrQualityThresholds};
+pub use page::PageConfig;
+#[cfg(feature = "pdf")]
+pub use pdf::{HierarchyConfig, PdfConfig};
+pub use processing::{
+    ChunkSizing, ChunkerType, ChunkingConfig, EmbeddingConfig, EmbeddingModelType, PostProcessorConfig,
+};
+#[cfg(feature = "tree-sitter")]
+pub use tree_sitter::{CodeContentMode, TreeSitterConfig, TreeSitterProcessConfig};

crates/kreuzberg/src/core/config/ocr.rs

@@ -0,0 +1,932 @@
+//! OCR configuration.
+//!
+//! Defines OCR-specific configuration including backend selection, language settings,
+//! Tesseract-specific parameters, quality thresholds, and multi-backend pipeline config.
+
+use serde::{Deserialize, Serialize};
+
+use super::formats::OutputFormat;
+#[cfg(test)]
+use crate::core::config_validation::validate_ocr_backend;
+#[cfg(test)]
+use crate::error::KreuzbergError;
+use crate::types::OcrElementConfig;
+
+/// Quality thresholds for OCR fallback decisions and pipeline quality gating.
+///
+/// All fields default to the values that match the previous hardcoded behavior,
+/// so `OcrQualityThresholds::default()` preserves existing semantics exactly.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OcrQualityThresholds {
+    /// Minimum total non-whitespace characters to consider text substantive.
+    #[serde(default = "default_min_total_non_whitespace")]
+    pub min_total_non_whitespace: usize,
+
+    /// Minimum non-whitespace characters per page on average.
+    #[serde(default = "default_min_non_whitespace_per_page")]
+    pub min_non_whitespace_per_page: f64,
+
+    /// Minimum character count for a word to be "meaningful".
+    #[serde(default = "default_min_meaningful_word_len")]
+    pub min_meaningful_word_len: usize,
+
+    /// Minimum count of meaningful words before text is accepted.
+    #[serde(default = "default_min_meaningful_words")]
+    pub min_meaningful_words: usize,
+
+    /// Minimum alphanumeric ratio (non-whitespace chars that are alphanumeric).
+    #[serde(default = "default_min_alnum_ratio")]
+    pub min_alnum_ratio: f64,
+
+    /// Minimum Unicode replacement characters (U+FFFD) to trigger OCR fallback.
+    #[serde(default = "default_min_garbage_chars")]
+    pub min_garbage_chars: usize,
+
+    /// Maximum fraction of short (1-2 char) words before text is considered fragmented.
+    #[serde(default = "default_max_fragmented_word_ratio")]
+    pub max_fragmented_word_ratio: f64,
+
+    /// Critical fragmentation threshold — triggers OCR regardless of meaningful words.
+    /// Normal English text has ~20-30% short words. 80%+ is definitive garbage.
+    #[serde(default = "default_critical_fragmented_word_ratio")]
+    pub critical_fragmented_word_ratio: f64,
+
+    /// Minimum average word length. Below this with enough words indicates garbled extraction.
+    #[serde(default = "default_min_avg_word_length")]
+    pub min_avg_word_length: f64,
+
+    /// Minimum word count before average word length check applies.
+    #[serde(default = "default_min_words_for_avg_length_check")]
+    pub min_words_for_avg_length_check: usize,
+
+    /// Minimum consecutive word repetition ratio to detect column scrambling.
+    #[serde(default = "default_min_consecutive_repeat_ratio")]
+    pub min_consecutive_repeat_ratio: f64,
+
+    /// Minimum word count before consecutive repetition check is applied.
+    #[serde(default = "default_min_words_for_repeat_check")]
+    pub min_words_for_repeat_check: usize,
+
+    /// Minimum character count for "substantive markdown" OCR skip gate.
+    #[serde(default = "default_substantive_min_chars")]
+    pub substantive_min_chars: usize,
+
+    /// Minimum character count for "non-text content" OCR skip gate.
+    #[serde(default = "default_non_text_min_chars")]
+    pub non_text_min_chars: usize,
+
+    /// Alphanumeric+whitespace ratio threshold for skip decisions.
+    #[serde(default = "default_alnum_ws_ratio_threshold")]
+    pub alnum_ws_ratio_threshold: f64,
+
+    /// Minimum quality score (0.0-1.0) for a pipeline stage result to be accepted.
+    /// If the result from a backend scores below this, try the next backend.
+    #[serde(default = "default_pipeline_min_quality")]
+    pub pipeline_min_quality: f64,
+}
+
+impl Default for OcrQualityThresholds {
+    fn default() -> Self {
+        Self {
+            min_total_non_whitespace: 64,
+            min_non_whitespace_per_page: 32.0,
+            min_meaningful_word_len: 4,
+            min_meaningful_words: 3,
+            min_alnum_ratio: 0.3,
+            min_garbage_chars: 5,
+            max_fragmented_word_ratio: 0.6,
+            critical_fragmented_word_ratio: 0.80,
+            min_avg_word_length: 2.0,
+            min_words_for_avg_length_check: 50,
+            min_consecutive_repeat_ratio: 0.08,
+            min_words_for_repeat_check: 50,
+            substantive_min_chars: 100,
+            non_text_min_chars: 20,
+            alnum_ws_ratio_threshold: 0.4,
+            pipeline_min_quality: 0.5,
+        }
+    }
+}
+
+fn default_min_total_non_whitespace() -> usize {
+    64
+}
+fn default_min_non_whitespace_per_page() -> f64 {
+    32.0
+}
+fn default_min_meaningful_word_len() -> usize {
+    4
+}
+fn default_min_meaningful_words() -> usize {
+    3
+}
+fn default_min_alnum_ratio() -> f64 {
+    0.3
+}
+fn default_min_garbage_chars() -> usize {
+    5
+}
+fn default_max_fragmented_word_ratio() -> f64 {
+    0.6
+}
+fn default_critical_fragmented_word_ratio() -> f64 {
+    0.80
+}
+fn default_min_avg_word_length() -> f64 {
+    2.0
+}
+fn default_min_words_for_avg_length_check() -> usize {
+    50
+}
+fn default_min_consecutive_repeat_ratio() -> f64 {
+    0.08
+}
+fn default_min_words_for_repeat_check() -> usize {
+    50
+}
+fn default_substantive_min_chars() -> usize {
+    100
+}
+fn default_non_text_min_chars() -> usize {
+    20
+}
+fn default_alnum_ws_ratio_threshold() -> f64 {
+    0.4
+}
+fn default_pipeline_min_quality() -> f64 {
+    0.5
+}
+
+/// A single backend stage in the OCR pipeline.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OcrPipelineStage {
+    /// Backend name: "tesseract", "paddleocr", "easyocr", or a custom registered name.
+    pub backend: String,
+
+    /// Priority weight (higher = tried first). Stages are sorted by priority descending.
+    #[serde(default = "default_priority")]
+    pub priority: u32,
+
+    /// Language override for this stage (None = use parent OcrConfig.language).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub language: Option<String>,
+
+    /// Tesseract-specific config override for this stage.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub tesseract_config: Option<crate::types::TesseractConfig>,
+
+    /// PaddleOCR-specific config for this stage.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub paddle_ocr_config: Option<serde_json::Value>,
+
+    /// VLM config override for this pipeline stage.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub vlm_config: Option<super::llm::LlmConfig>,
+
+    /// Arbitrary per-call options passed through to the backend unchanged.
+    ///
+    /// Backends that support runtime tuning (mode switching, preprocessing
+    /// flags, inference parameters, etc.) read this value and deserialize
+    /// the keys they care about. Keys unknown to the backend are silently
+    /// ignored, so options from different backends can coexist in the same
+    /// config without conflict.
+    ///
+    /// Example (custom backend):
+    /// ```json
+    /// { "mode": "fast", "enable_layout": true }
+    /// ```
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub backend_options: Option<serde_json::Value>,
+}
+
+fn default_priority() -> u32 {
+    100
+}
+
+/// Multi-backend OCR pipeline with quality-based fallback.
+///
+/// Backends are tried in priority order (highest first). After each backend
+/// produces output, quality is evaluated. If it meets `quality_thresholds.pipeline_min_quality`,
+/// the result is accepted. Otherwise the next backend is tried.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OcrPipelineConfig {
+    /// Ordered list of backends to try. Sorted by priority (descending) at runtime.
+    pub stages: Vec<OcrPipelineStage>,
+
+    /// Quality thresholds for deciding whether to accept a result or try the next backend.
+    #[serde(default)]
+    pub quality_thresholds: OcrQualityThresholds,
+}
+
+/// OCR configuration.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OcrConfig {
+    /// Whether OCR is enabled.
+    ///
+    /// Setting `enabled: false` is a shorthand for `disable_ocr: true` on the parent
+    /// [`ExtractionConfig`](crate::core::config::ExtractionConfig). Images return
+    /// metadata only; PDFs use native text extraction without OCR fallback.
+    ///
+    /// Defaults to `true`. When `false`, all other OCR settings are ignored.
+    #[serde(default = "default_ocr_enabled")]
+    pub enabled: bool,
+
+    /// OCR backend: tesseract, easyocr, paddleocr
+    #[serde(default = "default_tesseract_backend")]
+    pub backend: String,
+
+    /// Language code (e.g., "eng", "deu")
+    #[serde(default = "default_eng")]
+    pub language: String,
+
+    /// Tesseract-specific configuration (optional)
+    #[serde(default)]
+    pub tesseract_config: Option<crate::types::TesseractConfig>,
+
+    /// Output format for OCR results (optional, for format conversion)
+    #[serde(default)]
+    pub output_format: Option<OutputFormat>,
+
+    /// PaddleOCR-specific configuration (optional, JSON passthrough)
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub paddle_ocr_config: Option<serde_json::Value>,
+
+    /// Arbitrary per-call options passed through to the backend unchanged.
+    ///
+    /// Custom OCR backends and built-in backends that support runtime tuning
+    /// can read this value and deserialize the keys they care about. Keys
+    /// unknown to the backend are silently ignored.
+    ///
+    /// This is the recommended extension point for per-call parameters that
+    /// are not covered by the typed fields above (e.g. mode switching,
+    /// preprocessing flags, inference batch size).
+    ///
+    /// **Scope:** when `pipeline` is `None`, this value is propagated to the
+    /// primary stage of the auto-constructed pipeline. When `pipeline` is
+    /// explicitly set, this field has **no effect** — the caller must set
+    /// `OcrPipelineStage.backend_options` directly on the relevant stage(s)
+    /// instead.
+    ///
+    /// Example:
+    /// ```json
+    /// { "mode": "fast", "enable_layout": true, "timeout_ms": 5000 }
+    /// ```
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub backend_options: Option<serde_json::Value>,
+
+    /// OCR element extraction configuration
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub element_config: Option<OcrElementConfig>,
+
+    /// Quality thresholds for the native-text-to-OCR fallback decision.
+    /// When None, uses compiled defaults (matching previous hardcoded behavior).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub quality_thresholds: Option<OcrQualityThresholds>,
+
+    /// Multi-backend OCR pipeline configuration. When set, enables weighted
+    /// fallback across multiple OCR backends based on output quality.
+    /// When None, uses the single `backend` field (same as today).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub pipeline: Option<OcrPipelineConfig>,
+
+    /// Enable automatic page rotation based on orientation detection.
+    ///
+    /// When enabled, uses Tesseract's `DetectOrientationScript()` to detect
+    /// page orientation (0/90/180/270 degrees) before OCR. If the page is
+    /// rotated with high confidence, the image is corrected before recognition.
+    /// This is critical for handling rotated scanned documents.
+    #[serde(default)]
+    pub auto_rotate: bool,
+
+    /// VLM (Vision Language Model) OCR configuration.
+    ///
+    /// Required when `backend` is `"vlm"`. Uses liter-llm to send page
+    /// images to a vision model for text extraction.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub vlm_config: Option<super::llm::LlmConfig>,
+
+    /// Custom Jinja2 prompt template for VLM OCR.
+    ///
+    /// When `None`, uses the default template. Available variables:
+    /// - `{{ language }}` — The document language code (e.g., "eng", "deu").
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub vlm_prompt: Option<String>,
+
+    /// Hardware acceleration for ONNX Runtime models (e.g. PaddleOCR, layout detection).
+    ///
+    /// Not user-configurable via config files — injected at runtime from
+    /// `ExtractionConfig::acceleration` before each `process_image` call.
+    #[serde(skip)]
+    pub acceleration: Option<super::acceleration::AccelerationConfig>,
+
+    /// Caller-supplied Tesseract `traineddata` bytes per language code.
+    ///
+    /// Primary use case is the WASM build, which has no filesystem and cannot
+    /// download tessdata at runtime. Native builds typically rely on
+    /// `TessdataManager` and ignore this field. When present, the WASM
+    /// Tesseract backend prefers these bytes over its compile-time-bundled
+    /// English data.
+    ///
+    /// Skipped by serde to keep config files small — supply via the typed API
+    /// at runtime.
+    #[serde(skip)]
+    pub tessdata_bytes: Option<std::collections::HashMap<String, Vec<u8>>>,
+}
+
+impl Default for OcrConfig {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            backend: default_tesseract_backend(),
+            language: default_eng(),
+            tesseract_config: None,
+            output_format: None,
+            paddle_ocr_config: None,
+            backend_options: None,
+            element_config: None,
+            quality_thresholds: None,
+            pipeline: None,
+            auto_rotate: false,
+            vlm_config: None,
+            vlm_prompt: None,
+            acceleration: None,
+            tessdata_bytes: None,
+        }
+    }
+}
+
+impl OcrConfig {
+    /// Validates that the configured backend is supported.
+    ///
+    /// This method checks that the backend name is one of the supported OCR backends:
+    /// - tesseract
+    /// - easyocr
+    /// - paddleocr
+    ///
+    /// Typos in backend names are caught at configuration validation time, not at runtime.
+    /// Also validates pipeline stage backends when a pipeline is configured.
+    #[cfg(test)]
+    pub(crate) fn validate(&self) -> Result<(), KreuzbergError> {
+        validate_ocr_backend(&self.backend)?;
+        // When backend is "vlm", vlm_config must be present.
+        crate::core::config_validation::validate_vlm_backend_config(&self.backend, self.vlm_config.as_ref())?;
+        if let Some(ref pipeline) = self.pipeline {
+            for stage in &pipeline.stages {
+                validate_ocr_backend(&stage.backend)?;
+                crate::core::config_validation::validate_vlm_backend_config(&stage.backend, stage.vlm_config.as_ref())?;
+            }
+        }
+        Ok(())
+    }
+
+    /// Returns the effective quality thresholds, using configured values or defaults.
+    #[cfg(feature = "ocr")]
+    pub(crate) fn effective_thresholds(&self) -> OcrQualityThresholds {
+        self.quality_thresholds.clone().unwrap_or_default()
+    }
+
+    /// Returns the effective pipeline config.
+    ///
+    /// - If `pipeline` is explicitly set, returns it.
+    /// - If `paddle-ocr` is compiled in and the backend is the default
+    ///   (tesseract), auto-constructs `[tesseract @ 100, paddleocr @ 50]`.
+    /// - Otherwise returns `None` (single-backend mode).
+    ///
+    /// Explicit non-default backend selections are honored as-is — a silent
+    /// paddleocr fallback would mask errors from the chosen backend.
+    #[cfg(feature = "ocr")]
+    pub(crate) fn effective_pipeline(&self) -> Option<OcrPipelineConfig> {
+        if self.pipeline.is_some() {
+            return self.pipeline.clone();
+        }
+
+        #[cfg(feature = "paddle-ocr")]
+        {
+            if self.backend != default_tesseract_backend() {
+                return None;
+            }
+
+            let stages = vec![
+                OcrPipelineStage {
+                    backend: self.backend.clone(),
+                    priority: 100,
+                    language: None,
+                    tesseract_config: self.tesseract_config.clone(),
+                    paddle_ocr_config: None,
+                    vlm_config: self.vlm_config.clone(),
+                    backend_options: self.backend_options.clone(),
+                },
+                OcrPipelineStage {
+                    backend: "paddleocr".to_string(),
+                    priority: 50,
+                    language: None,
+                    tesseract_config: None,
+                    paddle_ocr_config: self.paddle_ocr_config.clone(),
+                    vlm_config: None,
+                    backend_options: None,
+                },
+            ];
+            Some(OcrPipelineConfig {
+                stages,
+                quality_thresholds: self.effective_thresholds(),
+            })
+        }
+
+        #[cfg(not(feature = "paddle-ocr"))]
+        {
+            None
+        }
+    }
+}
+
+fn default_ocr_enabled() -> bool {
+    true
+}
+
+fn default_tesseract_backend() -> String {
+    "tesseract".to_string()
+}
+
+fn default_eng() -> String {
+    "eng".to_string()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_ocr_config_default() {
+        let config = OcrConfig::default();
+        assert_eq!(config.backend, "tesseract");
+        assert_eq!(config.language, "eng");
+        assert!(config.tesseract_config.is_none());
+        assert!(config.output_format.is_none());
+    }
+
+    #[test]
+    fn test_ocr_config_with_tesseract() {
+        let config = OcrConfig {
+            backend: "tesseract".to_string(),
+            language: "fra".to_string(),
+            ..Default::default()
+        };
+        assert_eq!(config.backend, "tesseract");
+        assert_eq!(config.language, "fra");
+    }
+
+    #[test]
+    fn test_validate_tesseract_backend() {
+        let config = OcrConfig {
+            backend: "tesseract".to_string(),
+            ..Default::default()
+        };
+        assert!(config.validate().is_ok());
+    }
+
+    #[test]
+    fn test_validate_easyocr_backend() {
+        let config = OcrConfig {
+            backend: "easyocr".to_string(),
+            ..Default::default()
+        };
+        assert!(config.validate().is_ok());
+    }
+
+    #[test]
+    fn test_validate_paddleocr_backend() {
+        let config = OcrConfig {
+            backend: "paddleocr".to_string(),
+            ..Default::default()
+        };
+        assert!(config.validate().is_ok());
+    }
+
+    #[test]
+    fn test_validate_invalid_backend_typo() {
+        let config = OcrConfig {
+            backend: "tesseract_typo".to_string(),
+            ..Default::default()
+        };
+        let result = config.validate();
+        assert!(result.is_err());
+        let err_msg = result.unwrap_err().to_string();
+        assert!(err_msg.contains("Invalid OCR backend"));
+    }
+
+    #[test]
+    fn test_validate_invalid_backend_completely_wrong() {
+        let config = OcrConfig {
+            backend: "ocr_lib".to_string(),
+            ..Default::default()
+        };
+        let result = config.validate();
+        assert!(result.is_err());
+        let err_msg = result.unwrap_err().to_string();
+        assert!(err_msg.contains("Invalid OCR backend") || err_msg.contains("Valid options are"));
+    }
+
+    #[test]
+    fn test_validate_default_backend() {
+        let config = OcrConfig::default();
+        assert!(config.validate().is_ok());
+    }
+
+    // ── effective_pipeline tests ──
+
+    #[cfg(feature = "ocr")]
+    #[test]
+    fn test_effective_pipeline_explicit_pipeline_returned_unchanged() {
+        let explicit_pipeline = OcrPipelineConfig {
+            stages: vec![OcrPipelineStage {
+                backend: "easyocr".to_string(),
+                priority: 200,
+                language: Some("fra".to_string()),
+                tesseract_config: None,
+                paddle_ocr_config: None,
+                vlm_config: None,
+                backend_options: None,
+            }],
+            quality_thresholds: OcrQualityThresholds::default(),
+        };
+        let config = OcrConfig {
+            pipeline: Some(explicit_pipeline.clone()),
+            ..Default::default()
+        };
+        let result = config.effective_pipeline().unwrap();
+        assert_eq!(result.stages.len(), 1);
+        assert_eq!(result.stages[0].backend, "easyocr");
+        assert_eq!(result.stages[0].priority, 200);
+        assert_eq!(result.stages[0].language, Some("fra".to_string()));
+    }
+
+    #[cfg(feature = "ocr")]
+    #[test]
+    fn test_effective_pipeline_explicit_paddleocr_no_autofallback() {
+        let config = OcrConfig {
+            backend: "paddleocr".to_string(),
+            ..Default::default()
+        };
+        assert!(config.effective_pipeline().is_none());
+    }
+
+    #[cfg(feature = "ocr")]
+    #[test]
+    fn test_effective_pipeline_explicit_easyocr_no_autofallback() {
+        let config = OcrConfig {
+            backend: "easyocr".to_string(),
+            ..Default::default()
+        };
+        assert!(config.effective_pipeline().is_none());
+    }
+
+    #[cfg(feature = "ocr")]
+    #[test]
+    fn test_effective_pipeline_default_tesseract_backend() {
+        let config = OcrConfig::default();
+        let result = config.effective_pipeline();
+        #[cfg(feature = "paddle-ocr")]
+        {
+            let pipeline = result.unwrap();
+            assert_eq!(pipeline.stages.len(), 2);
+            assert_eq!(pipeline.stages[0].backend, "tesseract");
+            assert_eq!(pipeline.stages[0].priority, 100);
+            assert_eq!(pipeline.stages[1].backend, "paddleocr");
+            assert_eq!(pipeline.stages[1].priority, 50);
+        }
+        #[cfg(not(feature = "paddle-ocr"))]
+        {
+            assert!(result.is_none());
+        }
+    }
+
+    #[cfg(feature = "ocr")]
+    #[test]
+    fn test_effective_thresholds_custom_vs_default() {
+        // With custom thresholds
+        let custom = OcrQualityThresholds {
+            min_total_non_whitespace: 128,
+            min_meaningful_words: 10,
+            ..Default::default()
+        };
+        let config_custom = OcrConfig {
+            quality_thresholds: Some(custom.clone()),
+            ..Default::default()
+        };
+        let eff = config_custom.effective_thresholds();
+        assert_eq!(eff.min_total_non_whitespace, 128);
+        assert_eq!(eff.min_meaningful_words, 10);
+
+        // Without custom thresholds (should return defaults)
+        let config_default = OcrConfig::default();
+        let eff_default = config_default.effective_thresholds();
+        assert_eq!(eff_default.min_total_non_whitespace, 64);
+        assert_eq!(eff_default.min_meaningful_words, 3);
+    }
+
+    // ── Serde tests ──
+
+    #[test]
+    fn test_pipeline_config_serde_roundtrip() {
+        let pipeline = OcrPipelineConfig {
+            stages: vec![
+                OcrPipelineStage {
+                    backend: "tesseract".to_string(),
+                    priority: 100,
+                    language: Some("eng".to_string()),
+                    tesseract_config: None,
+                    paddle_ocr_config: None,
+                    vlm_config: None,
+                    backend_options: None,
+                },
+                OcrPipelineStage {
+                    backend: "paddleocr".to_string(),
+                    priority: 50,
+                    language: None,
+                    tesseract_config: None,
+                    paddle_ocr_config: Some(serde_json::json!({"use_gpu": false})),
+                    vlm_config: None,
+                    backend_options: None,
+                },
+            ],
+            quality_thresholds: OcrQualityThresholds::default(),
+        };
+        let json = serde_json::to_string(&pipeline).unwrap();
+        let deserialized: OcrPipelineConfig = serde_json::from_str(&json).unwrap();
+        assert_eq!(deserialized.stages.len(), 2);
+        assert_eq!(deserialized.stages[0].backend, "tesseract");
+        assert_eq!(deserialized.stages[0].priority, 100);
+        assert_eq!(deserialized.stages[1].backend, "paddleocr");
+        assert_eq!(deserialized.stages[1].priority, 50);
+        assert!(deserialized.stages[1].paddle_ocr_config.is_some());
+    }
+
+    #[test]
+    fn test_pipeline_stage_deserialization_missing_optional_fields() {
+        // Only backend is required; everything else should use defaults
+        let json = r#"{"backend": "tesseract"}"#;
+        let stage: OcrPipelineStage = serde_json::from_str(json).unwrap();
+        assert_eq!(stage.backend, "tesseract");
+        assert_eq!(stage.priority, 100); // default_priority
+        assert!(stage.language.is_none());
+        assert!(stage.tesseract_config.is_none());
+        assert!(stage.paddle_ocr_config.is_none());
+    }
+
+    #[test]
+    fn test_pipeline_stage_default_priority_is_100() {
+        let json = r#"{"backend": "easyocr"}"#;
+        let stage: OcrPipelineStage = serde_json::from_str(json).unwrap();
+        assert_eq!(stage.priority, 100);
+    }
+
+    #[test]
+    fn test_ocr_config_deserialization_missing_optional_fields() {
+        let json = r#"{}"#;
+        let config: OcrConfig = serde_json::from_str(json).unwrap();
+        assert_eq!(config.backend, "tesseract");
+        assert_eq!(config.language, "eng");
+        assert!(config.pipeline.is_none());
+        assert!(config.quality_thresholds.is_none());
+        assert!(config.element_config.is_none());
+    }
+
+    #[test]
+    fn test_quality_thresholds_deserialization_partial() {
+        let json = r#"{"min_total_non_whitespace": 256}"#;
+        let thresholds: OcrQualityThresholds = serde_json::from_str(json).unwrap();
+        assert_eq!(thresholds.min_total_non_whitespace, 256);
+        // All other fields should be defaults
+        assert_eq!(thresholds.min_meaningful_words, 3);
+        assert_eq!(thresholds.min_garbage_chars, 5);
+        assert!((thresholds.pipeline_min_quality - 0.5).abs() < f64::EPSILON);
+    }
+
+    // ── Validation tests ──
+
+    #[test]
+    fn test_validate_catches_invalid_pipeline_stage_backend() {
+        let config = OcrConfig {
+            pipeline: Some(OcrPipelineConfig {
+                stages: vec![
+                    OcrPipelineStage {
+                        backend: "tesseract".to_string(),
+                        priority: 100,
+                        language: None,
+                        tesseract_config: None,
+                        paddle_ocr_config: None,
+                        vlm_config: None,
+                        backend_options: None,
+                    },
+                    OcrPipelineStage {
+                        backend: "invalid_backend".to_string(),
+                        priority: 50,
+                        language: None,
+                        tesseract_config: None,
+                        paddle_ocr_config: None,
+                        vlm_config: None,
+                        backend_options: None,
+                    },
+                ],
+                quality_thresholds: OcrQualityThresholds::default(),
+            }),
+            ..Default::default()
+        };
+        let result = config.validate();
+        assert!(result.is_err(), "Should catch invalid backend in pipeline stages");
+        let err_msg = result.unwrap_err().to_string();
+        assert!(err_msg.contains("Invalid OCR backend") || err_msg.contains("invalid_backend"));
+    }
+
+    // ── backend_options tests ──
+
+    #[test]
+    fn test_ocr_config_backend_options_default_is_none() {
+        let config = OcrConfig::default();
+        assert!(config.backend_options.is_none());
+    }
+
+    #[test]
+    fn test_ocr_config_backend_options_serde_roundtrip() {
+        let config = OcrConfig {
+            backend_options: Some(serde_json::json!({"mode": "fast", "threshold": 0.8, "enable_layout": true})),
+            ..Default::default()
+        };
+        let json = serde_json::to_string(&config).unwrap();
+        let deserialized: OcrConfig = serde_json::from_str(&json).unwrap();
+        let opts = deserialized.backend_options.unwrap();
+        assert_eq!(opts["mode"], "fast");
+        assert!((opts["threshold"].as_f64().unwrap() - 0.8).abs() < f64::EPSILON);
+        assert_eq!(opts["enable_layout"], true);
+    }
+
+    #[test]
+    fn test_ocr_config_backend_options_omitted_when_none() {
+        let config = OcrConfig::default();
+        let json = serde_json::to_string(&config).unwrap();
+        assert!(
+            !json.contains("backend_options"),
+            "backend_options must be omitted when None"
+        );
+    }
+
+    #[test]
+    fn test_pipeline_stage_backend_options_serde_roundtrip() {
+        let stage = OcrPipelineStage {
+            backend: "custom".to_string(),
+            priority: 80,
+            language: None,
+            tesseract_config: None,
+            paddle_ocr_config: None,
+            vlm_config: None,
+            backend_options: Some(serde_json::json!({"batch_size": 4, "device": "cpu"})),
+        };
+        let json = serde_json::to_string(&stage).unwrap();
+        let deserialized: OcrPipelineStage = serde_json::from_str(&json).unwrap();
+        let opts = deserialized.backend_options.unwrap();
+        assert_eq!(opts["batch_size"], 4);
+        assert_eq!(opts["device"], "cpu");
+    }
+
+    #[test]
+    fn test_pipeline_stage_backend_options_omitted_when_none() {
+        let stage = OcrPipelineStage {
+            backend: "tesseract".to_string(),
+            priority: 100,
+            language: None,
+            tesseract_config: None,
+            paddle_ocr_config: None,
+            vlm_config: None,
+            backend_options: None,
+        };
+        let json = serde_json::to_string(&stage).unwrap();
+        assert!(
+            !json.contains("backend_options"),
+            "backend_options must be omitted when None"
+        );
+    }
+
+    #[cfg(all(feature = "ocr", feature = "paddle-ocr"))]
+    #[test]
+    fn test_effective_pipeline_propagates_backend_options_to_primary_stage() {
+        let config = OcrConfig {
+            backend_options: Some(serde_json::json!({"mode": "fast"})),
+            ..Default::default()
+        };
+        let pipeline = config
+            .effective_pipeline()
+            .expect("paddle-ocr feature must produce a pipeline");
+        assert_eq!(pipeline.stages.len(), 2);
+
+        let primary = &pipeline.stages[0];
+        assert_eq!(primary.backend, "tesseract");
+        let opts = primary
+            .backend_options
+            .as_ref()
+            .expect("primary stage must carry backend_options");
+        assert_eq!(opts["mode"], "fast");
+
+        // PaddleOCR stage should not inherit backend_options from the top-level config.
+        let fallback = &pipeline.stages[1];
+        assert_eq!(fallback.backend, "paddleocr");
+        assert!(
+            fallback.backend_options.is_none(),
+            "paddleocr stage must not inherit backend_options"
+        );
+    }
+
+    #[cfg(feature = "ocr")]
+    #[test]
+    fn test_explicit_pipeline_ignores_top_level_backend_options() {
+        // When the caller provides an explicit pipeline, OcrConfig.backend_options
+        // must NOT be injected into the returned stages — the stage owns its own value.
+        let config = OcrConfig {
+            backend_options: Some(serde_json::json!({"mode": "fast"})),
+            pipeline: Some(OcrPipelineConfig {
+                stages: vec![OcrPipelineStage {
+                    backend: "tesseract".to_string(),
+                    priority: 100,
+                    language: None,
+                    tesseract_config: None,
+                    paddle_ocr_config: None,
+                    vlm_config: None,
+                    backend_options: None,
+                }],
+                quality_thresholds: OcrQualityThresholds::default(),
+            }),
+            ..Default::default()
+        };
+        let pipeline = config
+            .effective_pipeline()
+            .expect("explicit pipeline must be returned as-is");
+        assert_eq!(pipeline.stages.len(), 1);
+        assert!(
+            pipeline.stages[0].backend_options.is_none(),
+            "top-level backend_options must not be injected into an explicit pipeline stage"
+        );
+    }
+
+    #[cfg(feature = "ocr")]
+    #[test]
+    fn test_stage_level_backend_options_preserved_in_explicit_pipeline() {
+        // Stage-level backend_options in an explicit pipeline are returned unchanged —
+        // neither cleared nor overridden by any top-level value.
+        let stage_opts = serde_json::json!({"device": "gpu", "batch": 8});
+        let config = OcrConfig {
+            backend_options: Some(serde_json::json!({"mode": "fast"})),
+            pipeline: Some(OcrPipelineConfig {
+                stages: vec![OcrPipelineStage {
+                    backend: "custom".to_string(),
+                    priority: 100,
+                    language: None,
+                    tesseract_config: None,
+                    paddle_ocr_config: None,
+                    vlm_config: None,
+                    backend_options: Some(stage_opts.clone()),
+                }],
+                quality_thresholds: OcrQualityThresholds::default(),
+            }),
+            ..Default::default()
+        };
+        let pipeline = config
+            .effective_pipeline()
+            .expect("explicit pipeline must be returned as-is");
+        let returned_opts = pipeline.stages[0]
+            .backend_options
+            .as_ref()
+            .expect("stage-level backend_options must be preserved");
+        assert_eq!(returned_opts["device"], "gpu");
+        assert_eq!(returned_opts["batch"], 8);
+    }
+
+    #[test]
+    fn test_validate_passes_with_valid_pipeline_stages() {
+        let config = OcrConfig {
+            pipeline: Some(OcrPipelineConfig {
+                stages: vec![
+                    OcrPipelineStage {
+                        backend: "tesseract".to_string(),
+                        priority: 100,
+                        language: None,
+                        tesseract_config: None,
+                        paddle_ocr_config: None,
+                        vlm_config: None,
+                        backend_options: None,
+                    },
+                    OcrPipelineStage {
+                        backend: "paddleocr".to_string(),
+                        priority: 50,
+                        language: None,
+                        tesseract_config: None,
+                        paddle_ocr_config: None,
+                        vlm_config: None,
+                        backend_options: None,
+                    },
+                ],
+                quality_thresholds: OcrQualityThresholds::default(),
+            }),
+            ..Default::default()
+        };
+        assert!(config.validate().is_ok());
+    }
+}

crates/kreuzberg/src/core/config/page.rs

@@ -0,0 +1,57 @@
+//! Page extraction and tracking configuration.
+//!
+//! Controls how pages are extracted, tracked, and represented in extraction results.
+//! When `None`, page tracking is disabled.
+
+use serde::{Deserialize, Serialize};
+
+/// Page extraction and tracking configuration.
+///
+/// Controls how pages are extracted, tracked, and represented in the extraction results.
+/// When `None`, page tracking is disabled.
+///
+/// Page range tracking in chunk metadata (first_page/last_page) is automatically enabled
+/// when page boundaries are available and chunking is configured.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default)]
+pub struct PageConfig {
+    /// Extract pages as separate array (ExtractionResult.pages)
+    #[serde(default)]
+    pub extract_pages: bool,
+
+    /// Insert page markers in main content string
+    #[serde(default)]
+    pub insert_page_markers: bool,
+
+    /// Page marker format (use {page_num} placeholder)
+    /// Default: "\n\n<!-- PAGE {page_num} -->\n\n"
+    #[serde(default = "default_page_marker_format")]
+    pub marker_format: String,
+}
+
+impl Default for PageConfig {
+    fn default() -> Self {
+        Self {
+            extract_pages: false,
+            insert_page_markers: false,
+            marker_format: "\n\n<!-- PAGE {page_num} -->\n\n".to_string(),
+        }
+    }
+}
+
+fn default_page_marker_format() -> String {
+    "\n\n<!-- PAGE {page_num} -->\n\n".to_string()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_page_config_default() {
+        let config = PageConfig::default();
+        assert!(!config.extract_pages);
+        assert!(!config.insert_page_markers);
+        assert_eq!(config.marker_format, "\n\n<!-- PAGE {page_num} -->\n\n");
+    }
+}

crates/kreuzberg/src/core/config/pdf.rs

@@ -0,0 +1,192 @@
+//! PDF-specific configuration.
+//!
+//! Defines PDF extraction options including metadata handling, image extraction,
+//! password management, and hierarchy extraction for document structure analysis.
+
+use serde::{Deserialize, Serialize};
+
+/// PDF-specific configuration.
+#[cfg(feature = "pdf")]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PdfConfig {
+    /// Extract images from PDF
+    #[serde(default)]
+    pub extract_images: bool,
+
+    /// Extract tables from PDF.
+    ///
+    /// When `true` (default), runs pdf_oxide's native grid detector and, if it
+    /// finds nothing, falls back to the heuristic text-layer reconstruction in
+    /// `pdf::oxide::table::extract_tables_heuristic`. Set to `false` to skip
+    /// both passes — `tables` will then be empty in the result.
+    #[serde(default = "default_true")]
+    pub extract_tables: bool,
+
+    /// List of passwords to try when opening encrypted PDFs
+    #[serde(default)]
+    pub passwords: Option<Vec<String>>,
+
+    /// Extract PDF metadata
+    #[serde(default = "default_true")]
+    pub extract_metadata: bool,
+
+    /// Hierarchy extraction configuration (None = hierarchy extraction disabled)
+    #[serde(default)]
+    pub hierarchy: Option<HierarchyConfig>,
+
+    /// Extract PDF annotations (text notes, highlights, links, stamps).
+    /// Default: false
+    #[serde(default)]
+    pub extract_annotations: bool,
+
+    /// Top margin fraction (0.0–1.0) of page height to exclude headers/running heads.
+    /// Default: 0.06 (6%)
+    #[serde(default)]
+    pub top_margin_fraction: Option<f32>,
+
+    /// Bottom margin fraction (0.0–1.0) of page height to exclude footers/page numbers.
+    /// Default: 0.05 (5%)
+    #[serde(default)]
+    pub bottom_margin_fraction: Option<f32>,
+
+    /// Allow single-column pseudo tables in extraction results.
+    ///
+    /// By default, tables with fewer than 2 columns (layout-guided) or 3 columns
+    /// (heuristic) are rejected. When `true`, the minimum column count is relaxed
+    /// to 1, allowing single-column structured data (glossaries, itemized lists)
+    /// to be emitted as tables. Other quality filters (density, sparsity, prose
+    /// detection) still apply.
+    #[serde(default)]
+    pub allow_single_column_tables: bool,
+
+    /// Perform OCR on inline images extracted from PDF pages and attach the
+    /// recognized text to each `ExtractedImage.ocr_result`. Requires Tesseract
+    /// to be available; if `ExtractionConfig.ocr` is `None` the extractor
+    /// falls back to `TesseractConfig::default()`. Per-image failures degrade
+    /// gracefully (the image is returned without OCR text rather than failing
+    /// the whole extraction). Default: `false`.
+    #[serde(default)]
+    pub ocr_inline_images: bool,
+}
+
+/// Hierarchy extraction configuration for PDF text structure analysis.
+///
+/// Enables extraction of document hierarchy levels (H1-H6) based on font size
+/// clustering and semantic analysis. When enabled, hierarchical blocks are
+/// included in page content.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct HierarchyConfig {
+    /// Enable hierarchy extraction
+    #[serde(default = "default_true")]
+    pub enabled: bool,
+
+    /// Number of font size clusters to use for hierarchy levels (1-7)
+    ///
+    /// Default: 6, which provides H1-H6 heading levels with body text.
+    /// Larger values create more fine-grained hierarchy levels.
+    #[serde(default = "default_k_clusters")]
+    pub k_clusters: usize,
+
+    /// Include bounding box information in hierarchy blocks
+    #[serde(default = "default_true")]
+    pub include_bbox: bool,
+
+    /// OCR coverage threshold for smart OCR triggering (0.0-1.0)
+    ///
+    /// Determines when OCR should be triggered based on text block coverage.
+    /// OCR is triggered when text blocks cover less than this fraction of the page.
+    /// Default: 0.5 (trigger OCR if less than 50% of page has text)
+    #[serde(default = "default_ocr_coverage_threshold")]
+    pub ocr_coverage_threshold: Option<f32>,
+}
+
+#[cfg(feature = "pdf")]
+impl Default for PdfConfig {
+    fn default() -> Self {
+        Self {
+            extract_images: false,
+            extract_tables: true,
+            passwords: None,
+            extract_metadata: true,
+            hierarchy: None,
+            extract_annotations: false,
+            top_margin_fraction: None,
+            bottom_margin_fraction: None,
+            allow_single_column_tables: false,
+            ocr_inline_images: false,
+        }
+    }
+}
+
+impl Default for HierarchyConfig {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            k_clusters: 3,
+            include_bbox: true,
+            ocr_coverage_threshold: None,
+        }
+    }
+}
+
+fn default_true() -> bool {
+    true
+}
+
+fn default_k_clusters() -> usize {
+    3
+}
+
+fn default_ocr_coverage_threshold() -> Option<f32> {
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    #[cfg(feature = "pdf")]
+    fn test_hierarchy_config_default() {
+        use super::*;
+        let config = HierarchyConfig::default();
+        assert!(config.enabled);
+        assert_eq!(config.k_clusters, 3);
+        assert!(config.include_bbox);
+        assert!(config.ocr_coverage_threshold.is_none());
+    }
+
+    #[test]
+    #[cfg(feature = "pdf")]
+    fn test_hierarchy_config_disabled() {
+        use super::*;
+        let config = HierarchyConfig {
+            enabled: false,
+            k_clusters: 3,
+            include_bbox: false,
+            ocr_coverage_threshold: Some(0.7),
+        };
+        assert!(!config.enabled);
+        assert_eq!(config.k_clusters, 3);
+        assert!(!config.include_bbox);
+        assert_eq!(config.ocr_coverage_threshold, Some(0.7));
+    }
+
+    #[test]
+    #[cfg(feature = "pdf")]
+    fn test_pdf_config_custom_margins() {
+        use super::*;
+        let config = PdfConfig {
+            extract_images: false,
+            extract_tables: true,
+            passwords: None,
+            extract_metadata: true,
+            hierarchy: None,
+            extract_annotations: false,
+            top_margin_fraction: Some(0.10),
+            bottom_margin_fraction: Some(0.08),
+            allow_single_column_tables: false,
+            ocr_inline_images: false,
+        };
+        assert_eq!(config.top_margin_fraction, Some(0.10));
+        assert_eq!(config.bottom_margin_fraction, Some(0.08));
+    }
+}

crates/kreuzberg/src/core/config/processing.rs

@@ -0,0 +1,838 @@
+//! Post-processing and chunking configuration.
+//!
+//! Defines configuration for post-processing pipelines, text chunking,
+//! and embedding generation.
+
+use ahash::AHashSet;
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+
+/// Type of text chunker to use.
+///
+/// # Variants
+///
+/// * `Text` - Generic text splitter, splits on whitespace and punctuation
+/// * `Markdown` - Markdown-aware splitter, preserves formatting and structure
+/// * `Yaml` - YAML-aware splitter, creates one chunk per top-level key
+/// * `Semantic` - Topic-aware chunker. With an `EmbeddingConfig`, splits at
+///   embedding-based topic shifts tuned by `topic_threshold` (default 0.75,
+///   lower = more splits). Without an embedding, falls back to a
+///   structural-boundary heuristic (ALL-CAPS headers, numbered sections,
+///   blank-line paragraphs) and merges groups into chunks capped at
+///   `max_characters` (default 1000). `topic_threshold` has no effect in the
+///   fallback path. For best results, pair with an embedding model.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
+#[serde(rename_all = "lowercase")]
+pub enum ChunkerType {
+    #[default]
+    Text,
+    Markdown,
+    Yaml,
+    Semantic,
+}
+
+/// How chunk size is measured.
+///
+/// Defaults to `Characters` (Unicode character count). When using token-based sizing,
+/// chunks are sized by token count according to the specified tokenizer.
+///
+/// Token-based sizing uses HuggingFace tokenizers loaded at runtime. Any tokenizer
+/// available on HuggingFace Hub can be used, including OpenAI-compatible tokenizers
+/// (e.g., `Xenova/gpt-4o`, `Xenova/cl100k_base`).
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum ChunkSizing {
+    /// Size measured in Unicode characters (default).
+    #[default]
+    Characters,
+    /// Size measured in tokens from a HuggingFace tokenizer.
+    #[cfg(feature = "chunking-tokenizers")]
+    Tokenizer {
+        /// HuggingFace model ID or path, e.g. "Xenova/gpt-4o", "bert-base-uncased".
+        model: String,
+        /// Optional cache directory override for tokenizer files.
+        /// Defaults to hf-hub's standard cache (`~/.cache/huggingface/`).
+        /// Can also be set via `KREUZBERG_TOKENIZER_CACHE_DIR` environment variable.
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        cache_dir: Option<std::path::PathBuf>,
+    },
+}
+
+/// Post-processor configuration.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PostProcessorConfig {
+    /// Enable post-processors
+    #[serde(default = "default_true")]
+    pub enabled: bool,
+
+    /// Whitelist of processor names to run (None = all enabled)
+    #[serde(default)]
+    pub enabled_processors: Option<Vec<String>>,
+
+    /// Blacklist of processor names to skip (None = none disabled)
+    #[serde(default)]
+    pub disabled_processors: Option<Vec<String>>,
+
+    /// Pre-computed AHashSet for O(1) enabled processor lookup
+    #[serde(skip)]
+    pub enabled_set: Option<AHashSet<String>>,
+
+    /// Pre-computed AHashSet for O(1) disabled processor lookup
+    #[serde(skip)]
+    pub disabled_set: Option<AHashSet<String>>,
+}
+
+impl PostProcessorConfig {
+    /// Pre-compute HashSets for O(1) processor name lookups.
+    ///
+    /// This method converts the enabled/disabled processor Vec to HashSet
+    /// for constant-time lookups in the pipeline.
+    #[cfg(test)]
+    pub(crate) fn build_lookup_sets(&mut self) {
+        if let Some(ref enabled) = self.enabled_processors {
+            self.enabled_set = Some(enabled.iter().cloned().collect());
+        }
+        if let Some(ref disabled) = self.disabled_processors {
+            self.disabled_set = Some(disabled.iter().cloned().collect());
+        }
+    }
+}
+
+impl Default for PostProcessorConfig {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            enabled_processors: None,
+            disabled_processors: None,
+            enabled_set: None,
+            disabled_set: None,
+        }
+    }
+}
+
+/// Chunking configuration.
+///
+/// Configures text chunking for document content, including chunk size,
+/// overlap, trimming behavior, and optional embeddings.
+///
+/// Use `..Default::default()` when constructing to allow for future field additions:
+/// ```rust
+/// # use kreuzberg::ChunkingConfig;
+/// let config = ChunkingConfig {
+///     max_characters: 500,
+///     ..Default::default()
+/// };
+/// ```
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ChunkingConfig {
+    /// Maximum size per chunk (in units determined by `sizing`).
+    ///
+    /// When `sizing` is `Characters` (default), this is the max character count.
+    /// When using token-based sizing, this is the max token count.
+    ///
+    /// Default: 1000
+    #[serde(default = "default_chunk_size", rename = "max_chars", alias = "max_characters")]
+    pub max_characters: usize,
+
+    /// Overlap between chunks (in units determined by `sizing`).
+    ///
+    /// Default: 200
+    #[serde(default = "default_chunk_overlap", rename = "max_overlap", alias = "overlap")]
+    pub overlap: usize,
+
+    /// Whether to trim whitespace from chunk boundaries.
+    ///
+    /// Default: true
+    #[serde(default = "default_trim")]
+    pub trim: bool,
+
+    /// Type of chunker to use (Text or Markdown).
+    ///
+    /// Default: Text
+    #[serde(default = "default_chunker_type")]
+    pub chunker_type: ChunkerType,
+
+    /// Optional embedding configuration for chunk embeddings.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub embedding: Option<EmbeddingConfig>,
+
+    /// Use a preset configuration (overrides individual settings if provided).
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub preset: Option<String>,
+
+    /// How to measure chunk size.
+    ///
+    /// Default: `Characters` (Unicode character count).
+    /// Enable `chunking-tiktoken` or `chunking-tokenizers` features for token-based sizing.
+    #[serde(default, deserialize_with = "deserialize_null_default")]
+    pub sizing: ChunkSizing,
+
+    /// When `true` and `chunker_type` is `Markdown`, prepend the heading hierarchy
+    /// path (e.g. `"# Title > ## Section\n\n"`) to each chunk's content string.
+    ///
+    /// This is useful for RAG pipelines where each chunk needs self-contained
+    /// context about its position in the document structure.
+    ///
+    /// Default: `false`
+    #[serde(default)]
+    pub prepend_heading_context: bool,
+
+    /// Optional cosine similarity threshold for semantic topic boundary detection.
+    ///
+    /// Only used when `chunker_type` is `Semantic` and an `EmbeddingConfig` is
+    /// provided. You almost never need to set this. When omitted, defaults to
+    /// `0.75` which works well for most documents. Lower values detect more
+    /// topic boundaries (more, smaller chunks); higher values detect fewer.
+    /// Range: `0.0..=1.0`.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub topic_threshold: Option<f32>,
+}
+
+impl ChunkingConfig {
+    /// Set the cosine similarity threshold for semantic topic boundary detection.
+    ///
+    /// # Panics
+    ///
+    /// Panics if `threshold` is outside `[0.0, 1.0]`.
+    #[cfg(test)]
+    pub(crate) fn with_topic_threshold(mut self, threshold: f32) -> Self {
+        assert!(
+            (0.0..=1.0).contains(&threshold),
+            "topic_threshold must be in [0.0, 1.0], got {threshold}"
+        );
+        self.topic_threshold = Some(threshold);
+        self
+    }
+
+    /// Resolve a preset name into concrete chunking and embedding configuration.
+    ///
+    /// When `preset` is set (e.g., `"balanced"`), this overrides `max_characters` and
+    /// `overlap` from the preset definition, and configures the embedding model if
+    /// no embedding config was explicitly provided.
+    ///
+    /// If the preset name is not recognized, a warning is logged and the config
+    /// is returned unchanged.
+    ///
+    /// Requires the `embeddings` feature. Without it, this is a no-op that returns
+    /// the config unchanged.
+    #[cfg(feature = "embeddings")]
+    pub(crate) fn resolve_preset(&self) -> Self {
+        let preset_name = match &self.preset {
+            Some(name) => name,
+            None => return self.clone(),
+        };
+
+        let preset = match crate::embeddings::get_preset(preset_name) {
+            Some(p) => p,
+            None => {
+                tracing::warn!(
+                    "Unknown chunking preset '{}', using manual config. Available: {:?}",
+                    preset_name,
+                    crate::embeddings::list_presets()
+                );
+                return self.clone();
+            }
+        };
+
+        // Preserve the caller's embedding choice, including None.
+        // Presets configure chunking parameters only; users must explicitly
+        // provide an EmbeddingConfig to opt into embedding generation.
+        let embedding = self.embedding.clone();
+
+        Self {
+            max_characters: preset.chunk_size,
+            overlap: preset.overlap,
+            embedding,
+            // Preserve caller's other settings
+            trim: self.trim,
+            chunker_type: self.chunker_type,
+            preset: self.preset.clone(),
+            sizing: self.sizing.clone(),
+            prepend_heading_context: self.prepend_heading_context,
+            topic_threshold: self.topic_threshold,
+        }
+    }
+
+    /// Resolve a preset name (no-op without the `embeddings` feature).
+    #[cfg(all(feature = "chunking", not(feature = "embeddings")))]
+    pub(crate) fn resolve_preset(&self) -> Self {
+        if self.preset.is_some() {
+            tracing::warn!("Chunking presets require the 'embeddings' feature");
+        }
+        self.clone()
+    }
+}
+
+impl Default for ChunkingConfig {
+    fn default() -> Self {
+        Self {
+            max_characters: 1000,
+            overlap: 200,
+            trim: true,
+            chunker_type: ChunkerType::Text,
+            embedding: None,
+            preset: None,
+            sizing: ChunkSizing::default(),
+            prepend_heading_context: false,
+            topic_threshold: None,
+        }
+    }
+}
+
+/// Embedding configuration for text chunks.
+///
+/// Configures embedding generation using ONNX models via the vendored embedding engine.
+/// Requires the `embeddings` feature to be enabled.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct EmbeddingConfig {
+    /// The embedding model to use (defaults to "balanced" preset if not specified)
+    #[serde(default = "default_model", deserialize_with = "deserialize_null_model")]
+    pub model: EmbeddingModelType,
+
+    /// Whether to normalize embedding vectors (recommended for cosine similarity)
+    #[serde(default = "default_normalize")]
+    pub normalize: bool,
+
+    /// Batch size for embedding generation
+    #[serde(default = "default_batch_size")]
+    pub batch_size: usize,
+
+    /// Show model download progress
+    #[serde(default)]
+    pub show_download_progress: bool,
+
+    /// Custom cache directory for model files
+    ///
+    /// Defaults to `~/.cache/kreuzberg/embeddings/` if not specified.
+    /// Allows full customization of model download location.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cache_dir: Option<PathBuf>,
+
+    /// Hardware acceleration for the embedding ONNX model.
+    ///
+    /// When set, controls which execution provider (CPU, CUDA, CoreML, TensorRT)
+    /// is used for inference. Defaults to `None` (auto-select per platform).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub acceleration: Option<super::acceleration::AccelerationConfig>,
+
+    /// Maximum wall-clock duration (in seconds) for a single `embed()` call when
+    /// using [`EmbeddingModelType::Plugin`].
+    ///
+    /// Applies only to the in-process plugin path — protects against hung
+    /// host-language backends (e.g. a Python callback deadlocked on the GIL,
+    /// a model stuck on CUDA OOM retries, etc.). On timeout, the dispatcher
+    /// returns [`crate::KreuzbergError::Plugin`] instead of blocking forever.
+    ///
+    /// `None` disables the timeout. The default (60 seconds) is conservative
+    /// for common in-process inference; increase for large batches on slow
+    /// hardware.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub max_embed_duration_secs: Option<u64>,
+}
+
+impl Default for EmbeddingConfig {
+    fn default() -> Self {
+        Self {
+            model: EmbeddingModelType::Preset {
+                name: "balanced".to_string(),
+            },
+            normalize: true,
+            batch_size: 32,
+            show_download_progress: false,
+            cache_dir: None,
+            acceleration: None,
+            max_embed_duration_secs: Some(60),
+        }
+    }
+}
+
+/// Embedding model types supported by Kreuzberg.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum EmbeddingModelType {
+    /// Use a preset model configuration (recommended)
+    Preset { name: String },
+
+    /// Use a custom ONNX model from HuggingFace
+    Custom { model_id: String, dimensions: usize },
+
+    /// Provider-hosted embedding model via liter-llm.
+    ///
+    /// Uses the model specified in the nested `LlmConfig` (e.g.,
+    /// `"openai/text-embedding-3-small"`).
+    Llm { llm: super::llm::LlmConfig },
+
+    /// In-process embedding backend registered via the plugin system.
+    ///
+    /// The caller registers an [`EmbeddingBackend`](crate::plugins::EmbeddingBackend) once
+    /// (e.g. a wrapper around an already-loaded `llama-cpp-python`, `sentence-transformers`,
+    /// or tuned ONNX model), then references it by name in config. Kreuzberg calls back
+    /// into the registered backend during chunking and standalone embed requests —
+    /// no HuggingFace download, no ONNX Runtime requirement, no HTTP sidecar.
+    ///
+    /// When this variant is selected, only the following [`EmbeddingConfig`] fields
+    /// apply: `normalize` (post-call L2 normalization) and `max_embed_duration_secs`
+    /// (dispatcher timeout). Model-loading fields (`batch_size`, `cache_dir`,
+    /// `show_download_progress`, `acceleration`) are ignored — the host owns the
+    /// model lifecycle.
+    ///
+    /// Semantic chunking falls back to [`ChunkingConfig::max_characters`] when this variant
+    /// is used, since there is no preset to look a chunk-size ceiling up against — size your
+    /// context window via `max_characters` directly.
+    ///
+    /// See [`crate::plugins::register_embedding_backend`].
+    Plugin { name: String },
+}
+
+impl Default for EmbeddingModelType {
+    /// Returns the "balanced" preset as the default model.
+    ///
+    /// Previously returned `Preset { name: "" }` (empty string) which caused
+    /// "Unknown embedding preset: " errors in every language binding that calls
+    /// `EmbeddingModelType::default()` — including generated bindings that
+    /// use struct-level `#[serde(default)]` instead of `default_model()`.
+    /// All defaults across the codebase converge on "balanced".
+    fn default() -> Self {
+        Self::Preset {
+            name: "balanced".to_string(),
+        }
+    }
+}
+
+fn default_true() -> bool {
+    true
+}
+
+fn default_chunk_size() -> usize {
+    1000
+}
+
+/// Deserialize a value that may be explicitly `null` into its `Default` value.
+///
+/// Internally-tagged serde enums (e.g. `#[serde(tag = "type")]`) reject `null`
+/// even when the containing field has `#[serde(default)]`, because that attribute
+/// only covers the *missing* case. Polyglot bindings frequently emit explicit
+/// `"field": null` from zero-valued mirror structs, so this helper accepts either
+/// `null` or a present value and falls back to `T::default()` for null.
+fn deserialize_null_default<'de, D, T>(deserializer: D) -> Result<T, D::Error>
+where
+    D: serde::Deserializer<'de>,
+    T: Default + serde::Deserialize<'de>,
+{
+    let opt = Option::<T>::deserialize(deserializer)?;
+    Ok(opt.unwrap_or_default())
+}
+
+fn default_chunk_overlap() -> usize {
+    200
+}
+
+fn default_trim() -> bool {
+    true
+}
+
+fn default_chunker_type() -> ChunkerType {
+    ChunkerType::Text
+}
+
+fn default_normalize() -> bool {
+    true
+}
+
+fn default_batch_size() -> usize {
+    32
+}
+
+fn default_model() -> EmbeddingModelType {
+    EmbeddingModelType::Preset {
+        name: "balanced".to_string(),
+    }
+}
+
+/// `deserialize_with` companion for `EmbeddingModelType` fields that may be
+/// explicitly `null` in polyglot binding payloads. Treats null as the configured
+/// `default_model()` (the "balanced" preset) rather than the trait `Default` impl
+/// (which is an empty-name placeholder unsuitable for live use).
+fn deserialize_null_model<'de, D>(deserializer: D) -> Result<EmbeddingModelType, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    let opt = Option::<EmbeddingModelType>::deserialize(deserializer)?;
+    Ok(opt.unwrap_or_else(default_model))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_postprocessor_config_default() {
+        let config = PostProcessorConfig::default();
+        assert!(config.enabled);
+        assert!(config.enabled_processors.is_none());
+        assert!(config.disabled_processors.is_none());
+    }
+
+    #[test]
+    fn test_postprocessor_config_build_lookup_sets() {
+        let mut config = PostProcessorConfig {
+            enabled: true,
+            enabled_processors: Some(vec!["a".to_string(), "b".to_string()]),
+            disabled_processors: Some(vec!["c".to_string()]),
+            enabled_set: None,
+            disabled_set: None,
+        };
+
+        config.build_lookup_sets();
+
+        assert!(config.enabled_set.is_some());
+        assert!(config.disabled_set.is_some());
+        assert!(config.enabled_set.unwrap().contains("a"));
+        assert!(config.disabled_set.unwrap().contains("c"));
+    }
+
+    #[test]
+    fn test_chunking_config_defaults() {
+        let config = ChunkingConfig::default();
+        assert_eq!(config.max_characters, 1000);
+        assert_eq!(config.overlap, 200);
+        assert!(config.trim);
+        assert_eq!(config.chunker_type, ChunkerType::Text);
+        assert!(matches!(config.sizing, ChunkSizing::Characters));
+    }
+
+    #[test]
+    fn test_embedding_config_default() {
+        let config = EmbeddingConfig::default();
+        assert!(config.normalize);
+        assert_eq!(config.batch_size, 32);
+        assert!(config.cache_dir.is_none());
+    }
+
+    /// Tests that `EmbeddingModelType::default()` returns the "balanced" preset.
+    ///
+    /// Language bindings that use struct-level `#[serde(default)]` resolve absent
+    /// `model` fields via this impl. An empty-string name caused "Unknown embedding
+    /// preset: " panics in `get_preset()`; the default must be a valid preset.
+    #[test]
+    fn test_embedding_model_type_default_is_balanced() {
+        match EmbeddingModelType::default() {
+            EmbeddingModelType::Preset { name } => {
+                assert_eq!(name, "balanced", "Default model should be the balanced preset");
+            }
+            other => panic!("Expected Preset variant, got {:?}", other),
+        }
+    }
+
+    /// Tests that EmbeddingModelType::Preset serializes with "type" field (internally-tagged).
+    /// This validates the API schema matches the documented format:
+    /// `{"type": "preset", "name": "fast"}` NOT `{"preset": {"name": "fast"}}`
+    #[test]
+    fn test_embedding_model_type_preset_serialization() {
+        let model = EmbeddingModelType::Preset {
+            name: "fast".to_string(),
+        };
+        let json = serde_json::to_string(&model).unwrap();
+
+        // Should use internally-tagged format with "type" discriminator
+        assert!(json.contains(r#""type":"preset""#), "Should contain type:preset field");
+        assert!(json.contains(r#""name":"fast""#), "Should contain name:fast field");
+
+        // Should NOT use adjacently-tagged format
+        assert!(
+            !json.contains(r#"{"preset":"#),
+            "Should NOT use adjacently-tagged format"
+        );
+    }
+
+    /// Tests that EmbeddingModelType::Preset deserializes from the documented API format.
+    /// API documentation shows: `{"type": "preset", "name": "fast"}`
+    #[test]
+    fn test_embedding_model_type_preset_deserialization() {
+        // This is the documented API format that users should send
+        let json = r#"{"type": "preset", "name": "fast"}"#;
+        let model: EmbeddingModelType = serde_json::from_str(json).unwrap();
+
+        match model {
+            EmbeddingModelType::Preset { name } => {
+                assert_eq!(name, "fast");
+            }
+            _ => panic!("Expected Preset variant"),
+        }
+    }
+
+    /// Tests that the wrong format (adjacently-tagged) is rejected.
+    /// This ensures the API doesn't accept the old/wrong documentation format.
+    #[test]
+    fn test_embedding_model_type_rejects_wrong_format() {
+        // This is the WRONG format that was in the old documentation
+        let wrong_json = r#"{"preset": {"name": "fast"}}"#;
+        let result: Result<EmbeddingModelType, _> = serde_json::from_str(wrong_json);
+
+        // Should fail to parse - the wrong format should be rejected
+        assert!(result.is_err(), "Should reject adjacently-tagged format");
+    }
+
+    /// Tests round-trip serialization/deserialization of EmbeddingConfig.
+    #[test]
+    fn test_embedding_config_roundtrip() {
+        let config = EmbeddingConfig {
+            model: EmbeddingModelType::Preset {
+                name: "balanced".to_string(),
+            },
+            normalize: true,
+            batch_size: 64,
+            show_download_progress: false,
+            cache_dir: None,
+            acceleration: None,
+            max_embed_duration_secs: Some(60),
+        };
+
+        let json = serde_json::to_string(&config).unwrap();
+        let deserialized: EmbeddingConfig = serde_json::from_str(&json).unwrap();
+
+        match deserialized.model {
+            EmbeddingModelType::Preset { name } => {
+                assert_eq!(name, "balanced");
+            }
+            _ => panic!("Expected Preset variant"),
+        }
+        assert!(deserialized.normalize);
+        assert_eq!(deserialized.batch_size, 64);
+    }
+
+    /// Tests Custom model type serialization format.
+    #[test]
+    fn test_embedding_model_type_custom_serialization() {
+        let model = EmbeddingModelType::Custom {
+            model_id: "sentence-transformers/all-MiniLM-L6-v2".to_string(),
+            dimensions: 384,
+        };
+        let json = serde_json::to_string(&model).unwrap();
+
+        assert!(json.contains(r#""type":"custom""#), "Should contain type:custom field");
+        assert!(json.contains(r#""model_id":"#), "Should contain model_id field");
+        assert!(json.contains(r#""dimensions":384"#), "Should contain dimensions field");
+    }
+
+    #[test]
+    #[cfg(feature = "embeddings")]
+    fn test_resolve_preset_balanced() {
+        let config = ChunkingConfig {
+            preset: Some("balanced".to_string()),
+            ..Default::default()
+        };
+        let resolved = config.resolve_preset();
+        assert_eq!(resolved.max_characters, 1024);
+        assert_eq!(resolved.overlap, 100);
+        // Preset configures chunking parameters only; embedding stays None unless
+        // the caller explicitly provided one (#797).
+        assert!(resolved.embedding.is_none());
+    }
+
+    #[test]
+    #[cfg(feature = "embeddings")]
+    fn test_resolve_preset_preserves_explicit_embedding() {
+        let explicit_embedding = EmbeddingConfig {
+            model: EmbeddingModelType::Custom {
+                model_id: "custom/model".to_string(),
+                dimensions: 512,
+            },
+            batch_size: 64,
+            ..Default::default()
+        };
+        let config = ChunkingConfig {
+            preset: Some("fast".to_string()),
+            embedding: Some(explicit_embedding),
+            ..Default::default()
+        };
+        let resolved = config.resolve_preset();
+        assert_eq!(resolved.max_characters, 512);
+        assert_eq!(resolved.overlap, 50);
+        // Explicit embedding config preserved
+        match &resolved.embedding.unwrap().model {
+            EmbeddingModelType::Custom { model_id, .. } => assert_eq!(model_id, "custom/model"),
+            _ => panic!("Expected Custom model type to be preserved"),
+        }
+    }
+
+    #[cfg(any(feature = "embeddings", feature = "chunking"))]
+    #[test]
+    fn test_resolve_preset_no_preset_returns_unchanged() {
+        let config = ChunkingConfig {
+            max_characters: 500,
+            overlap: 50,
+            ..Default::default()
+        };
+        let resolved = config.resolve_preset();
+        assert_eq!(resolved.max_characters, 500);
+        assert_eq!(resolved.overlap, 50);
+        assert!(resolved.embedding.is_none());
+    }
+
+    #[cfg(any(feature = "embeddings", feature = "chunking"))]
+    #[test]
+    fn test_resolve_preset_unknown_name_returns_unchanged() {
+        let config = ChunkingConfig {
+            max_characters: 500,
+            preset: Some("nonexistent".to_string()),
+            ..Default::default()
+        };
+        let resolved = config.resolve_preset();
+        assert_eq!(resolved.max_characters, 500);
+    }
+
+    #[test]
+    fn test_embedding_model_type_llm_roundtrip() {
+        let model_type = EmbeddingModelType::Llm {
+            llm: crate::core::config::llm::LlmConfig {
+                model: "openai/text-embedding-3-small".to_string(),
+                api_key: None,
+                base_url: None,
+                timeout_secs: None,
+                max_retries: None,
+                temperature: None,
+                max_tokens: None,
+            },
+        };
+        let json = serde_json::to_string(&model_type).unwrap();
+        assert!(json.contains("\"type\":\"llm\""));
+        assert!(json.contains("openai/text-embedding-3-small"));
+
+        let deserialized: EmbeddingModelType = serde_json::from_str(&json).unwrap();
+        match deserialized {
+            EmbeddingModelType::Llm { llm } => {
+                assert_eq!(llm.model, "openai/text-embedding-3-small");
+            }
+            _ => panic!("Expected Llm variant"),
+        }
+    }
+
+    #[test]
+    #[should_panic(expected = "topic_threshold must be in [0.0, 1.0]")]
+    fn test_with_topic_threshold_panics_above_one() {
+        ChunkingConfig::default().with_topic_threshold(1.1);
+    }
+
+    #[test]
+    #[should_panic(expected = "topic_threshold must be in [0.0, 1.0]")]
+    fn test_with_topic_threshold_panics_below_zero() {
+        ChunkingConfig::default().with_topic_threshold(-0.1);
+    }
+
+    #[test]
+    fn test_with_topic_threshold_accepts_boundary_values() {
+        let config = ChunkingConfig::default().with_topic_threshold(0.0);
+        assert_eq!(config.topic_threshold, Some(0.0));
+
+        let config = ChunkingConfig::default().with_topic_threshold(1.0);
+        assert_eq!(config.topic_threshold, Some(1.0));
+    }
+
+    /// Tests Custom model type deserialization.
+    #[test]
+    fn test_embedding_model_type_custom_deserialization() {
+        let json = r#"{"type": "custom", "model_id": "test/model", "dimensions": 512}"#;
+        let model: EmbeddingModelType = serde_json::from_str(json).unwrap();
+
+        match model {
+            EmbeddingModelType::Custom { model_id, dimensions } => {
+                assert_eq!(model_id, "test/model");
+                assert_eq!(dimensions, 512);
+            }
+            _ => panic!("Expected Custom variant"),
+        }
+    }
+
+    #[test]
+    fn test_embedding_model_type_plugin_roundtrip() {
+        let model = EmbeddingModelType::Plugin {
+            name: "lilbee-llamacpp".to_string(),
+        };
+        let json = serde_json::to_string(&model).unwrap();
+        assert!(json.contains("\"type\":\"plugin\""));
+        assert!(json.contains("lilbee-llamacpp"));
+
+        let deserialized: EmbeddingModelType = serde_json::from_str(&json).unwrap();
+        match deserialized {
+            EmbeddingModelType::Plugin { name } => assert_eq!(name, "lilbee-llamacpp"),
+            _ => panic!("Expected Plugin variant"),
+        }
+    }
+
+    #[test]
+    fn test_embedding_model_type_plugin_deserialization() {
+        let json = r#"{"type": "plugin", "name": "my-embedder"}"#;
+        let model: EmbeddingModelType = serde_json::from_str(json).unwrap();
+        match model {
+            EmbeddingModelType::Plugin { name } => assert_eq!(name, "my-embedder"),
+            _ => panic!("Expected Plugin variant"),
+        }
+    }
+
+    // --- Issue #797 regression tests ---
+
+    /// Preset with no explicit embedding: embedding must remain None.
+    ///
+    /// Before the fix, `resolve_preset()` would silently inject an
+    /// `EmbeddingConfig` whenever a preset was configured, causing every
+    /// chunk to have an unexpected `.embedding` field populated.
+    #[test]
+    #[cfg(feature = "embeddings")]
+    fn test_resolve_preset_does_not_inject_embedding_when_none() {
+        let config = ChunkingConfig {
+            preset: Some("multilingual".to_string()),
+            embedding: None,
+            ..Default::default()
+        };
+        let resolved = config.resolve_preset();
+        assert!(
+            resolved.embedding.is_none(),
+            "preset alone must not inject an EmbeddingConfig (#797)"
+        );
+    }
+
+    /// Preset with an explicit embedding: the embedding must be preserved unchanged.
+    #[test]
+    #[cfg(feature = "embeddings")]
+    fn test_resolve_preset_preserves_explicit_embedding_config() {
+        let explicit = EmbeddingConfig {
+            model: EmbeddingModelType::Custom {
+                model_id: "my-org/model".to_string(),
+                dimensions: 768,
+            },
+            batch_size: 16,
+            ..Default::default()
+        };
+        let config = ChunkingConfig {
+            preset: Some("multilingual".to_string()),
+            embedding: Some(explicit),
+            ..Default::default()
+        };
+        let resolved = config.resolve_preset();
+        let emb = resolved
+            .embedding
+            .expect("explicit embedding must survive resolve_preset");
+        assert_eq!(emb.batch_size, 16);
+        match emb.model {
+            EmbeddingModelType::Custom { model_id, dimensions } => {
+                assert_eq!(model_id, "my-org/model");
+                assert_eq!(dimensions, 768);
+            }
+            other => panic!("expected Custom model type, got {other:?}"),
+        }
+    }
+
+    /// No preset, no embedding: embedding must stay None (regression guard).
+    #[cfg(any(feature = "embeddings", feature = "chunking"))]
+    #[test]
+    fn test_resolve_preset_no_preset_no_embedding_stays_none() {
+        let config = ChunkingConfig {
+            preset: None,
+            embedding: None,
+            ..Default::default()
+        };
+        let resolved = config.resolve_preset();
+        assert!(resolved.embedding.is_none(), "no-preset path must not touch embedding");
+    }
+}

crates/kreuzberg/src/core/config/tree_sitter.rs

@@ -0,0 +1,160 @@
+//! Tree-sitter language pack configuration.
+//!
+//! This module contains configuration types for the tree-sitter integration,
+//! including grammar download settings and code analysis processing options.
+
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+
+/// Content rendering mode for code extraction.
+///
+/// Controls how extracted code content is represented in the `content` field
+/// of `ExtractionResult`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum CodeContentMode {
+    /// Use TSLP semantic chunks as content (default).
+    #[default]
+    Chunks,
+    /// Use raw source code as content.
+    Raw,
+    /// Emit function/class headings + docstrings (no code bodies).
+    Structure,
+}
+
+/// Configuration for tree-sitter language pack integration.
+///
+/// Controls grammar download behavior and code analysis options.
+///
+/// # Example (TOML)
+///
+/// ```toml
+/// [tree_sitter]
+/// languages = ["python", "rust"]
+/// groups = ["web"]
+///
+/// [tree_sitter.process]
+/// structure = true
+/// comments = true
+/// docstrings = true
+/// ```
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TreeSitterConfig {
+    /// Enable code intelligence processing (default: true).
+    ///
+    /// When `false`, tree-sitter analysis is completely skipped even if
+    /// the config section is present.
+    #[serde(default = "default_true")]
+    pub enabled: bool,
+
+    /// Custom cache directory for downloaded grammars.
+    ///
+    /// When `None`, uses the default: `~/.cache/tree-sitter-language-pack/v{version}/libs/`.
+    #[serde(default)]
+    pub cache_dir: Option<PathBuf>,
+
+    /// Languages to pre-download on init (e.g., `["python", "rust"]`).
+    #[serde(default)]
+    pub languages: Option<Vec<String>>,
+
+    /// Language groups to pre-download (e.g., `["web", "systems", "scripting"]`).
+    #[serde(default)]
+    pub groups: Option<Vec<String>>,
+
+    /// Processing options for code analysis.
+    #[serde(default)]
+    pub process: TreeSitterProcessConfig,
+}
+
+/// Processing options for tree-sitter code analysis.
+///
+/// Controls which analysis features are enabled when extracting code files.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TreeSitterProcessConfig {
+    /// Extract structural items (functions, classes, structs, etc.). Default: true.
+    #[serde(default = "default_true")]
+    pub structure: bool,
+
+    /// Extract import statements. Default: true.
+    #[serde(default = "default_true")]
+    pub imports: bool,
+
+    /// Extract export statements. Default: true.
+    #[serde(default = "default_true")]
+    pub exports: bool,
+
+    /// Extract comments. Default: false.
+    #[serde(default)]
+    pub comments: bool,
+
+    /// Extract docstrings. Default: false.
+    #[serde(default)]
+    pub docstrings: bool,
+
+    /// Extract symbol definitions. Default: false.
+    #[serde(default)]
+    pub symbols: bool,
+
+    /// Include parse diagnostics. Default: false.
+    #[serde(default)]
+    pub diagnostics: bool,
+
+    /// Maximum chunk size in bytes. `None` disables chunking.
+    #[serde(default)]
+    pub chunk_max_size: Option<usize>,
+
+    /// Content rendering mode for code extraction.
+    #[serde(default)]
+    pub content_mode: CodeContentMode,
+}
+
+impl Default for TreeSitterConfig {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            cache_dir: None,
+            languages: None,
+            groups: None,
+            process: TreeSitterProcessConfig::default(),
+        }
+    }
+}
+
+impl Default for TreeSitterProcessConfig {
+    fn default() -> Self {
+        Self {
+            structure: true,
+            imports: true,
+            exports: true,
+            comments: false,
+            docstrings: false,
+            symbols: false,
+            diagnostics: false,
+            chunk_max_size: None,
+            content_mode: CodeContentMode::default(),
+        }
+    }
+}
+
+fn default_true() -> bool {
+    true
+}
+
+/// Convert kreuzberg's process config to TSLP's `ProcessConfig`.
+///
+/// The language field is left empty — callers must set it before use.
+impl From<&TreeSitterProcessConfig> for tree_sitter_language_pack::ProcessConfig {
+    fn from(p: &TreeSitterProcessConfig) -> Self {
+        Self {
+            language: std::borrow::Cow::Borrowed(""),
+            structure: p.structure,
+            imports: p.imports,
+            exports: p.exports,
+            comments: p.comments,
+            docstrings: p.docstrings,
+            symbols: p.symbols,
+            diagnostics: p.diagnostics,
+            chunk_max_size: p.chunk_max_size,
+        }
+    }
+}

crates/kreuzberg/src/core/config_validation/dependencies.rs

@@ -0,0 +1,97 @@
+//! Cross-section dependency validation.
+//!
+//! This module contains validation functions that check dependencies and relationships
+//! between different configuration sections. These validators ensure that related
+//! configuration values are consistent and compatible with each other.
+
+#[cfg(test)]
+use crate::{KreuzbergError, Result};
+
+#[cfg(test)]
+pub(crate) fn validate_port(port: u32) -> Result<()> {
+    if port == 0 || port > 65535 {
+        Err(KreuzbergError::Validation {
+            message: format!("Port must be 1-65535, got {}", port),
+            source: None,
+        })
+    } else {
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+pub(crate) fn validate_host(host: &str) -> Result<()> {
+    let host = host.trim();
+
+    if host.is_empty() {
+        return Err(KreuzbergError::Validation {
+            message: "Invalid host '': must be a valid IP address or hostname".to_string(),
+            source: None,
+        });
+    }
+
+    // Check if it's a valid IPv4 address
+    if host.parse::<std::net::Ipv4Addr>().is_ok() {
+        return Ok(());
+    }
+
+    // Check if it's a valid IPv6 address
+    if host.parse::<std::net::Ipv6Addr>().is_ok() {
+        return Ok(());
+    }
+
+    // Check if it's a valid hostname (basic validation)
+    // Hostnames must contain only alphanumeric characters, dots, and hyphens
+    // Must not look like an invalid IPv4 address (all numeric with dots)
+    let looks_like_ipv4 = host
+        .split('.')
+        .all(|part| !part.is_empty() && part.chars().all(|c| c.is_numeric()));
+    if !looks_like_ipv4
+        && host.chars().all(|c| c.is_alphanumeric() || c == '.' || c == '-')
+        && !host.starts_with('-')
+        && !host.ends_with('-')
+    {
+        return Ok(());
+    }
+
+    Err(KreuzbergError::Validation {
+        message: format!("Invalid host '{}': must be a valid IP address or hostname", host),
+        source: None,
+    })
+}
+
+#[cfg(test)]
+pub(crate) fn validate_cors_origin(origin: &str) -> Result<()> {
+    let origin = origin.trim();
+
+    if origin == "*" {
+        return Ok(());
+    }
+
+    if origin.starts_with("http://") || origin.starts_with("https://") {
+        // Basic validation: ensure there's something after the protocol
+        if origin.len() > 8 && (origin.starts_with("http://") && origin.len() > 7 || origin.starts_with("https://")) {
+            return Ok(());
+        }
+    }
+
+    Err(KreuzbergError::Validation {
+        message: format!(
+            "Invalid CORS origin '{}': must be a valid HTTP/HTTPS URL or '*'",
+            origin
+        ),
+        source: None,
+    })
+}
+
+#[cfg(test)]
+pub(crate) fn validate_upload_size(size: usize) -> Result<()> {
+    if size > 0 {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!("Upload size must be greater than 0, got {}", size),
+            source: None,
+        })
+    }
+}

crates/kreuzberg/src/core/config_validation/mod.rs

@@ -0,0 +1,399 @@
+//! Configuration validation module.
+//!
+//! Provides centralized validation for configuration values across all bindings.
+//! This eliminates duplication of validation logic in Python, TypeScript, Java, Go, and other language bindings.
+//!
+//! All validation functions return `Result<()>` and produce detailed error messages
+//! suitable for user-facing error handling.
+//!
+//! # Examples
+//!
+//! ```ignore
+//! use kreuzberg::core::config_validation::{
+//!     validate_binarization_method,
+//!     validate_token_reduction_level,
+//!     validate_language_code,
+//! };
+//!
+//! // Valid values
+//! assert!(validate_binarization_method("otsu").is_ok());
+//! assert!(validate_token_reduction_level("moderate").is_ok());
+//! assert!(validate_language_code("en").is_ok());
+//!
+//! // Invalid values
+//! assert!(validate_binarization_method("invalid").is_err());
+//! assert!(validate_token_reduction_level("extreme").is_err());
+//! ```
+
+mod dependencies;
+mod sections;
+
+// Re-export validation functions used in production code
+pub(crate) use sections::{
+    validate_chunking_params, validate_language_code, validate_ocr_backend, validate_token_reduction_level,
+};
+
+// Re-export validation functions used only in tests
+#[cfg(test)]
+pub(crate) use dependencies::{validate_cors_origin, validate_host, validate_port, validate_upload_size};
+#[cfg(test)]
+pub(crate) use sections::{
+    validate_binarization_method, validate_confidence, validate_dpi, validate_output_format, validate_tesseract_oem,
+    validate_tesseract_psm, validate_vlm_backend_config,
+};
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // Tests for section validation functions
+    #[test]
+    fn test_validate_binarization_method_valid() {
+        assert!(validate_binarization_method("otsu").is_ok());
+        assert!(validate_binarization_method("adaptive").is_ok());
+        assert!(validate_binarization_method("sauvola").is_ok());
+    }
+
+    #[test]
+    fn test_validate_binarization_method_case_insensitive() {
+        assert!(validate_binarization_method("OTSU").is_ok());
+        assert!(validate_binarization_method("Adaptive").is_ok());
+        assert!(validate_binarization_method("SAUVOLA").is_ok());
+    }
+
+    #[test]
+    fn test_validate_binarization_method_invalid() {
+        let result = validate_binarization_method("invalid");
+        assert!(result.is_err());
+        let msg = result.unwrap_err().to_string();
+        assert!(msg.contains("Invalid binarization method"));
+        assert!(msg.contains("otsu"));
+    }
+
+    #[test]
+    fn test_validate_token_reduction_level_valid() {
+        assert!(validate_token_reduction_level("off").is_ok());
+        assert!(validate_token_reduction_level("light").is_ok());
+        assert!(validate_token_reduction_level("moderate").is_ok());
+        assert!(validate_token_reduction_level("aggressive").is_ok());
+        assert!(validate_token_reduction_level("maximum").is_ok());
+    }
+
+    #[test]
+    fn test_validate_token_reduction_level_case_insensitive() {
+        assert!(validate_token_reduction_level("OFF").is_ok());
+        assert!(validate_token_reduction_level("Moderate").is_ok());
+        assert!(validate_token_reduction_level("MAXIMUM").is_ok());
+    }
+
+    #[test]
+    fn test_validate_token_reduction_level_invalid() {
+        let result = validate_token_reduction_level("extreme");
+        assert!(result.is_err());
+        let msg = result.unwrap_err().to_string();
+        assert!(msg.contains("Invalid token reduction level"));
+    }
+
+    #[test]
+    fn test_validate_ocr_backend_valid() {
+        assert!(validate_ocr_backend("tesseract").is_ok());
+        assert!(validate_ocr_backend("easyocr").is_ok());
+        assert!(validate_ocr_backend("paddleocr").is_ok());
+    }
+
+    #[test]
+    fn test_validate_ocr_backend_case_insensitive() {
+        assert!(validate_ocr_backend("TESSERACT").is_ok());
+        assert!(validate_ocr_backend("EasyOCR").is_ok());
+        assert!(validate_ocr_backend("PADDLEOCR").is_ok());
+    }
+
+    #[test]
+    fn test_validate_ocr_backend_invalid() {
+        let result = validate_ocr_backend("invalid_backend");
+        assert!(result.is_err());
+        let msg = result.unwrap_err().to_string();
+        assert!(msg.contains("Invalid OCR backend"));
+    }
+
+    #[test]
+    fn test_validate_language_code_valid_iso639_1() {
+        assert!(validate_language_code("en").is_ok());
+        assert!(validate_language_code("de").is_ok());
+        assert!(validate_language_code("fr").is_ok());
+        assert!(validate_language_code("es").is_ok());
+        assert!(validate_language_code("zh").is_ok());
+        assert!(validate_language_code("ja").is_ok());
+        assert!(validate_language_code("ko").is_ok());
+    }
+
+    #[test]
+    fn test_validate_language_code_valid_iso639_3() {
+        assert!(validate_language_code("eng").is_ok());
+        assert!(validate_language_code("deu").is_ok());
+        assert!(validate_language_code("fra").is_ok());
+        assert!(validate_language_code("spa").is_ok());
+        assert!(validate_language_code("zho").is_ok());
+        assert!(validate_language_code("jpn").is_ok());
+        assert!(validate_language_code("kor").is_ok());
+    }
+
+    #[test]
+    fn test_validate_language_code_case_insensitive() {
+        assert!(validate_language_code("EN").is_ok());
+        assert!(validate_language_code("ENG").is_ok());
+        assert!(validate_language_code("De").is_ok());
+        assert!(validate_language_code("DEU").is_ok());
+    }
+
+    #[test]
+    fn test_validate_language_code_all_keyword() {
+        assert!(validate_language_code("all").is_ok());
+        assert!(validate_language_code("ALL").is_ok());
+        assert!(validate_language_code("All").is_ok());
+        assert!(validate_language_code("*").is_ok());
+    }
+
+    #[test]
+    fn test_validate_language_code_invalid() {
+        let result = validate_language_code("invalid");
+        assert!(result.is_err());
+        let msg = result.unwrap_err().to_string();
+        assert!(msg.contains("Invalid language code"));
+        assert!(msg.contains("ISO 639"));
+    }
+
+    #[test]
+    fn test_validate_tesseract_psm_valid() {
+        for psm in 0..=13 {
+            assert!(validate_tesseract_psm(psm).is_ok(), "PSM {} should be valid", psm);
+        }
+    }
+
+    #[test]
+    fn test_validate_tesseract_psm_invalid() {
+        assert!(validate_tesseract_psm(-1).is_err());
+        assert!(validate_tesseract_psm(14).is_err());
+        assert!(validate_tesseract_psm(100).is_err());
+    }
+
+    #[test]
+    fn test_validate_tesseract_oem_valid() {
+        for oem in 0..=3 {
+            assert!(validate_tesseract_oem(oem).is_ok(), "OEM {} should be valid", oem);
+        }
+    }
+
+    #[test]
+    fn test_validate_tesseract_oem_invalid() {
+        assert!(validate_tesseract_oem(-1).is_err());
+        assert!(validate_tesseract_oem(4).is_err());
+        assert!(validate_tesseract_oem(10).is_err());
+    }
+
+    #[test]
+    fn test_validate_output_format_valid() {
+        assert!(validate_output_format("text").is_ok());
+        assert!(validate_output_format("markdown").is_ok());
+    }
+
+    #[test]
+    fn test_validate_output_format_case_insensitive() {
+        assert!(validate_output_format("TEXT").is_ok());
+        assert!(validate_output_format("Markdown").is_ok());
+    }
+
+    #[test]
+    fn test_validate_output_format_invalid() {
+        let result = validate_output_format("xml");
+        assert!(result.is_err());
+        let msg = result.unwrap_err().to_string();
+        assert!(msg.contains("Invalid output format"));
+    }
+
+    #[test]
+    fn test_validate_confidence_valid() {
+        assert!(validate_confidence(0.0).is_ok());
+        assert!(validate_confidence(0.5).is_ok());
+        assert!(validate_confidence(1.0).is_ok());
+        assert!(validate_confidence(0.75).is_ok());
+    }
+
+    #[test]
+    fn test_validate_confidence_invalid() {
+        assert!(validate_confidence(-0.1).is_err());
+        assert!(validate_confidence(1.1).is_err());
+        assert!(validate_confidence(2.0).is_err());
+    }
+
+    #[test]
+    fn test_validate_dpi_valid() {
+        assert!(validate_dpi(72).is_ok());
+        assert!(validate_dpi(96).is_ok());
+        assert!(validate_dpi(300).is_ok());
+        assert!(validate_dpi(600).is_ok());
+        assert!(validate_dpi(1).is_ok());
+    }
+
+    #[test]
+    fn test_validate_dpi_invalid() {
+        assert!(validate_dpi(0).is_err());
+        assert!(validate_dpi(-1).is_err());
+        assert!(validate_dpi(2401).is_err());
+    }
+
+    #[test]
+    fn test_validate_chunking_params_valid() {
+        assert!(validate_chunking_params(1000, 200).is_ok());
+        assert!(validate_chunking_params(500, 50).is_ok());
+        assert!(validate_chunking_params(1, 0).is_ok());
+    }
+
+    #[test]
+    fn test_validate_chunking_params_zero_chars() {
+        let result = validate_chunking_params(0, 100);
+        assert!(result.is_err());
+        assert!(result.unwrap_err().to_string().contains("max_chars"));
+    }
+
+    #[test]
+    fn test_validate_chunking_params_overlap_too_large() {
+        let result = validate_chunking_params(100, 100);
+        assert!(result.is_err());
+        assert!(result.unwrap_err().to_string().contains("overlap"));
+
+        let result = validate_chunking_params(100, 150);
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_error_messages_are_helpful() {
+        let err = validate_binarization_method("bad").unwrap_err().to_string();
+        assert!(err.contains("otsu"));
+        assert!(err.contains("adaptive"));
+        assert!(err.contains("sauvola"));
+
+        let err = validate_token_reduction_level("bad").unwrap_err().to_string();
+        assert!(err.contains("off"));
+        assert!(err.contains("moderate"));
+
+        let err = validate_language_code("bad").unwrap_err().to_string();
+        assert!(err.contains("ISO 639"));
+        assert!(err.contains("en"));
+    }
+
+    // Tests for dependency validation functions
+    #[test]
+    fn test_validate_port_valid() {
+        assert!(validate_port(1).is_ok());
+        assert!(validate_port(80).is_ok());
+        assert!(validate_port(443).is_ok());
+        assert!(validate_port(8000).is_ok());
+        assert!(validate_port(65535).is_ok());
+    }
+
+    #[test]
+    fn test_validate_port_invalid() {
+        let result = validate_port(0);
+        assert!(result.is_err());
+        let msg = result.unwrap_err().to_string();
+        assert!(msg.contains("Port must be 1-65535"));
+        assert!(msg.contains("0"));
+    }
+
+    #[test]
+    fn test_validate_host_ipv4() {
+        assert!(validate_host("127.0.0.1").is_ok());
+        assert!(validate_host("0.0.0.0").is_ok());
+        assert!(validate_host("192.168.1.1").is_ok());
+        assert!(validate_host("10.0.0.1").is_ok());
+        assert!(validate_host("255.255.255.255").is_ok());
+    }
+
+    #[test]
+    fn test_validate_host_ipv6() {
+        assert!(validate_host("::1").is_ok());
+        assert!(validate_host("::").is_ok());
+        assert!(validate_host("2001:db8::1").is_ok());
+        assert!(validate_host("fe80::1").is_ok());
+    }
+
+    #[test]
+    fn test_validate_host_hostname() {
+        assert!(validate_host("localhost").is_ok());
+        assert!(validate_host("example.com").is_ok());
+        assert!(validate_host("sub.example.com").is_ok());
+        assert!(validate_host("api-server").is_ok());
+        assert!(validate_host("app123").is_ok());
+    }
+
+    #[test]
+    fn test_validate_host_invalid() {
+        let result = validate_host("");
+        assert!(result.is_err());
+        let msg = result.unwrap_err().to_string();
+        assert!(msg.contains("Invalid host"));
+
+        let result = validate_host("not a valid host");
+        assert!(result.is_err());
+
+        let result = validate_host("256.256.256.256");
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_validate_cors_origin_https() {
+        assert!(validate_cors_origin("https://example.com").is_ok());
+        assert!(validate_cors_origin("https://localhost:3000").is_ok());
+        assert!(validate_cors_origin("https://sub.example.com").is_ok());
+        assert!(validate_cors_origin("https://192.168.1.1").is_ok());
+        assert!(validate_cors_origin("https://example.com/path").is_ok());
+    }
+
+    #[test]
+    fn test_validate_cors_origin_http() {
+        assert!(validate_cors_origin("http://example.com").is_ok());
+        assert!(validate_cors_origin("http://localhost:3000").is_ok());
+        assert!(validate_cors_origin("http://127.0.0.1:8000").is_ok());
+    }
+
+    #[test]
+    fn test_validate_cors_origin_wildcard() {
+        assert!(validate_cors_origin("*").is_ok());
+    }
+
+    #[test]
+    fn test_validate_cors_origin_invalid() {
+        let result = validate_cors_origin("not-a-url");
+        assert!(result.is_err());
+        let msg = result.unwrap_err().to_string();
+        assert!(msg.contains("Invalid CORS origin"));
+
+        let result = validate_cors_origin("ftp://example.com");
+        assert!(result.is_err());
+
+        let result = validate_cors_origin("example.com");
+        assert!(result.is_err());
+
+        let result = validate_cors_origin("http://");
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_validate_upload_size_valid() {
+        assert!(validate_upload_size(1).is_ok());
+        assert!(validate_upload_size(1024).is_ok());
+        assert!(validate_upload_size(1_000_000).is_ok());
+        assert!(validate_upload_size(1_000_000_000).is_ok());
+        assert!(validate_upload_size(usize::MAX).is_ok());
+    }
+
+    #[test]
+    fn test_validate_upload_size_invalid() {
+        let result = validate_upload_size(0);
+        assert!(result.is_err());
+        let msg = result.unwrap_err().to_string();
+        assert!(msg.contains("Upload size must be greater than 0"));
+        assert!(msg.contains("0"));
+    }
+}

crates/kreuzberg/src/core/config_validation/sections.rs

@@ -0,0 +1,574 @@
+//! Per-section validation functions.
+//!
+//! This module contains validation functions for individual configuration sections
+//! and their specific parameters. Each function validates a specific aspect of
+//! the configuration and returns detailed error messages when validation fails.
+
+use crate::{KreuzbergError, Result};
+
+/// Valid binarization methods for image preprocessing.
+#[cfg(test)]
+const VALID_BINARIZATION_METHODS: &[&str] = &["otsu", "adaptive", "sauvola"];
+
+/// Valid token reduction levels.
+const VALID_TOKEN_REDUCTION_LEVELS: &[&str] = &["off", "light", "moderate", "aggressive", "maximum"];
+
+/// Valid OCR backends.
+const VALID_OCR_BACKENDS: &[&str] = &["tesseract", "easyocr", "paddleocr", "paddle-ocr", "vlm"];
+
+/// Common ISO 639-1 language codes (extended list).
+/// Covers most major languages and variants used in document processing.
+const VALID_LANGUAGE_CODES: &[&str] = &[
+    "en",
+    "de",
+    "fr",
+    "es",
+    "it",
+    "pt",
+    "nl",
+    "pl",
+    "ru",
+    "zh",
+    "ja",
+    "ko",
+    "bg",
+    "cs",
+    "da",
+    "el",
+    "et",
+    "fi",
+    "hu",
+    "lt",
+    "lv",
+    "ro",
+    "sk",
+    "sl",
+    "sv",
+    "uk",
+    "ar",
+    "hi",
+    "th",
+    "tr",
+    "vi",
+    "eng",
+    "deu",
+    "fra",
+    "spa",
+    "ita",
+    "por",
+    "nld",
+    "pol",
+    "rus",
+    "zho",
+    "jpn",
+    "kor",
+    "ces",
+    "dan",
+    "ell",
+    "est",
+    "fin",
+    "hun",
+    "lit",
+    "lav",
+    "ron",
+    "slk",
+    "slv",
+    "swe",
+    "tur",
+    // PaddleOCR-specific language codes (non-ISO but widely used)
+    "ch",
+    "chinese_cht",
+    "latin",
+    "cyrillic",
+    "devanagari",
+    "arabic",
+];
+
+/// Valid tesseract PSM (Page Segmentation Mode) values.
+#[cfg(test)]
+const VALID_TESSERACT_PSM: &[i32] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13];
+
+/// Valid tesseract OEM (OCR Engine Mode) values.
+#[cfg(test)]
+const VALID_TESSERACT_OEM: &[i32] = &[0, 1, 2, 3];
+
+/// Valid output formats for document extraction.
+/// Supports plain text, markdown, djot, HTML, and structured (JSON) output formats.
+/// Also accepts aliases: "text" for "plain", "md" for "markdown", "json" for "structured".
+#[cfg(test)]
+const VALID_OUTPUT_FORMATS: &[&str] = &["plain", "text", "markdown", "md", "djot", "html", "structured", "json"];
+
+/// Validate a binarization method string.
+///
+/// # Arguments
+///
+/// * `method` - The binarization method to validate (e.g., "otsu", "adaptive", "sauvola")
+///
+/// # Returns
+///
+/// `Ok(())` if the method is valid, or a `ValidationError` with details about valid options.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_binarization_method;
+///
+/// assert!(validate_binarization_method("otsu").is_ok());
+/// assert!(validate_binarization_method("adaptive").is_ok());
+/// assert!(validate_binarization_method("invalid").is_err());
+/// ```
+#[cfg(test)]
+pub(crate) fn validate_binarization_method(method: &str) -> Result<()> {
+    let method = method.to_lowercase();
+    if VALID_BINARIZATION_METHODS.contains(&method.as_str()) {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!(
+                "Invalid binarization method '{}'. Valid options are: {}",
+                method,
+                VALID_BINARIZATION_METHODS.join(", ")
+            ),
+            source: None,
+        })
+    }
+}
+
+/// Validate a token reduction level string.
+///
+/// # Arguments
+///
+/// * `level` - The token reduction level to validate (e.g., "off", "light", "moderate")
+///
+/// # Returns
+///
+/// `Ok(())` if the level is valid, or a `ValidationError` with details about valid options.
+///
+/// # Examples
+///
+/// ```ignore
+/// use kreuzberg::core::config_validation::validate_token_reduction_level;
+///
+/// assert!(validate_token_reduction_level("off").is_ok());
+/// assert!(validate_token_reduction_level("moderate").is_ok());
+/// assert!(validate_token_reduction_level("extreme").is_err());
+/// ```
+pub(crate) fn validate_token_reduction_level(level: &str) -> Result<()> {
+    let level = level.to_lowercase();
+    if VALID_TOKEN_REDUCTION_LEVELS.contains(&level.as_str()) {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!(
+                "Invalid token reduction level '{}'. Valid options are: {}",
+                level,
+                VALID_TOKEN_REDUCTION_LEVELS.join(", ")
+            ),
+            source: None,
+        })
+    }
+}
+
+/// Validate an OCR backend string.
+///
+/// # Arguments
+///
+/// * `backend` - The OCR backend to validate (e.g., "tesseract", "easyocr", "paddleocr")
+///
+/// # Returns
+///
+/// `Ok(())` if the backend is valid, or a `ValidationError` with details about valid options.
+///
+/// # Examples
+///
+/// ```ignore
+/// use kreuzberg::core::config_validation::validate_ocr_backend;
+///
+/// assert!(validate_ocr_backend("tesseract").is_ok());
+/// assert!(validate_ocr_backend("easyocr").is_ok());
+/// assert!(validate_ocr_backend("invalid").is_err());
+/// ```
+pub(crate) fn validate_ocr_backend(backend: &str) -> Result<()> {
+    let backend = backend.to_lowercase();
+    if VALID_OCR_BACKENDS.contains(&backend.as_str()) {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!(
+                "Invalid OCR backend '{}'. Valid options are: {}",
+                backend,
+                VALID_OCR_BACKENDS.join(", ")
+            ),
+            source: None,
+        })
+    }
+}
+
+/// Validate a language code (ISO 639-1 or 639-3 format).
+///
+/// Accepts both 2-letter ISO 639-1 codes (e.g., "en", "de") and
+/// 3-letter ISO 639-3 codes (e.g., "eng", "deu") for broader compatibility.
+///
+/// # Arguments
+///
+/// * `code` - The language code to validate
+///
+/// # Returns
+///
+/// `Ok(())` if the code is valid, or a `ValidationError` indicating an invalid language code.
+///
+/// # Examples
+///
+/// ```ignore
+/// use kreuzberg::core::config_validation::validate_language_code;
+///
+/// assert!(validate_language_code("en").is_ok());
+/// assert!(validate_language_code("eng").is_ok());
+/// assert!(validate_language_code("de").is_ok());
+/// assert!(validate_language_code("deu").is_ok());
+/// assert!(validate_language_code("invalid").is_err());
+/// ```
+#[cfg_attr(alef, alef(skip))]
+pub(crate) fn validate_language_code(code: &str) -> Result<()> {
+    let code_lower = code.to_lowercase();
+
+    // Accept "all" and "*" as special values to auto-detect installed languages
+    if code_lower == "all" || code_lower == "*" {
+        return Ok(());
+    }
+
+    if VALID_LANGUAGE_CODES.contains(&code_lower.as_str()) {
+        return Ok(());
+    }
+
+    Err(KreuzbergError::Validation {
+        message: format!(
+            "Invalid language code '{}'. Use ISO 639-1 (2-letter, e.g., 'en', 'de') \
+             or ISO 639-3 (3-letter, e.g., 'eng', 'deu') codes. \
+             Common codes: en, de, fr, es, it, pt, nl, pl, ru, zh, ja, ko, ar, hi, th.",
+            code
+        ),
+        source: None,
+    })
+}
+
+/// Validate a tesseract Page Segmentation Mode (PSM).
+///
+/// # Arguments
+///
+/// * `psm` - The PSM value to validate (0-13)
+///
+/// # Returns
+///
+/// `Ok(())` if the PSM is valid, or a `ValidationError` with details about valid ranges.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_tesseract_psm;
+///
+/// assert!(validate_tesseract_psm(3).is_ok());  // Fully automatic
+/// assert!(validate_tesseract_psm(6).is_ok());  // Single block of text
+/// assert!(validate_tesseract_psm(14).is_err()); // Out of range
+/// ```
+#[cfg(test)]
+pub(crate) fn validate_tesseract_psm(psm: i32) -> Result<()> {
+    if VALID_TESSERACT_PSM.contains(&psm) {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!(
+                "Invalid tesseract PSM value '{}'. Valid range is 0-13. \
+                 Common values: 3 (auto), 6 (single block), 11 (sparse text).",
+                psm
+            ),
+            source: None,
+        })
+    }
+}
+
+/// Validate a tesseract OCR Engine Mode (OEM).
+///
+/// # Arguments
+///
+/// * `oem` - The OEM value to validate (0-3)
+///
+/// # Returns
+///
+/// `Ok(())` if the OEM is valid, or a `ValidationError` with details about valid options.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_tesseract_oem;
+///
+/// assert!(validate_tesseract_oem(1).is_ok());  // Neural nets (LSTM)
+/// assert!(validate_tesseract_oem(2).is_ok());  // Legacy + LSTM
+/// assert!(validate_tesseract_oem(4).is_err()); // Out of range
+/// ```
+#[cfg(test)]
+pub(crate) fn validate_tesseract_oem(oem: i32) -> Result<()> {
+    if VALID_TESSERACT_OEM.contains(&oem) {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!(
+                "Invalid tesseract OEM value '{}'. Valid range is 0-3. \
+                 0=Legacy, 1=LSTM, 2=Legacy+LSTM, 3=Default",
+                oem
+            ),
+            source: None,
+        })
+    }
+}
+
+/// Validate a document extraction output format.
+///
+/// Accepts the following formats and aliases:
+/// - "plain" or "text" for plain text output
+/// - "markdown" or "md" for Markdown output
+/// - "djot" for Djot markup format
+/// - "html" for HTML output
+///
+/// # Arguments
+///
+/// * `format` - The output format to validate
+///
+/// # Returns
+///
+/// `Ok(())` if the format is valid, or a `ValidationError` with details about valid options.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_output_format;
+///
+/// assert!(validate_output_format("text").is_ok());
+/// assert!(validate_output_format("plain").is_ok());
+/// assert!(validate_output_format("markdown").is_ok());
+/// assert!(validate_output_format("md").is_ok());
+/// assert!(validate_output_format("djot").is_ok());
+/// assert!(validate_output_format("html").is_ok());
+/// assert!(validate_output_format("json").is_ok());
+/// ```
+#[cfg(test)]
+pub(crate) fn validate_output_format(format: &str) -> Result<()> {
+    let format = format.to_lowercase();
+    if VALID_OUTPUT_FORMATS.contains(&format.as_str()) {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!(
+                "Invalid output format '{}'. Valid options are: {}",
+                format,
+                VALID_OUTPUT_FORMATS.join(", ")
+            ),
+            source: None,
+        })
+    }
+}
+
+/// Validate a confidence threshold value.
+///
+/// Confidence thresholds should be between 0.0 and 1.0 inclusive.
+///
+/// # Arguments
+///
+/// * `confidence` - The confidence threshold to validate
+///
+/// # Returns
+///
+/// `Ok(())` if the confidence is valid, or a `ValidationError` with details about valid ranges.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_confidence;
+///
+/// assert!(validate_confidence(0.5).is_ok());
+/// assert!(validate_confidence(0.0).is_ok());
+/// assert!(validate_confidence(1.0).is_ok());
+/// assert!(validate_confidence(1.5).is_err());
+/// assert!(validate_confidence(-0.1).is_err());
+/// ```
+#[cfg(test)]
+pub(crate) fn validate_confidence(confidence: f64) -> Result<()> {
+    if (0.0..=1.0).contains(&confidence) {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!(
+                "Invalid confidence threshold '{}'. Must be between 0.0 and 1.0.",
+                confidence
+            ),
+            source: None,
+        })
+    }
+}
+
+/// Validate a DPI (dots per inch) value.
+///
+/// DPI should be a positive integer, typically 72-600.
+///
+/// # Arguments
+///
+/// * `dpi` - The DPI value to validate
+///
+/// # Returns
+///
+/// `Ok(())` if the DPI is valid, or a `ValidationError` with details about valid ranges.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_dpi;
+///
+/// assert!(validate_dpi(96).is_ok());
+/// assert!(validate_dpi(300).is_ok());
+/// assert!(validate_dpi(0).is_err());
+/// assert!(validate_dpi(-1).is_err());
+/// ```
+#[cfg(test)]
+pub(crate) fn validate_dpi(dpi: i32) -> Result<()> {
+    if dpi > 0 && dpi <= 2400 {
+        Ok(())
+    } else {
+        Err(KreuzbergError::Validation {
+            message: format!(
+                "Invalid DPI value '{}'. Must be a positive integer, typically 72-600.",
+                dpi
+            ),
+            source: None,
+        })
+    }
+}
+
+/// Validate chunk size parameters.
+///
+/// Checks that max_chars > 0 and max_overlap < max_chars.
+///
+/// # Arguments
+///
+/// * `max_chars` - The maximum characters per chunk
+/// * `max_overlap` - The maximum overlap between chunks
+///
+/// # Returns
+///
+/// `Ok(())` if the parameters are valid, or a `ValidationError` with details about constraints.
+///
+/// # Examples
+///
+/// ```ignore
+/// use kreuzberg::core::config_validation::validate_chunking_params;
+///
+/// assert!(validate_chunking_params(1000, 200).is_ok());
+/// assert!(validate_chunking_params(500, 50).is_ok());
+/// assert!(validate_chunking_params(0, 100).is_err()); // max_chars must be > 0
+/// assert!(validate_chunking_params(100, 150).is_err()); // overlap >= max_chars
+/// ```
+pub(crate) fn validate_chunking_params(max_chars: usize, max_overlap: usize) -> Result<()> {
+    if max_chars == 0 {
+        return Err(KreuzbergError::Validation {
+            message: "max_chars must be greater than 0".to_string(),
+            source: None,
+        });
+    }
+
+    if max_overlap >= max_chars {
+        return Err(KreuzbergError::Validation {
+            message: format!(
+                "max_overlap ({}) must be less than max_chars ({})",
+                max_overlap, max_chars
+            ),
+            source: None,
+        });
+    }
+
+    Ok(())
+}
+
+/// Validate that an [`LlmConfig`](crate::core::config::LlmConfig) has a non-empty model string.
+///
+/// # Arguments
+///
+/// * `model` - The model string to validate
+///
+/// # Returns
+///
+/// `Ok(())` if the model is non-empty, or a `ValidationError` otherwise.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_llm_config_model;
+///
+/// assert!(validate_llm_config_model("openai/gpt-4o").is_ok());
+/// assert!(validate_llm_config_model("").is_err());
+/// ```
+#[cfg(test)]
+pub(crate) fn validate_llm_config_model(model: &str) -> Result<()> {
+    if model.trim().is_empty() {
+        return Err(KreuzbergError::Validation {
+            message: "LLM config 'model' must not be empty. Provide a model identifier (e.g., 'openai/gpt-4o')."
+                .to_string(),
+            source: None,
+        });
+    }
+    Ok(())
+}
+
+/// Validate that a VLM OCR backend has the required `vlm_config`.
+///
+/// When the OCR backend is set to `"vlm"`, the `vlm_config` field must be present
+/// to provide the model endpoint configuration, and the model string must be non-empty.
+///
+/// # Arguments
+///
+/// * `backend` - The OCR backend name
+/// * `vlm_config` - The optional VLM config to validate
+///
+/// # Returns
+///
+/// `Ok(())` if the backend is not `"vlm"` or `vlm_config` is present with a valid model,
+/// or a `ValidationError` if `"vlm"` backend is used without `vlm_config` or with an empty model.
+///
+/// # Examples
+///
+/// ```rust
+/// use kreuzberg::core::config_validation::validate_vlm_backend_config;
+/// use kreuzberg::core::config::LlmConfig;
+///
+/// assert!(validate_vlm_backend_config("tesseract", None).is_ok());
+/// let config = LlmConfig {
+///     model: "openai/gpt-4o".to_string(),
+///     api_key: None,
+///     base_url: None,
+///     timeout_secs: None,
+///     max_retries: None,
+///     temperature: None,
+///     max_tokens: None,
+/// };
+/// assert!(validate_vlm_backend_config("vlm", Some(&config)).is_ok());
+/// assert!(validate_vlm_backend_config("vlm", None).is_err());
+/// ```
+#[cfg(test)]
+pub(crate) fn validate_vlm_backend_config(
+    backend: &str,
+    vlm_config: Option<&crate::core::config::LlmConfig>,
+) -> Result<()> {
+    if backend.to_lowercase() == "vlm" {
+        match vlm_config {
+            None => {
+                return Err(KreuzbergError::Validation {
+                    message: "OCR backend 'vlm' requires 'vlm_config' to be set with model endpoint configuration."
+                        .to_string(),
+                    source: None,
+                });
+            }
+            Some(config) => {
+                validate_llm_config_model(&config.model)?;
+            }
+        }
+    }
+    Ok(())
+}

crates/kreuzberg/src/core/extractor/batch.rs

@@ -0,0 +1,331 @@
+//! Batch extraction operations for concurrent processing.
+//!
+//! This module provides parallel extraction capabilities for processing
+//! multiple files or byte arrays concurrently with automatic resource management.
+
+#[cfg(feature = "tokio-runtime")]
+use crate::core::config::BatchBytesItem;
+#[cfg(feature = "tokio-runtime")]
+use crate::core::config::BatchFileItem;
+use crate::core::config::ExtractionConfig;
+use crate::core::config::extraction::FileExtractionConfig;
+use crate::types::ExtractionResult;
+use crate::{KreuzbergError, Result};
+use std::future::Future;
+use std::sync::Arc;
+use std::time::Instant;
+
+use super::bytes::extract_bytes;
+use super::file::extract_file;
+use super::helpers::error_extraction_result;
+
+/// Shared batch result collection: spawns tasks via callback, collects ordered results.
+#[cfg(feature = "tokio-runtime")]
+async fn collect_batch<F, Fut>(count: usize, config: &ExtractionConfig, spawn_task: F) -> Result<Vec<ExtractionResult>>
+where
+    F: Fn(usize, Arc<tokio::sync::Semaphore>) -> Fut,
+    Fut: Future<Output = (usize, Result<ExtractionResult>, u64)> + Send + 'static,
+{
+    use tokio::sync::Semaphore;
+    use tokio::task::JoinSet;
+
+    if count == 0 {
+        return Ok(vec![]);
+    }
+
+    let max_concurrent = config
+        .max_concurrent_extractions
+        .or_else(|| config.concurrency.as_ref().and_then(|c| c.max_threads))
+        .unwrap_or_else(|| crate::core::config::concurrency::resolve_thread_budget(config.concurrency.as_ref()));
+    let semaphore = Arc::new(Semaphore::new(max_concurrent));
+
+    let mut tasks = JoinSet::new();
+
+    for index in 0..count {
+        let sem = Arc::clone(&semaphore);
+        tasks.spawn(spawn_task(index, sem));
+    }
+
+    let mut results: Vec<Option<ExtractionResult>> = vec![None; count];
+
+    while let Some(task_result) = tasks.join_next().await {
+        match task_result {
+            Ok((index, Ok(result), _elapsed_ms)) => {
+                results[index] = Some(result);
+            }
+            Ok((index, Err(e), elapsed_ms)) => {
+                results[index] = Some(error_extraction_result(&e, Some(elapsed_ms)));
+            }
+            Err(join_err) => {
+                return Err(KreuzbergError::Other(format!("Task panicked: {}", join_err)));
+            }
+        }
+    }
+
+    #[allow(clippy::unwrap_used)]
+    Ok(results.into_iter().map(|r| r.unwrap()).collect())
+}
+
+/// Run a single extraction task with semaphore gating, timing, optional timeout, and batch mode.
+///
+/// When `cancel_token` is provided and the timeout fires, the token is signalled so that
+/// any blocking PDF operations in progress can observe the cancellation at the next
+/// inter-page checkpoint and stop early.
+#[cfg(feature = "tokio-runtime")]
+async fn run_timed_extraction<F, Fut>(
+    index: usize,
+    semaphore: Arc<tokio::sync::Semaphore>,
+    timeout_secs: Option<u64>,
+    cancel_token: Option<crate::cancellation::CancellationToken>,
+    extract_fn: F,
+) -> (usize, Result<ExtractionResult>, u64)
+where
+    F: FnOnce() -> Fut,
+    Fut: Future<Output = Result<ExtractionResult>>,
+{
+    let _permit = semaphore.acquire().await.unwrap();
+    let start = Instant::now();
+
+    let extraction_future = crate::core::batch_mode::with_batch_mode(extract_fn());
+
+    let mut result = match timeout_secs {
+        Some(secs) => match tokio::time::timeout(std::time::Duration::from_secs(secs), extraction_future).await {
+            Ok(inner) => inner,
+            Err(_elapsed) => {
+                // Signal the cancellation token so that any blocking PDF thread can
+                // detect it at the next inter-page checkpoint and stop processing.
+                if let Some(ref token) = cancel_token {
+                    token.cancel();
+                }
+                let elapsed_ms = start.elapsed().as_millis() as u64;
+                Err(KreuzbergError::Timeout {
+                    elapsed_ms,
+                    limit_ms: secs * 1000,
+                })
+            }
+        },
+        None => extraction_future.await,
+    };
+
+    let elapsed_ms = start.elapsed().as_millis() as u64;
+
+    if let Ok(ref mut r) = result {
+        r.metadata.extraction_duration_ms = Some(elapsed_ms);
+    }
+
+    (index, result, elapsed_ms)
+}
+
+/// Resolve a per-file config against a base config. Returns owned config.
+fn resolve_config(base: &ExtractionConfig, file_config: &Option<FileExtractionConfig>) -> ExtractionConfig {
+    match file_config {
+        Some(fc) => base.with_file_overrides(fc),
+        None => base.clone(),
+    }
+}
+
+/// Extract content from multiple files concurrently.
+///
+/// This function processes multiple files in parallel, automatically managing
+/// concurrency to prevent resource exhaustion. The concurrency limit can be
+/// configured via `ExtractionConfig::max_concurrent_extractions` or defaults
+/// to `(num_cpus * 1.5).ceil()`.
+///
+/// Each file can optionally specify a [`FileExtractionConfig`] that overrides specific
+/// fields from the batch-level `config`. Pass `None` for a file to use the batch defaults.
+/// Batch-level settings like `max_concurrent_extractions` and `use_cache` are always
+/// taken from the batch-level `config`.
+///
+/// # Arguments
+///
+/// * `items` - Vector of `BatchFileItem` structs, each containing a path and optional
+///   per-file configuration overrides.
+/// * `config` - Batch-level extraction configuration (provides defaults and batch settings)
+///
+/// # Returns
+///
+/// A vector of `ExtractionResult` in the same order as the input items.
+///
+/// # Errors
+///
+/// Individual file errors are captured in the result metadata. System errors
+/// (IO, RuntimeError equivalents) will bubble up and fail the entire batch.
+///
+/// # Examples
+///
+/// Simple usage with no per-file overrides:
+///
+/// ```rust,no_run
+/// use kreuzberg::core::extractor::batch_extract_files;
+/// use kreuzberg::core::config::{ExtractionConfig, BatchFileItem};
+/// use std::path::PathBuf;
+///
+/// # async fn example() -> kreuzberg::Result<()> {
+/// let config = ExtractionConfig::default();
+/// let items = vec![
+///     BatchFileItem { path: "doc1.pdf".into(), config: None },
+///     BatchFileItem { path: "doc2.pdf".into(), config: None },
+/// ];
+/// let results = batch_extract_files(items, &config).await?;
+/// println!("Processed {} files", results.len());
+/// # Ok(())
+/// # }
+/// ```
+///
+/// Per-file configuration overrides:
+///
+/// ```rust,no_run
+/// use kreuzberg::core::extractor::batch_extract_files;
+/// use kreuzberg::core::config::{ExtractionConfig, BatchFileItem, FileExtractionConfig};
+/// use std::path::PathBuf;
+///
+/// # async fn example() -> kreuzberg::Result<()> {
+/// let config = ExtractionConfig::default();
+/// let items = vec![
+///     BatchFileItem {
+///         path: "scan.pdf".into(),
+///         config: Some(FileExtractionConfig { force_ocr: Some(true), ..Default::default() }),
+///     },
+///     BatchFileItem { path: "notes.txt".into(), config: None },
+/// ];
+/// let results = batch_extract_files(items, &config).await?;
+/// # Ok(())
+/// # }
+/// ```
+#[cfg(feature = "tokio-runtime")]
+#[cfg_attr(feature = "otel", tracing::instrument(
+    skip(config, items),
+    fields(
+        extraction.batch_size = items.len(),
+    )
+))]
+pub async fn batch_extract_files(
+    items: Vec<BatchFileItem>,
+    config: &ExtractionConfig,
+) -> Result<Vec<ExtractionResult>> {
+    let config_arc = Arc::new(config.clone());
+    // Use Arc<Vec> for file items — paths are small, so keeping them all alive is fine.
+    let items_arc = Arc::new(items);
+    let count = items_arc.len();
+
+    collect_batch(count, config, |index, sem| {
+        let cfg = Arc::clone(&config_arc);
+        let items = Arc::clone(&items_arc);
+        async move {
+            let item = &items[index];
+            let resolved = resolve_config(&cfg, &item.config);
+            let timeout = resolved.extraction_timeout_secs;
+            let cancel_token = resolved.cancel_token.clone();
+            run_timed_extraction(index, sem, timeout, cancel_token, || {
+                let path = item.path.clone();
+                async move { extract_file(&path, None, &resolved).await }
+            })
+            .await
+        }
+    })
+    .await
+}
+
+/// Extract content from multiple byte arrays concurrently.
+///
+/// This function processes multiple byte arrays in parallel, automatically managing
+/// concurrency to prevent resource exhaustion. The concurrency limit can be
+/// configured via `ExtractionConfig::max_concurrent_extractions` or defaults
+/// to `(num_cpus * 1.5).ceil()`.
+///
+/// Each item can optionally specify a [`FileExtractionConfig`] that overrides specific
+/// fields from the batch-level `config`. Pass `None` as the config to use
+/// the batch-level defaults for that item.
+///
+/// # Arguments
+///
+/// * `items` - Vector of `BatchBytesItem` structs, each containing content bytes,
+///   MIME type, and optional per-item configuration overrides.
+/// * `config` - Batch-level extraction configuration
+///
+/// # Returns
+///
+/// A vector of `ExtractionResult` in the same order as the input items.
+///
+/// # Examples
+///
+/// Simple usage with no per-item overrides:
+///
+/// ```rust,no_run
+/// use kreuzberg::core::extractor::batch_extract_bytes;
+/// use kreuzberg::core::config::{ExtractionConfig, BatchBytesItem};
+///
+/// # async fn example() -> kreuzberg::Result<()> {
+/// let config = ExtractionConfig::default();
+/// let items = vec![
+///     BatchBytesItem { content: b"content 1".to_vec(), mime_type: "text/plain".to_string(), config: None },
+///     BatchBytesItem { content: b"content 2".to_vec(), mime_type: "text/plain".to_string(), config: None },
+/// ];
+/// let results = batch_extract_bytes(items, &config).await?;
+/// println!("Processed {} items", results.len());
+/// # Ok(())
+/// # }
+/// ```
+///
+/// Per-item configuration overrides:
+///
+/// ```rust,no_run
+/// use kreuzberg::core::extractor::batch_extract_bytes;
+/// use kreuzberg::core::config::{ExtractionConfig, BatchBytesItem, FileExtractionConfig};
+///
+/// # async fn example() -> kreuzberg::Result<()> {
+/// let config = ExtractionConfig::default();
+/// let items = vec![
+///     BatchBytesItem { content: b"content".to_vec(), mime_type: "text/plain".to_string(), config: None },
+///     BatchBytesItem {
+///         content: b"<html>test</html>".to_vec(),
+///         mime_type: "text/html".to_string(),
+///         config: Some(FileExtractionConfig { force_ocr: Some(true), ..Default::default() }),
+///     },
+/// ];
+/// let results = batch_extract_bytes(items, &config).await?;
+/// # Ok(())
+/// # }
+/// ```
+#[cfg(feature = "tokio-runtime")]
+#[cfg_attr(feature = "otel", tracing::instrument(
+    skip(config, items),
+    fields(
+        extraction.batch_size = items.len(),
+    )
+))]
+pub async fn batch_extract_bytes(
+    items: Vec<BatchBytesItem>,
+    config: &ExtractionConfig,
+) -> Result<Vec<ExtractionResult>> {
+    let config_arc = Arc::new(config.clone());
+    let count = items.len();
+
+    // Move items into individually-indexed slots so each task can take ownership
+    // of its bytes without cloning. This avoids the memory regression of
+    // Arc<Vec<BatchBytesItem>> which would keep all byte arrays alive for the
+    // entire batch duration.
+    type BytesSlot = parking_lot::Mutex<Option<BatchBytesItem>>;
+    let slots: Arc<Vec<BytesSlot>> = Arc::new(
+        items
+            .into_iter()
+            .map(|item| parking_lot::Mutex::new(Some(item)))
+            .collect(),
+    );
+
+    collect_batch(count, config, |index, sem| {
+        let cfg = Arc::clone(&config_arc);
+        let slots = Arc::clone(&slots);
+        async move {
+            let item = slots[index].lock().take().expect("batch item already consumed");
+            let resolved = resolve_config(&cfg, &item.config);
+            let timeout = resolved.extraction_timeout_secs;
+            let cancel_token = resolved.cancel_token.clone();
+            run_timed_extraction(index, sem, timeout, cancel_token, || async move {
+                extract_bytes(&item.content, &item.mime_type, &resolved).await
+            })
+            .await
+        }
+    })
+    .await
+}

crates/kreuzberg/src/core/extractor/bytes.rs

@@ -0,0 +1,170 @@
+//! Byte array extraction operations.
+//!
+//! This module handles extraction from in-memory byte arrays, including:
+//! - MIME type validation
+//! - Legacy format conversion (DOC, PPT)
+//! - Extraction pipeline orchestration
+
+#[cfg(not(feature = "office"))]
+use crate::KreuzbergError;
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::core::mime::{LEGACY_POWERPOINT_MIME_TYPE, LEGACY_WORD_MIME_TYPE};
+use crate::types::ExtractionResult;
+
+use super::file::extract_bytes_with_extractor;
+
+/// Extract content from a byte array.
+///
+/// This is the main entry point for in-memory extraction. It performs the following steps:
+/// 1. Validate MIME type
+/// 2. Handle legacy format conversion if needed
+/// 3. Select appropriate extractor from registry
+/// 4. Extract content
+/// 5. Run post-processing pipeline
+///
+/// # Arguments
+///
+/// * `content` - The byte array to extract
+/// * `mime_type` - MIME type of the content
+/// * `config` - Extraction configuration
+///
+/// # Returns
+///
+/// An `ExtractionResult` containing the extracted content and metadata.
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Validation` if MIME type is invalid.
+/// Returns `KreuzbergError::UnsupportedFormat` if MIME type is not supported.
+///
+/// # Example
+///
+/// ```rust,no_run
+/// use kreuzberg::core::extractor::extract_bytes;
+/// use kreuzberg::core::config::ExtractionConfig;
+///
+/// # async fn example() -> kreuzberg::Result<()> {
+/// let config = ExtractionConfig::default();
+/// let bytes = b"Hello, world!";
+/// let result = extract_bytes(bytes, "text/plain", &config).await?;
+/// println!("Content: {}", result.content);
+/// # Ok(())
+/// # }
+/// ```
+#[cfg_attr(feature = "otel", tracing::instrument(
+    skip(config, content),
+    fields(
+        { crate::telemetry::conventions::OPERATION } = crate::telemetry::conventions::operations::EXTRACT_BYTES,
+        { crate::telemetry::conventions::DOCUMENT_MIME_TYPE } = mime_type,
+        { crate::telemetry::conventions::DOCUMENT_SIZE_BYTES } = content.len(),
+        { crate::telemetry::conventions::OTEL_STATUS_CODE } = tracing::field::Empty,
+        { crate::telemetry::conventions::ERROR_TYPE } = tracing::field::Empty,
+        { crate::telemetry::conventions::ERROR_MESSAGE } = tracing::field::Empty,
+    )
+))]
+pub async fn extract_bytes(content: &[u8], mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
+    use crate::core::mime;
+
+    let extraction_future = async {
+        if config.force_ocr && config.effective_disable_ocr() {
+            return Err(crate::KreuzbergError::Validation {
+                message: "force_ocr and disable_ocr cannot both be true".to_string(),
+                source: None,
+            });
+        }
+
+        let validated_mime = if mime_type == "application/octet-stream" {
+            // When tree-sitter is configured, check if content is recognized source code.
+            // This allows octet-stream files with tree-sitter config to be detected as code.
+            #[cfg(feature = "tree-sitter")]
+            {
+                if config.tree_sitter.is_some() {
+                    if let Ok(text) = std::str::from_utf8(content) {
+                        let trimmed = text.trim_start();
+                        if tree_sitter_language_pack::detect_language_from_content(trimmed).is_some() {
+                            // Recognize as source code when tree-sitter can detect a language.
+                            mime::SOURCE_CODE_MIME_TYPE.to_string()
+                        } else {
+                            mime::detect_mime_type_from_bytes(content)?
+                        }
+                    } else {
+                        mime::detect_mime_type_from_bytes(content)?
+                    }
+                } else {
+                    mime::detect_mime_type_from_bytes(content)?
+                }
+            }
+            #[cfg(not(feature = "tree-sitter"))]
+            {
+                let _ = config;
+                mime::detect_mime_type_from_bytes(content)?
+            }
+        } else {
+            mime::validate_mime_type(mime_type)?
+        };
+
+        // Native DOC/PPT extractors are registered in the plugin registry.
+        // When the office feature is disabled, these MIME types are unsupported.
+        #[cfg(not(feature = "office"))]
+        match validated_mime.as_str() {
+            LEGACY_WORD_MIME_TYPE => {
+                return Err(KreuzbergError::UnsupportedFormat(
+                    "Legacy Word extraction requires the `office` feature".to_string(),
+                ));
+            }
+            LEGACY_POWERPOINT_MIME_TYPE => {
+                return Err(KreuzbergError::UnsupportedFormat(
+                    "Legacy PowerPoint extraction requires the `office` feature".to_string(),
+                ));
+            }
+            _ => {}
+        }
+
+        // Suppress unused import warnings when office feature is enabled
+        #[cfg(feature = "office")]
+        {
+            let _ = LEGACY_WORD_MIME_TYPE;
+            let _ = LEGACY_POWERPOINT_MIME_TYPE;
+        }
+
+        extract_bytes_with_extractor(content, &validated_mime, config).await
+    };
+
+    #[cfg(feature = "tokio-runtime")]
+    let result = if let Some(secs) = config.extraction_timeout_secs {
+        let start = std::time::Instant::now();
+        match tokio::time::timeout(std::time::Duration::from_secs(secs), extraction_future).await {
+            Ok(inner) => inner,
+            Err(_elapsed) => {
+                if let Some(ref token) = config.cancel_token {
+                    token.cancel();
+                }
+                Err(crate::KreuzbergError::Timeout {
+                    elapsed_ms: start.elapsed().as_millis() as u64,
+                    limit_ms: secs * 1000,
+                })
+            }
+        }
+    } else {
+        extraction_future.await
+    };
+
+    #[cfg(not(feature = "tokio-runtime"))]
+    let result = {
+        if config.extraction_timeout_secs.is_some() {
+            return Err(crate::KreuzbergError::Validation {
+                message: "extraction_timeout_secs requires the 'tokio-runtime' feature to be enabled".to_string(),
+                source: None,
+            });
+        }
+        extraction_future.await
+    };
+
+    #[cfg(feature = "otel")]
+    if let Err(ref e) = result {
+        crate::telemetry::spans::record_error_on_current_span(e);
+    }
+
+    result
+}

crates/kreuzberg/src/core/extractor/file.rs

@@ -0,0 +1,279 @@
+//! File-based extraction operations.
+//!
+//! This module handles extraction from filesystem paths, including:
+//! - MIME type detection and validation
+//! - Legacy format conversion (DOC, PPT)
+//! - File validation and reading
+//! - Extraction pipeline orchestration
+
+#[cfg(not(feature = "office"))]
+use crate::KreuzbergError;
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::core::mime::{LEGACY_POWERPOINT_MIME_TYPE, LEGACY_WORD_MIME_TYPE};
+use crate::types::ExtractionResult;
+use std::path::Path;
+
+use super::helpers::get_extractor;
+
+/// Extract content from a file.
+///
+/// This is the main entry point for file-based extraction. It performs the following steps:
+/// 1. Check cache for existing result (if caching enabled)
+/// 2. Detect or validate MIME type
+/// 3. Select appropriate extractor from registry
+/// 4. Extract content
+/// 5. Run post-processing pipeline
+/// 6. Store result in cache (if caching enabled)
+///
+/// # Arguments
+///
+/// * `path` - Path to the file to extract
+/// * `mime_type` - Optional MIME type override. If None, will be auto-detected
+/// * `config` - Extraction configuration
+///
+/// # Returns
+///
+/// An `ExtractionResult` containing the extracted content and metadata.
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Io` if the file doesn't exist (NotFound) or for other file I/O errors.
+/// Returns `KreuzbergError::UnsupportedFormat` if MIME type is not supported.
+///
+/// # Example
+///
+/// ```rust,no_run
+/// use kreuzberg::core::extractor::extract_file;
+/// use kreuzberg::core::config::ExtractionConfig;
+///
+/// # async fn example() -> kreuzberg::Result<()> {
+/// let config = ExtractionConfig::default();
+/// let result = extract_file("document.pdf", None, &config).await?;
+/// println!("Content: {}", result.content);
+/// # Ok(())
+/// # }
+/// ```
+#[cfg_attr(feature = "otel", tracing::instrument(
+    skip(config, path),
+    fields(
+        { crate::telemetry::conventions::OPERATION } = crate::telemetry::conventions::operations::EXTRACT_FILE,
+        { crate::telemetry::conventions::DOCUMENT_FILENAME } = tracing::field::Empty,
+        { crate::telemetry::conventions::OTEL_STATUS_CODE } = tracing::field::Empty,
+        { crate::telemetry::conventions::ERROR_TYPE } = tracing::field::Empty,
+        { crate::telemetry::conventions::ERROR_MESSAGE } = tracing::field::Empty,
+    )
+))]
+pub async fn extract_file(
+    path: impl AsRef<Path>,
+    mime_type: Option<&str>,
+    config: &ExtractionConfig,
+) -> Result<ExtractionResult> {
+    use crate::core::{io, mime};
+
+    let path = path.as_ref();
+
+    #[cfg(feature = "otel")]
+    {
+        let span = tracing::Span::current();
+        span.record(
+            crate::telemetry::conventions::DOCUMENT_FILENAME,
+            crate::telemetry::spans::sanitize_path(path),
+        );
+    }
+
+    let extraction_future = async {
+        io::validate_file_exists(path)?;
+
+        if config.force_ocr && config.effective_disable_ocr() {
+            return Err(crate::KreuzbergError::Validation {
+                message: "force_ocr and disable_ocr cannot both be true".to_string(),
+                source: None,
+            });
+        }
+
+        let detected_mime = mime::detect_or_validate(path.to_str(), mime_type)?;
+
+        // Native DOC/PPT extractors are registered in the plugin registry.
+        // When the office feature is disabled, these MIME types are unsupported.
+        #[cfg(not(feature = "office"))]
+        match detected_mime.as_str() {
+            LEGACY_WORD_MIME_TYPE => {
+                return Err(KreuzbergError::UnsupportedFormat(
+                    "Legacy Word extraction requires the `office` feature".to_string(),
+                ));
+            }
+            LEGACY_POWERPOINT_MIME_TYPE => {
+                return Err(KreuzbergError::UnsupportedFormat(
+                    "Legacy PowerPoint extraction requires the `office` feature".to_string(),
+                ));
+            }
+            _ => {}
+        }
+
+        // Suppress unused import warnings when office feature is enabled
+        #[cfg(feature = "office")]
+        {
+            let _ = LEGACY_WORD_MIME_TYPE;
+            let _ = LEGACY_POWERPOINT_MIME_TYPE;
+        }
+
+        extract_file_with_extractor(path, &detected_mime, config).await
+    };
+
+    #[cfg(feature = "tokio-runtime")]
+    let result = if let Some(secs) = config.extraction_timeout_secs {
+        let start = std::time::Instant::now();
+        match tokio::time::timeout(std::time::Duration::from_secs(secs), extraction_future).await {
+            Ok(inner) => inner,
+            Err(_elapsed) => {
+                if let Some(ref token) = config.cancel_token {
+                    token.cancel();
+                }
+                Err(crate::KreuzbergError::Timeout {
+                    elapsed_ms: start.elapsed().as_millis() as u64,
+                    limit_ms: secs * 1000,
+                })
+            }
+        }
+    } else {
+        extraction_future.await
+    };
+
+    #[cfg(not(feature = "tokio-runtime"))]
+    let result = {
+        if config.extraction_timeout_secs.is_some() {
+            return Err(crate::KreuzbergError::Validation {
+                message: "extraction_timeout_secs requires the 'tokio-runtime' feature to be enabled".to_string(),
+                source: None,
+            });
+        }
+        extraction_future.await
+    };
+
+    #[cfg(feature = "otel")]
+    if let Err(ref e) = result {
+        crate::telemetry::spans::record_error_on_current_span(e);
+    }
+
+    result
+}
+
+pub(in crate::core::extractor) async fn extract_file_with_extractor(
+    path: &Path,
+    mime_type: &str,
+    config: &ExtractionConfig,
+) -> Result<ExtractionResult> {
+    // Normalize config so cache keys are consistent for ElementBased requests
+    // regardless of whether the caller explicitly set extract_pages.
+    let config = config.normalized();
+    let config = config.as_ref();
+
+    // Skip cache if disabled or TTL=0
+    if !config.use_cache || config.cache_ttl_secs == Some(0) {
+        return extract_file_uncached(path, mime_type, config).await;
+    }
+
+    // Generate cache key from file content hash + config fingerprint
+    let content_hash = crate::cache::blake3_hash_file(path)?;
+    let config_hash = hash_extraction_config(config, mime_type);
+    let cache_key = format!("{content_hash}_{config_hash}");
+
+    let namespace = config.cache_namespace.as_deref();
+
+    // Try cache read
+    if let Some(cache) = get_extraction_cache()
+        && let Ok(Some(data)) = cache.get(&cache_key, path.to_str(), namespace, config.cache_ttl_secs)
+        && let Ok(result) = rmp_serde::from_slice::<ExtractionResult>(&data)
+    {
+        tracing::debug!(cache_key = %cache_key, "Extraction cache hit");
+        return Ok(result);
+    }
+
+    // Cache miss — extract
+    let result = extract_file_uncached(path, mime_type, config).await?;
+
+    // Cache write (best-effort)
+    if let Some(cache) = get_extraction_cache()
+        && let Ok(data) = rmp_serde::to_vec(&result)
+    {
+        let _ = cache.set(&cache_key, data, path.to_str(), namespace, config.cache_ttl_secs);
+    }
+
+    Ok(result)
+}
+
+/// Extract without caching logic.
+async fn extract_file_uncached(path: &Path, mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
+    let budget = crate::core::config::concurrency::resolve_thread_budget(config.concurrency.as_ref());
+    crate::core::config::concurrency::init_thread_pools(budget);
+
+    crate::extractors::ensure_initialized()?;
+
+    let extractor = get_extractor(mime_type)?;
+    let doc = extractor.extract_file(path, mime_type, config).await?;
+    let result = crate::core::pipeline::run_pipeline(doc, config).await?;
+    Ok(result)
+}
+
+/// Hash ExtractionConfig fields that affect extraction output.
+///
+/// Excludes cache-control fields (use_cache, cache_namespace, cache_ttl_secs)
+/// since they don't affect the extraction result. Uses a clone-and-normalize
+/// approach to ensure determinism: cache fields are zeroed, then the struct
+/// is serialized to canonical JSON via serde_json's sorted-keys representation.
+fn hash_extraction_config(config: &ExtractionConfig, mime_type: &str) -> String {
+    let mut normalized = config.clone();
+    // Zero out cache-control fields so they don't affect the hash
+    normalized.use_cache = true;
+    normalized.cache_namespace = None;
+    normalized.cache_ttl_secs = None;
+
+    let mut hasher = blake3::Hasher::new();
+    hasher.update(mime_type.as_bytes());
+    // Use MessagePack for deterministic serialization (no float formatting issues,
+    // no HashMap key ordering issues — serde serializes struct fields in declaration order).
+    if let Ok(bytes) = rmp_serde::to_vec(&normalized) {
+        hasher.update(&bytes);
+    }
+    let hash = hasher.finalize();
+    hex::encode(&hash.as_bytes()[..16])
+}
+
+/// Get or initialize the global extraction cache.
+fn get_extraction_cache() -> Option<&'static crate::cache::GenericCache> {
+    use std::sync::OnceLock;
+    static CACHE: OnceLock<Option<crate::cache::GenericCache>> = OnceLock::new();
+
+    CACHE
+        .get_or_init(|| {
+            crate::cache::GenericCache::new(
+                "extraction".to_string(),
+                None,
+                30.0,   // 30-day default TTL
+                2000.0, // 2 GB max cache size
+                500.0,  // 500 MB min free space
+            )
+            .ok()
+        })
+        .as_ref()
+}
+
+pub(in crate::core::extractor) async fn extract_bytes_with_extractor(
+    content: &[u8],
+    mime_type: &str,
+    config: &ExtractionConfig,
+) -> Result<ExtractionResult> {
+    let config = config.normalized();
+    let config = config.as_ref();
+
+    let budget = crate::core::config::concurrency::resolve_thread_budget(config.concurrency.as_ref());
+    crate::core::config::concurrency::init_thread_pools(budget);
+
+    crate::extractors::ensure_initialized()?;
+
+    let extractor = get_extractor(mime_type)?;
+    let doc = extractor.extract_bytes(content, mime_type, config).await?;
+    let result = crate::core::pipeline::run_pipeline(doc, config).await?;
+    Ok(result)
+}

crates/kreuzberg/src/core/extractor/helpers.rs

@@ -0,0 +1,85 @@
+//! Helper functions and utilities for extraction operations.
+//!
+//! This module provides shared utilities used across extraction modules.
+
+use crate::plugins::DocumentExtractor;
+use crate::types::{ErrorMetadata, ExtractionResult, Metadata};
+use crate::{KreuzbergError, Result};
+use std::borrow::Cow;
+use std::sync::Arc;
+
+/// Get an extractor from the registry.
+///
+/// This function acquires the registry read lock and retrieves the appropriate
+/// extractor for the given MIME type.
+///
+/// When the `otel` feature is enabled, the returned extractor is wrapped in an
+/// [`InstrumentedExtractor`](crate::plugins::extractor::instrumented::InstrumentedExtractor)
+/// that adds tracing spans and metrics automatically.
+///
+/// # Performance
+///
+/// RwLock read + HashMap lookup is ~100ns, fast enough without caching.
+/// Removed thread-local cache to avoid Tokio work-stealing scheduler issues.
+pub(in crate::core::extractor) fn get_extractor(mime_type: &str) -> Result<Arc<dyn DocumentExtractor>> {
+    let registry = crate::plugins::registry::get_document_extractor_registry();
+    let registry_read = registry.read();
+    let extractor = registry_read.get(mime_type)?;
+
+    #[cfg(feature = "otel")]
+    {
+        Ok(Arc::new(
+            crate::plugins::extractor::instrumented::InstrumentedExtractor::new(extractor),
+        ))
+    }
+
+    #[cfg(not(feature = "otel"))]
+    {
+        Ok(extractor)
+    }
+}
+
+/// Get optimal pool sizing hint for a document.
+///
+/// This function calculates recommended pool sizes based on the document's
+/// file size and MIME type. The hint can be used to create appropriately
+/// sized thread pools for extraction, reducing memory waste from over-allocation.
+///
+/// # Arguments
+///
+/// * `file_size` - The size of the file in bytes
+/// * `mime_type` - The MIME type of the document
+///
+/// # Returns
+///
+/// A `PoolSizeHint` with recommended pool configurations
+///
+/// # Example
+///
+/// ```rust,ignore
+/// use kreuzberg::core::extractor::get_pool_sizing_hint;
+///
+/// let hint = get_pool_sizing_hint(5_000_000, "application/pdf");
+/// println!("Recommended string buffers: {}", hint.string_buffer_count);
+/// ```
+/// Build an error `ExtractionResult` for failed batch items.
+///
+/// Used by both tokio-based batch functions and WASM synchronous fallbacks
+/// to construct a uniform error result.
+pub(crate) fn error_extraction_result(e: &KreuzbergError, elapsed_ms: Option<u64>) -> ExtractionResult {
+    let metadata = Metadata {
+        error: Some(ErrorMetadata {
+            error_type: format!("{:?}", e),
+            message: e.to_string(),
+        }),
+        extraction_duration_ms: elapsed_ms,
+        ..Default::default()
+    };
+
+    ExtractionResult {
+        content: format!("Error: {}", e),
+        mime_type: Cow::Borrowed("text/plain"),
+        metadata,
+        ..Default::default()
+    }
+}

crates/kreuzberg/src/core/extractor/legacy.rs

@@ -0,0 +1,67 @@
+//! Legacy synchronous extraction for WASM compatibility.
+//!
+//! This module provides truly synchronous extraction implementations
+//! for environments where Tokio runtime is not available (e.g., WASM).
+
+/// Synchronous extraction implementation for WASM compatibility.
+///
+/// This function performs extraction without requiring a tokio runtime.
+/// It calls the sync extractor methods directly.
+///
+/// # Arguments
+///
+/// * `content` - The byte content to extract
+/// * `mime_type` - Optional MIME type to validate/use
+/// * `config` - Optional extraction configuration
+///
+/// # Returns
+///
+/// An `ExtractionResult` or a `KreuzbergError`
+///
+/// # Implementation Notes
+///
+/// This is called when the `tokio-runtime` feature is disabled.
+/// It replicates the logic of `extract_bytes` but uses synchronous extractor methods.
+#[cfg(not(feature = "tokio-runtime"))]
+pub(super) fn extract_bytes_sync_impl(
+    content: &[u8],
+    mime_type: Option<&str>,
+    config: Option<&crate::core::config::ExtractionConfig>,
+) -> crate::Result<crate::types::ExtractionResult> {
+    use crate::KreuzbergError;
+    use crate::core::extractor::helpers::get_extractor;
+    use crate::core::mime;
+
+    let cfg = config.cloned().unwrap_or_default();
+    let cfg = cfg.normalized().into_owned();
+
+    let validated_mime = if let Some(mime) = mime_type {
+        if mime == "application/octet-stream" {
+            mime::detect_mime_type_from_bytes(content)?
+        } else {
+            mime::validate_mime_type(mime)?
+        }
+    } else {
+        return Err(KreuzbergError::Validation {
+            message: "MIME type is required for synchronous extraction".to_string(),
+            source: None,
+        });
+    };
+
+    crate::extractors::ensure_initialized()?;
+
+    let extractor = get_extractor(&validated_mime)?;
+
+    let sync_extractor = extractor.as_sync_extractor().ok_or_else(|| {
+        KreuzbergError::UnsupportedFormat(format!(
+            "Extractor for '{}' does not support synchronous extraction",
+            validated_mime
+        ))
+    })?;
+
+    let doc = sync_extractor.extract_sync(content, &validated_mime, &cfg)?;
+
+    let result = crate::core::pipeline::run_pipeline_sync(doc, &cfg)?;
+
+    Ok(result)
+}

crates/kreuzberg/src/core/extractor/mod.rs

@@ -0,0 +1,681 @@
+//! Main extraction entry points.
+//!
+//! This module provides the primary API for extracting content from files and byte arrays.
+//! It orchestrates the entire extraction pipeline: cache checking, MIME detection,
+//! extractor selection, extraction, post-processing, and cache storage.
+//!
+//! # Functions
+//!
+//! - [`extract_file`] - Extract content from a file path
+//! - [`extract_bytes`] - Extract content from a byte array
+//! - [`batch_extract_files`] - Extract content from multiple files concurrently
+//! - [`batch_extract_bytes`] - Extract content from multiple byte arrays concurrently
+
+mod bytes;
+mod file;
+mod helpers;
+mod legacy;
+mod sync;
+
+#[cfg(feature = "tokio-runtime")]
+mod batch;
+
+// Re-export public API
+pub use bytes::extract_bytes;
+pub use file::extract_file;
+pub use sync::{batch_extract_bytes_sync, extract_bytes_sync};
+
+#[cfg(feature = "tokio-runtime")]
+pub use sync::extract_file_sync;
+
+#[cfg(feature = "tokio-runtime")]
+pub use batch::{batch_extract_bytes, batch_extract_files};
+#[cfg(feature = "tokio-runtime")]
+pub use sync::batch_extract_files_sync;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::core::config::{BatchBytesItem, BatchFileItem, ExtractionConfig};
+    use serial_test::serial;
+    use std::fs::File;
+    use std::io::Write;
+    use std::sync::Arc;
+    use tempfile::tempdir;
+
+    fn assert_text_content(actual: &str, expected: &str) {
+        assert_eq!(actual.trim_end_matches('\n'), expected);
+    }
+
+    #[tokio::test]
+    async fn test_extract_file_basic() {
+        let dir = tempdir().unwrap();
+        let file_path = dir.path().join("test.txt");
+        let mut file = File::create(&file_path).unwrap();
+        file.write_all(b"Hello, world!").unwrap();
+
+        let config = ExtractionConfig::default();
+        let result = extract_file(&file_path, None, &config).await;
+
+        assert!(result.is_ok());
+        let result = result.unwrap();
+        assert_text_content(&result.content, "Hello, world!");
+        assert_eq!(result.mime_type, "text/plain");
+    }
+
+    #[tokio::test]
+    async fn test_extract_file_with_mime_override() {
+        let dir = tempdir().unwrap();
+        let file_path = dir.path().join("test.dat");
+        let mut file = File::create(&file_path).unwrap();
+        file.write_all(b"test content").unwrap();
+
+        let config = ExtractionConfig::default();
+        let result = extract_file(&file_path, Some("text/plain"), &config).await;
+
+        assert!(result.is_ok());
+        let result = result.unwrap();
+        assert_eq!(result.mime_type, "text/plain");
+    }
+
+    #[tokio::test]
+    async fn test_extract_file_nonexistent() {
+        let config = ExtractionConfig::default();
+        let result = extract_file("/nonexistent/file.txt", None, &config).await;
+        assert!(result.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_extract_bytes_basic() {
+        let config = ExtractionConfig::default();
+        let result = extract_bytes(b"test content", "text/plain", &config).await;
+
+        assert!(result.is_ok());
+        let result = result.unwrap();
+        assert_text_content(&result.content, "test content");
+        assert_eq!(result.mime_type, "text/plain");
+    }
+
+    #[tokio::test]
+    async fn test_extract_bytes_invalid_mime() {
+        let config = ExtractionConfig::default();
+        let result = extract_bytes(b"test", "invalid/mime", &config).await;
+        assert!(result.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_batch_extract_file() {
+        let dir = tempdir().unwrap();
+
+        let file1 = dir.path().join("test1.txt");
+        let file2 = dir.path().join("test2.txt");
+
+        File::create(&file1).unwrap().write_all(b"content 1").unwrap();
+        File::create(&file2).unwrap().write_all(b"content 2").unwrap();
+
+        let config = ExtractionConfig::default();
+        let items = vec![
+            BatchFileItem {
+                path: file1,
+                config: None,
+            },
+            BatchFileItem {
+                path: file2,
+                config: None,
+            },
+        ];
+        let results = batch_extract_files(items, &config).await;
+
+        assert!(results.is_ok());
+        let results = results.unwrap();
+        assert_eq!(results.len(), 2);
+        assert_text_content(&results[0].content, "content 1");
+        assert_text_content(&results[1].content, "content 2");
+    }
+
+    #[tokio::test]
+    async fn test_batch_extract_file_empty() {
+        let config = ExtractionConfig::default();
+        let items: Vec<BatchFileItem> = vec![];
+        let results = batch_extract_files(items, &config).await;
+
+        assert!(results.is_ok());
+        assert_eq!(results.unwrap().len(), 0);
+    }
+
+    #[tokio::test]
+    async fn test_batch_extract_bytes() {
+        let config = ExtractionConfig::default();
+        let items = vec![
+            BatchBytesItem {
+                content: b"content 1".to_vec(),
+                mime_type: "text/plain".to_string(),
+                config: None,
+            },
+            BatchBytesItem {
+                content: b"content 2".to_vec(),
+                mime_type: "text/plain".to_string(),
+                config: None,
+            },
+        ];
+        let results = batch_extract_bytes(items, &config).await;
+
+        assert!(results.is_ok());
+        let results = results.unwrap();
+        assert_eq!(results.len(), 2);
+        assert_text_content(&results[0].content, "content 1");
+        assert_text_content(&results[1].content, "content 2");
+    }
+
+    #[test]
+    fn test_sync_wrappers() {
+        let dir = tempdir().unwrap();
+        let file_path = dir.path().join("test.txt");
+        File::create(&file_path).unwrap().write_all(b"sync test").unwrap();
+
+        let config = ExtractionConfig::default();
+
+        let result = extract_file_sync(&file_path, None, &config);
+        assert!(result.is_ok());
+        let result = result.unwrap();
+        assert_text_content(&result.content, "sync test");
+
+        let result = extract_bytes_sync(b"test", "text/plain", &config);
+        assert!(result.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_extractor_cache() {
+        let config = ExtractionConfig::default();
+
+        let result1 = extract_bytes(b"test 1", "text/plain", &config).await;
+        assert!(result1.is_ok());
+        let result1 = result1.unwrap();
+
+        let result2 = extract_bytes(b"test 2", "text/plain", &config).await;
+        assert!(result2.is_ok());
+        let result2 = result2.unwrap();
+
+        assert_text_content(&result1.content, "test 1");
+        assert_text_content(&result2.content, "test 2");
+
+        let result3 = extract_bytes(b"# test 3", "text/markdown", &config).await;
+        assert!(result3.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_extract_file_empty() {
+        let dir = tempdir().unwrap();
+        let file_path = dir.path().join("empty.txt");
+        File::create(&file_path).unwrap();
+
+        let config = ExtractionConfig::default();
+        let result = extract_file(&file_path, None, &config).await;
+
+        assert!(result.is_ok());
+        let result = result.unwrap();
+        assert_eq!(result.content, "");
+    }
+
+    #[tokio::test]
+    async fn test_extract_bytes_empty() {
+        let config = ExtractionConfig::default();
+        let result = extract_bytes(b"", "text/plain", &config).await;
+
+        assert!(result.is_ok());
+        let result = result.unwrap();
+        assert_eq!(result.content, "");
+    }
+
+    #[tokio::test]
+    async fn test_extract_file_whitespace_only() {
+        let dir = tempdir().unwrap();
+        let file_path = dir.path().join("whitespace.txt");
+        File::create(&file_path).unwrap().write_all(b"   \n\t  \n  ").unwrap();
+
+        let config = ExtractionConfig::default();
+        let result = extract_file(&file_path, None, &config).await;
+
+        assert!(result.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_extract_file_very_long_path() {
+        let dir = tempdir().unwrap();
+        let long_name = "a".repeat(200);
+        let file_path = dir.path().join(format!("{}.txt", long_name));
+
+        if let Ok(mut f) = File::create(&file_path) {
+            f.write_all(b"content").unwrap();
+            let config = ExtractionConfig::default();
+            let result = extract_file(&file_path, None, &config).await;
+            assert!(result.is_ok() || result.is_err());
+        }
+    }
+
+    #[tokio::test]
+    async fn test_extract_file_special_characters_in_path() {
+        let dir = tempdir().unwrap();
+        let file_path = dir.path().join("test with spaces & symbols!.txt");
+        File::create(&file_path).unwrap().write_all(b"content").unwrap();
+
+        let config = ExtractionConfig::default();
+        let result = extract_file(&file_path, None, &config).await;
+
+        assert!(result.is_ok());
+        let result = result.unwrap();
+        assert_text_content(&result.content, "content");
+    }
+
+    #[tokio::test]
+    async fn test_extract_file_unicode_filename() {
+        let dir = tempdir().unwrap();
+        let file_path = dir.path().join("测试文件名.txt");
+        File::create(&file_path).unwrap().write_all(b"content").unwrap();
+
+        let config = ExtractionConfig::default();
+        let result = extract_file(&file_path, None, &config).await;
+
+        assert!(result.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_extract_bytes_unsupported_mime() {
+        let config = ExtractionConfig::default();
+        let result = extract_bytes(b"test", "application/x-unknown-format", &config).await;
+
+        assert!(result.is_err());
+        use crate::KreuzbergError;
+        assert!(matches!(result.unwrap_err(), KreuzbergError::UnsupportedFormat(_)));
+    }
+
+    #[tokio::test]
+    async fn test_batch_extract_file_with_errors() {
+        let dir = tempdir().unwrap();
+
+        let valid_file = dir.path().join("valid.txt");
+        File::create(&valid_file).unwrap().write_all(b"valid content").unwrap();
+
+        let invalid_file = dir.path().join("nonexistent.txt");
+
+        let config = ExtractionConfig::default();
+        let items = vec![
+            BatchFileItem {
+                path: valid_file,
+                config: None,
+            },
+            BatchFileItem {
+                path: invalid_file,
+                config: None,
+            },
+        ];
+        let results = batch_extract_files(items, &config).await;
+
+        assert!(results.is_ok());
+        let results = results.unwrap();
+        assert_eq!(results.len(), 2);
+        assert_text_content(&results[0].content, "valid content");
+        assert!(results[1].metadata.error.is_some());
+    }
+
+    #[tokio::test]
+    async fn test_batch_extract_bytes_mixed_valid_invalid() {
+        let config = ExtractionConfig::default();
+        let items = vec![
+            BatchBytesItem {
+                content: b"valid 1".to_vec(),
+                mime_type: "text/plain".to_string(),
+                config: None,
+            },
+            BatchBytesItem {
+                content: b"invalid".to_vec(),
+                mime_type: "invalid/mime".to_string(),
+                config: None,
+            },
+            BatchBytesItem {
+                content: b"valid 2".to_vec(),
+                mime_type: "text/plain".to_string(),
+                config: None,
+            },
+        ];
+        let results = batch_extract_bytes(items, &config).await;
+
+        assert!(results.is_ok());
+        let results = results.unwrap();
+        assert_eq!(results.len(), 3);
+        assert_text_content(&results[0].content, "valid 1");
+        assert!(results[1].metadata.error.is_some());
+        assert_text_content(&results[2].content, "valid 2");
+    }
+
+    #[tokio::test]
+    async fn test_batch_extract_bytes_all_invalid() {
+        let config = ExtractionConfig::default();
+        let items = vec![
+            BatchBytesItem {
+                content: b"test 1".to_vec(),
+                mime_type: "invalid/mime1".to_string(),
+                config: None,
+            },
+            BatchBytesItem {
+                content: b"test 2".to_vec(),
+                mime_type: "invalid/mime2".to_string(),
+                config: None,
+            },
+        ];
+        let results = batch_extract_bytes(items, &config).await;
+
+        assert!(results.is_ok());
+        let results = results.unwrap();
+        assert_eq!(results.len(), 2);
+        assert!(results[0].metadata.error.is_some());
+        assert!(results[1].metadata.error.is_some());
+    }
+
+    #[tokio::test]
+    async fn test_extract_bytes_very_large() {
+        let large_content = vec![b'a'; 10_000_000];
+        let config = ExtractionConfig::default();
+        let result = extract_bytes(&large_content, "text/plain", &config).await;
+
+        assert!(result.is_ok());
+        let result = result.unwrap();
+        let trimmed_len = result.content.trim_end_matches('\n').len();
+        assert_eq!(trimmed_len, 10_000_000);
+    }
+
+    #[tokio::test]
+    async fn test_batch_extract_large_count() {
+        let dir = tempdir().unwrap();
+        let mut items = Vec::new();
+
+        for i in 0..100 {
+            let file_path = dir.path().join(format!("file{}.txt", i));
+            File::create(&file_path)
+                .unwrap()
+                .write_all(format!("content {}", i).as_bytes())
+                .unwrap();
+            items.push(BatchFileItem {
+                path: file_path,
+                config: None,
+            });
+        }
+
+        let config = ExtractionConfig::default();
+        let results = batch_extract_files(items, &config).await;
+
+        assert!(results.is_ok());
+        let results = results.unwrap();
+        assert_eq!(results.len(), 100);
+
+        for (i, result) in results.iter().enumerate() {
+            assert_text_content(&result.content, &format!("content {}", i));
+        }
+    }
+
+    #[tokio::test]
+    async fn test_extract_file_mime_detection_fallback() {
+        let dir = tempdir().unwrap();
+        let file_path = dir.path().join("testfile");
+        File::create(&file_path)
+            .unwrap()
+            .write_all(b"plain text content")
+            .unwrap();
+
+        let config = ExtractionConfig::default();
+        let result = extract_file(&file_path, None, &config).await;
+
+        assert!(result.is_ok() || result.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_extract_file_wrong_mime_override() {
+        let dir = tempdir().unwrap();
+        let file_path = dir.path().join("test.txt");
+        File::create(&file_path).unwrap().write_all(b"plain text").unwrap();
+
+        let config = ExtractionConfig::default();
+        let result = extract_file(&file_path, Some("application/pdf"), &config).await;
+
+        assert!(result.is_err() || result.is_ok());
+    }
+
+    #[test]
+    fn test_sync_wrapper_nonexistent_file() {
+        let config = ExtractionConfig::default();
+        let result = extract_file_sync("/nonexistent/path.txt", None, &config);
+
+        assert!(result.is_err());
+        use crate::KreuzbergError;
+        // File validation returns Io error, not Validation error
+        assert!(matches!(result.unwrap_err(), KreuzbergError::Io { .. }));
+    }
+
+    #[test]
+    fn test_sync_wrapper_batch_empty() {
+        let config = ExtractionConfig::default();
+        let items: Vec<BatchFileItem> = vec![];
+        let results = batch_extract_files_sync(items, &config);
+
+        assert!(results.is_ok());
+        assert_eq!(results.unwrap().len(), 0);
+    }
+
+    #[test]
+    fn test_sync_wrapper_batch_bytes_empty() {
+        let config = ExtractionConfig::default();
+        let items: Vec<BatchBytesItem> = vec![];
+        let results = batch_extract_bytes_sync(items, &config);
+
+        assert!(results.is_ok());
+        assert_eq!(results.unwrap().len(), 0);
+    }
+
+    #[tokio::test]
+    async fn test_concurrent_extractions_same_mime() {
+        use tokio::task::JoinSet;
+
+        let config = Arc::new(ExtractionConfig::default());
+        let mut tasks = JoinSet::new();
+
+        for i in 0..50 {
+            let config_clone = Arc::clone(&config);
+            tasks.spawn(async move {
+                let content = format!("test content {}", i);
+                extract_bytes(content.as_bytes(), "text/plain", &config_clone).await
+            });
+        }
+
+        let mut success_count = 0;
+        while let Some(task_result) = tasks.join_next().await {
+            if let Ok(Ok(_)) = task_result {
+                success_count += 1;
+            }
+        }
+
+        assert_eq!(success_count, 50);
+    }
+
+    #[serial]
+    #[tokio::test]
+    async fn test_concurrent_extractions_different_mimes() {
+        use tokio::task::JoinSet;
+
+        let config = Arc::new(ExtractionConfig::default());
+        let mut tasks = JoinSet::new();
+
+        let mime_types = ["text/plain", "text/markdown"];
+
+        for i in 0..30 {
+            let config_clone = Arc::clone(&config);
+            let mime = mime_types[i % mime_types.len()];
+            tasks.spawn(async move {
+                let content = format!("test {}", i);
+                extract_bytes(content.as_bytes(), mime, &config_clone).await
+            });
+        }
+
+        let mut success_count = 0;
+        while let Some(task_result) = tasks.join_next().await {
+            if let Ok(Ok(_)) = task_result {
+                success_count += 1;
+            }
+        }
+
+        assert_eq!(success_count, 30);
+    }
+
+    #[tokio::test]
+    async fn test_batch_extract_file_with_per_file_configs() {
+        let dir = tempdir().unwrap();
+
+        let file1 = dir.path().join("test1.txt");
+        let file2 = dir.path().join("test2.txt");
+        File::create(&file1).unwrap().write_all(b"content 1").unwrap();
+        File::create(&file2).unwrap().write_all(b"content 2").unwrap();
+
+        let config = ExtractionConfig::default();
+        let items = vec![
+            BatchFileItem {
+                path: file1,
+                config: Some(crate::FileExtractionConfig::default()),
+            },
+            BatchFileItem {
+                path: file2,
+                config: None,
+            },
+        ];
+        let results = batch_extract_files(items, &config).await;
+
+        assert!(results.is_ok());
+        let results = results.unwrap();
+        assert_eq!(results.len(), 2);
+        assert_text_content(&results[0].content, "content 1");
+        assert_text_content(&results[1].content, "content 2");
+    }
+
+    #[tokio::test]
+    async fn test_batch_extract_file_with_configs_empty() {
+        let config = ExtractionConfig::default();
+        let items: Vec<BatchFileItem> = vec![];
+        let results = batch_extract_files(items, &config).await;
+
+        assert!(results.is_ok());
+        assert_eq!(results.unwrap().len(), 0);
+    }
+
+    #[tokio::test]
+    async fn test_batch_extract_bytes_with_per_item_configs() {
+        let config = ExtractionConfig::default();
+        let items = vec![
+            BatchBytesItem {
+                content: b"hello".to_vec(),
+                mime_type: "text/plain".to_string(),
+                config: None,
+            },
+            BatchBytesItem {
+                content: b"world".to_vec(),
+                mime_type: "text/plain".to_string(),
+                config: Some(crate::FileExtractionConfig::default()),
+            },
+        ];
+        let results = batch_extract_bytes(items, &config).await;
+
+        assert!(results.is_ok());
+        let results = results.unwrap();
+        assert_eq!(results.len(), 2);
+        assert_text_content(&results[0].content, "hello");
+        assert_text_content(&results[1].content, "world");
+    }
+
+    #[tokio::test]
+    async fn test_batch_extract_bytes_with_configs_error_handling() {
+        let config = ExtractionConfig::default();
+        let items = vec![
+            BatchBytesItem {
+                content: b"valid".to_vec(),
+                mime_type: "text/plain".to_string(),
+                config: None,
+            },
+            BatchBytesItem {
+                content: b"invalid".to_vec(),
+                mime_type: "invalid/mime".to_string(),
+                config: Some(crate::FileExtractionConfig::default()),
+            },
+        ];
+        let results = batch_extract_bytes(items, &config).await;
+
+        assert!(results.is_ok());
+        let results = results.unwrap();
+        assert_eq!(results.len(), 2);
+        assert_text_content(&results[0].content, "valid");
+        assert!(results[1].metadata.error.is_some());
+    }
+
+    #[test]
+    fn test_batch_extract_file_sync_with_configs() {
+        let dir = tempdir().unwrap();
+        let file_path = dir.path().join("test.txt");
+        File::create(&file_path).unwrap().write_all(b"sync test").unwrap();
+
+        let config = ExtractionConfig::default();
+        let items = vec![BatchFileItem {
+            path: file_path,
+            config: None,
+        }];
+        let results = batch_extract_files_sync(items, &config);
+
+        assert!(results.is_ok());
+        let results = results.unwrap();
+        assert_eq!(results.len(), 1);
+        assert_text_content(&results[0].content, "sync test");
+    }
+
+    #[test]
+    fn test_with_file_overrides_single_field() {
+        let base = ExtractionConfig::default();
+        assert!(!base.force_ocr);
+
+        let overrides = crate::FileExtractionConfig {
+            force_ocr: Some(true),
+            ..Default::default()
+        };
+        let resolved = base.with_file_overrides(&overrides);
+        assert!(resolved.force_ocr);
+        // Other fields unchanged
+        assert_eq!(resolved.use_cache, base.use_cache);
+        assert_eq!(resolved.enable_quality_processing, base.enable_quality_processing);
+    }
+
+    #[test]
+    fn test_with_file_overrides_none_keeps_default() {
+        let base = ExtractionConfig::default();
+        let overrides = crate::FileExtractionConfig::default(); // all None
+        let resolved = base.with_file_overrides(&overrides);
+        // All fields should match base
+        assert_eq!(resolved.use_cache, base.use_cache);
+        assert_eq!(resolved.force_ocr, base.force_ocr);
+        assert_eq!(resolved.enable_quality_processing, base.enable_quality_processing);
+        assert_eq!(resolved.include_document_structure, base.include_document_structure);
+    }
+
+    #[test]
+    fn test_with_file_overrides_batch_fields_unaffected() {
+        let base = ExtractionConfig {
+            max_concurrent_extractions: Some(42),
+            use_cache: false,
+            ..Default::default()
+        };
+
+        let overrides = crate::FileExtractionConfig {
+            force_ocr: Some(true),
+            ..Default::default()
+        };
+        let resolved = base.with_file_overrides(&overrides);
+        // Batch-level fields must be preserved from base
+        assert_eq!(resolved.max_concurrent_extractions, Some(42));
+        assert!(!resolved.use_cache);
+        // Override applied
+        assert!(resolved.force_ocr);
+    }
+}

crates/kreuzberg/src/core/extractor/sync.rs

@@ -0,0 +1,200 @@
+//! Synchronous wrappers for extraction operations.
+//!
+//! This module provides blocking synchronous wrappers around async extraction functions
+//! for use in non-async contexts. Uses a global Tokio runtime for optimal performance.
+
+use crate::Result;
+use crate::core::config::BatchBytesItem;
+#[cfg(feature = "tokio-runtime")]
+use crate::core::config::BatchFileItem;
+use crate::core::config::ExtractionConfig;
+use crate::types::ExtractionResult;
+
+#[cfg(feature = "tokio-runtime")]
+use std::path::Path;
+
+#[cfg(feature = "tokio-runtime")]
+use once_cell::sync::OnceCell;
+
+#[cfg(feature = "tokio-runtime")]
+use super::batch::{batch_extract_bytes, batch_extract_files};
+#[cfg(feature = "tokio-runtime")]
+use super::bytes::extract_bytes;
+#[cfg(feature = "tokio-runtime")]
+use super::file::extract_file;
+
+#[cfg(not(feature = "tokio-runtime"))]
+use super::helpers::error_extraction_result;
+
+/// Global Tokio runtime cell for synchronous operations.
+///
+/// Lazily initialized on first use and shared across all sync wrappers.
+/// Using a global runtime instead of creating one per call provides 100x+ performance improvement.
+///
+/// # Availability
+///
+/// This static is only available when the `tokio-runtime` feature is enabled.
+/// For WASM targets, use the truly synchronous extraction functions instead.
+#[cfg(feature = "tokio-runtime")]
+static GLOBAL_RUNTIME: OnceCell<tokio::runtime::Runtime> = OnceCell::new();
+
+/// Returns a reference to the global Tokio runtime, initializing it on first call.
+///
+/// Returns an error if the runtime cannot be created (e.g. system resource exhaustion).
+#[cfg(feature = "tokio-runtime")]
+fn global_runtime() -> crate::Result<&'static tokio::runtime::Runtime> {
+    GLOBAL_RUNTIME.get_or_try_init(|| {
+        tokio::runtime::Builder::new_multi_thread()
+            .enable_all()
+            .build()
+            .map_err(|e| crate::KreuzbergError::Plugin {
+                message: format!("Failed to create global Tokio runtime: {e}"),
+                plugin_name: "runtime".to_string(),
+            })
+    })
+}
+
+/// Synchronous wrapper for `extract_file`.
+///
+/// This is a convenience function that blocks the current thread until extraction completes.
+/// For async code, use `extract_file` directly.
+///
+/// Uses the global Tokio runtime for 100x+ performance improvement over creating
+/// a new runtime per call. Always uses the global runtime to avoid nested runtime issues.
+///
+/// This function is only available with the `tokio-runtime` feature. For WASM targets,
+/// use a truly synchronous extraction approach instead.
+///
+/// # Example
+///
+/// ```rust,no_run
+/// use kreuzberg::core::extractor::extract_file_sync;
+/// use kreuzberg::core::config::ExtractionConfig;
+///
+/// let config = ExtractionConfig::default();
+/// let result = extract_file_sync("document.pdf", None, &config)?;
+/// println!("Content: {}", result.content);
+/// # Ok::<(), kreuzberg::KreuzbergError>(())
+/// ```
+#[cfg(feature = "tokio-runtime")]
+pub fn extract_file_sync(
+    path: impl AsRef<Path>,
+    mime_type: Option<&str>,
+    config: &ExtractionConfig,
+) -> Result<ExtractionResult> {
+    global_runtime()?.block_on(extract_file(path, mime_type, config))
+}
+
+/// Synchronous wrapper for `extract_bytes`.
+///
+/// Uses the global Tokio runtime for 100x+ performance improvement over creating
+/// a new runtime per call.
+///
+/// With the `tokio-runtime` feature, this blocks the current thread using the global
+/// Tokio runtime. Without it (WASM), this calls a truly synchronous implementation.
+///
+/// # Example
+///
+/// ```rust,no_run
+/// use kreuzberg::core::extractor::extract_bytes_sync;
+/// use kreuzberg::core::config::ExtractionConfig;
+///
+/// let config = ExtractionConfig::default();
+/// let bytes = b"Hello, world!";
+/// let result = extract_bytes_sync(bytes, "text/plain", &config)?;
+/// println!("Content: {}", result.content);
+/// # Ok::<(), kreuzberg::KreuzbergError>(())
+/// ```
+#[cfg(feature = "tokio-runtime")]
+pub fn extract_bytes_sync(content: &[u8], mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
+    global_runtime()?.block_on(extract_bytes(content, mime_type, config))
+}
+
+/// Synchronous wrapper for `extract_bytes` (WASM-compatible version).
+///
+/// This is a truly synchronous implementation without tokio runtime dependency.
+/// It calls `extract_bytes_sync_impl()` to perform the extraction.
+#[cfg(not(feature = "tokio-runtime"))]
+pub fn extract_bytes_sync(content: &[u8], mime_type: &str, config: &ExtractionConfig) -> Result<ExtractionResult> {
+    super::legacy::extract_bytes_sync_impl(content, Some(mime_type), Some(config))
+}
+
+/// Synchronous wrapper for `batch_extract_files`.
+///
+/// Uses the global Tokio runtime for optimal performance.
+/// Only available with `tokio-runtime` (WASM has no filesystem).
+///
+/// # Example
+///
+/// ```rust,no_run
+/// use kreuzberg::core::extractor::batch_extract_files_sync;
+/// use kreuzberg::core::config::{ExtractionConfig, BatchFileItem, FileExtractionConfig};
+///
+/// let config = ExtractionConfig::default();
+/// let items = vec![
+///     BatchFileItem {
+///         path: "doc1.pdf".into(),
+///         config: Some(FileExtractionConfig { force_ocr: Some(true), ..Default::default() }),
+///     },
+///     BatchFileItem { path: "doc2.pdf".into(), config: None },
+/// ];
+/// let results = batch_extract_files_sync(items, &config)?;
+/// # Ok::<(), kreuzberg::KreuzbergError>(())
+/// ```
+#[cfg(feature = "tokio-runtime")]
+pub fn batch_extract_files_sync(items: Vec<BatchFileItem>, config: &ExtractionConfig) -> Result<Vec<ExtractionResult>> {
+    global_runtime()?.block_on(batch_extract_files(items, config))
+}
+
+/// Synchronous wrapper for `batch_extract_bytes`.
+///
+/// Uses the global Tokio runtime for optimal performance.
+/// With the `tokio-runtime` feature, this blocks the current thread using the global
+/// Tokio runtime. Without it (WASM), this calls a truly synchronous implementation
+/// that iterates through items and calls `extract_bytes_sync()`.
+///
+/// # Example
+///
+/// ```rust,no_run
+/// use kreuzberg::core::extractor::batch_extract_bytes_sync;
+/// use kreuzberg::core::config::{ExtractionConfig, BatchBytesItem, FileExtractionConfig};
+///
+/// let config = ExtractionConfig::default();
+/// let items = vec![
+///     BatchBytesItem { content: b"content".to_vec(), mime_type: "text/plain".to_string(), config: None },
+///     BatchBytesItem {
+///         content: b"other".to_vec(),
+///         mime_type: "text/plain".to_string(),
+///         config: Some(FileExtractionConfig { force_ocr: Some(true), ..Default::default() }),
+///     },
+/// ];
+/// let results = batch_extract_bytes_sync(items, &config)?;
+/// # Ok::<(), kreuzberg::KreuzbergError>(())
+/// ```
+#[cfg(feature = "tokio-runtime")]
+pub fn batch_extract_bytes_sync(
+    items: Vec<BatchBytesItem>,
+    config: &ExtractionConfig,
+) -> Result<Vec<ExtractionResult>> {
+    global_runtime()?.block_on(batch_extract_bytes(items, config))
+}
+
+/// Synchronous wrapper for `batch_extract_bytes` (WASM-compatible version).
+///
+/// Iterates through items sequentially, applying per-file config overrides.
+#[cfg(not(feature = "tokio-runtime"))]
+pub fn batch_extract_bytes_sync(
+    items: Vec<BatchBytesItem>,
+    config: &ExtractionConfig,
+) -> Result<Vec<ExtractionResult>> {
+    let mut results = Vec::with_capacity(items.len());
+    for item in items {
+        let resolved = match &item.config {
+            Some(fc) => config.with_file_overrides(fc),
+            None => config.clone(),
+        };
+        let result = extract_bytes_sync(&item.content, &item.mime_type, &resolved);
+        results.push(result.unwrap_or_else(|e| error_extraction_result(&e, None)));
+    }
+    Ok(results)
+}

crates/kreuzberg/src/core/formats.rs

@@ -0,0 +1,237 @@
+//! Format field validation and metadata.
+//!
+//! This module provides a compile-time registry of known format fields used across
+//! document extractors. It serves as the single source of truth for format validation
+//! across all language bindings (Rust, Python, TypeScript, Ruby, Java, Go).
+//!
+//! # Known Format Fields
+//!
+//! The registry contains 55 standardized format fields organized by category:
+//! - **Document Properties**: title, author, keywords, creator, producer, etc.
+//! - **Dates**: creation_date, modification_date
+//! - **Pagination**: page_count
+//! - **Email Metadata**: from_email, from_name, to_emails, cc_emails, bcc_emails, message_id
+//! - **Attachments**: attachments
+//! - **Descriptions**: description, summary
+//! - **Typography**: fonts
+//! - **Archive/Compression**: format, file_count, file_list, total_size, compressed_size
+//! - **Images**: width, height
+//! - **Content Metrics**: element_count, unique_elements, line_count, word_count, character_count
+//! - **HTML Structure**: headers, links, code_blocks
+//! - **Meta Tags**: canonical, base_href, og_*, twitter_*, link_*
+//! - **OCR**: psm, output_format
+//! - **Tables**: table_count, table_rows, table_cols
+//!
+//! # Example
+//!
+//! ```ignore
+//! use kreuzberg::core::formats::{KNOWN_FORMATS, is_valid_format_field};
+//!
+//! assert!(is_valid_format_field("title"));
+//! assert!(!is_valid_format_field("invalid_field"));
+//! assert_eq!(KNOWN_FORMATS.len(), 55);
+//! ```
+
+#[cfg(test)]
+use ahash::AHashSet;
+#[cfg(test)]
+use std::sync::LazyLock;
+
+/// All known format field names across all extractors.
+///
+/// This is a compile-time constant array of standardized field names used by document
+/// extractors. Each binding (Python, TypeScript, Ruby, Java, Go) should reference this
+/// as the single source of truth for format field validation.
+///
+/// Format fields are organized by document type:
+/// - PDF/Office: title, author, creation_date, page_count, etc.
+/// - Email: from_email, to_emails, cc_emails, bcc_emails, etc.
+/// - Web: og_title, twitter_card, canonical, headers, links, etc.
+/// - Images: width, height, format
+/// - Archives: file_count, file_list, total_size, etc.
+#[cfg(test)]
+pub(crate) const KNOWN_FORMATS: &[&str] = &[
+    "title",
+    "author",
+    "keywords",
+    "creator",
+    "producer",
+    "creation_date",
+    "modification_date",
+    "page_count",
+    "from_email",
+    "from_name",
+    "to_emails",
+    "cc_emails",
+    "bcc_emails",
+    "message_id",
+    "attachments",
+    "description",
+    "summary",
+    "fonts",
+    "format",
+    "file_count",
+    "file_list",
+    "total_size",
+    "compressed_size",
+    "width",
+    "height",
+    "element_count",
+    "unique_elements",
+    "line_count",
+    "word_count",
+    "character_count",
+    "headers",
+    "links",
+    "code_blocks",
+    "canonical",
+    "base_href",
+    "og_title",
+    "og_description",
+    "og_image",
+    "og_url",
+    "og_type",
+    "og_site_name",
+    "twitter_card",
+    "twitter_title",
+    "twitter_description",
+    "twitter_image",
+    "twitter_site",
+    "twitter_creator",
+    "link_author",
+    "link_license",
+    "link_alternate",
+    "psm",
+    "output_format",
+    "table_count",
+    "table_rows",
+    "table_cols",
+];
+
+/// Cached format field set for fast O(1) lookups.
+///
+/// Uses AHashSet for its excellent cache locality and performance characteristics
+/// with string keys. Built lazily on first use with minimal overhead.
+#[cfg(test)]
+static FORMAT_FIELD_SET: LazyLock<AHashSet<&'static str>> = LazyLock::new(|| KNOWN_FORMATS.iter().copied().collect());
+
+/// Validates whether a field name is in the known formats registry.
+///
+/// This uses a pre-built hash set for O(1) lookups instead of linear search,
+/// providing significant performance improvements for repeated validations.
+///
+/// # Arguments
+///
+/// * `field` - The field name to validate
+///
+/// # Returns
+///
+/// `true` if the field is in KNOWN_FORMATS, `false` otherwise.
+///
+/// # Example
+///
+/// ```rust
+/// use kreuzberg::core::formats::is_valid_format_field;
+///
+/// assert!(is_valid_format_field("title"));
+/// assert!(is_valid_format_field("creation_date"));
+/// assert!(!is_valid_format_field("invalid_field"));
+/// ```
+#[cfg(test)]
+#[inline]
+pub(crate) fn is_valid_format_field(field: &str) -> bool {
+    FORMAT_FIELD_SET.contains(field)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_known_formats_count() {
+        assert_eq!(KNOWN_FORMATS.len(), 55, "Expected 55 known format fields");
+    }
+
+    #[test]
+    fn test_known_formats_no_duplicates() {
+        let mut seen = std::collections::HashSet::new();
+        for field in KNOWN_FORMATS {
+            assert!(seen.insert(field), "Duplicate format field found: {}", field);
+        }
+    }
+
+    #[test]
+    fn test_is_valid_format_field_true_cases() {
+        assert!(is_valid_format_field("title"));
+        assert!(is_valid_format_field("author"));
+        assert!(is_valid_format_field("creation_date"));
+        assert!(is_valid_format_field("page_count"));
+        assert!(is_valid_format_field("from_email"));
+        assert!(is_valid_format_field("og_title"));
+        assert!(is_valid_format_field("twitter_card"));
+    }
+
+    #[test]
+    fn test_is_valid_format_field_false_cases() {
+        assert!(!is_valid_format_field("invalid_field"));
+        assert!(!is_valid_format_field("unknown_metadata"));
+        assert!(!is_valid_format_field(""));
+        assert!(!is_valid_format_field("TITLE"));
+        assert!(!is_valid_format_field("title "));
+    }
+
+    #[test]
+    fn test_all_document_property_fields() {
+        let doc_fields = ["title", "author", "keywords", "creator", "producer"];
+        for field in &doc_fields {
+            assert!(is_valid_format_field(field), "Missing field: {}", field);
+        }
+    }
+
+    #[test]
+    fn test_all_email_fields() {
+        let email_fields = [
+            "from_email",
+            "from_name",
+            "to_emails",
+            "cc_emails",
+            "bcc_emails",
+            "message_id",
+            "attachments",
+        ];
+        for field in &email_fields {
+            assert!(is_valid_format_field(field), "Missing email field: {}", field);
+        }
+    }
+
+    #[test]
+    fn test_all_web_meta_fields() {
+        let web_fields = [
+            "og_title",
+            "og_description",
+            "og_image",
+            "og_url",
+            "og_type",
+            "og_site_name",
+            "twitter_card",
+            "twitter_title",
+            "twitter_description",
+            "twitter_image",
+            "twitter_site",
+            "twitter_creator",
+            "canonical",
+            "base_href",
+        ];
+        for field in &web_fields {
+            assert!(is_valid_format_field(field), "Missing web field: {}", field);
+        }
+    }
+
+    #[test]
+    fn test_all_table_fields() {
+        let table_fields = ["table_count", "table_rows", "table_cols"];
+        for field in &table_fields {
+            assert!(is_valid_format_field(field), "Missing table field: {}", field);
+        }
+    }
+}

crates/kreuzberg/src/core/io.rs

@@ -0,0 +1,421 @@
+//! File I/O utilities.
+//!
+//! This module provides async and sync file reading utilities with proper error handling.
+//! For large files (> 1 MiB) on non-WASM platforms, memory-mapped I/O is used to avoid
+//! heap-allocating the entire file contents, reducing memory pressure and syscall overhead.
+
+use crate::{KreuzbergError, Result};
+use std::path::Path;
+
+/// Size threshold above which memory-mapped I/O is preferred over `read()`.
+///
+/// Files smaller than this are read with a regular `read()` call since the
+/// mmap overhead (open, fstat, mmap syscalls + TLB pressure) outweighs the
+/// benefit for small allocations.
+#[cfg(not(target_arch = "wasm32"))]
+const MMAP_THRESHOLD_BYTES: u64 = 1_048_576; // 1 MiB
+
+/// An owned buffer of file bytes.
+///
+/// On non-WASM platforms this may be backed by a memory-mapped file (zero heap
+/// allocation for the file contents) or by a `Vec<u8>` for small files.
+/// On WASM it is always a `Vec<u8>`.
+///
+/// Implements `Deref<Target = [u8]>` so callers can pass `&FileBytes` as `&[u8]`
+/// without any additional copy.
+#[cfg_attr(alef, alef(skip))]
+pub struct FileBytes {
+    inner: FileBytesInner,
+}
+
+enum FileBytesInner {
+    /// Regular heap-allocated buffer (small files or WASM).
+    Heap(Vec<u8>),
+    /// Memory-mapped file (large files on native platforms).
+    #[cfg(not(target_arch = "wasm32"))]
+    Mapped(memmap2::Mmap),
+}
+
+impl std::ops::Deref for FileBytes {
+    type Target = [u8];
+
+    fn deref(&self) -> &[u8] {
+        match &self.inner {
+            FileBytesInner::Heap(v) => v.as_slice(),
+            #[cfg(not(target_arch = "wasm32"))]
+            FileBytesInner::Mapped(m) => m.as_ref(),
+        }
+    }
+}
+
+impl AsRef<[u8]> for FileBytes {
+    fn as_ref(&self) -> &[u8] {
+        self
+    }
+}
+
+/// Open a file and return its bytes with zero-copy for large files.
+///
+/// On non-WASM targets, files larger than [`MMAP_THRESHOLD_BYTES`] are
+/// memory-mapped so that the file contents are never copied to the heap.
+/// The mapping is read-only; the file must not be modified while the returned
+/// [`FileBytes`] is alive, which is safe for document extraction.
+///
+/// On WASM or for small files, falls back to a plain `std::fs::read`.
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Io` for any I/O failure.
+#[allow(unsafe_code)]
+pub(crate) fn open_file_bytes(path: &Path) -> Result<FileBytes> {
+    #[cfg(not(target_arch = "wasm32"))]
+    {
+        let metadata = std::fs::metadata(path).map_err(KreuzbergError::Io)?;
+        if metadata.len() > MMAP_THRESHOLD_BYTES {
+            let file = std::fs::File::open(path).map_err(KreuzbergError::Io)?;
+            // SAFETY: The file is opened read-only and we do not write to the
+            // mapped region.  The `FileBytes` value owns the `Mmap` handle and
+            // the mapping is live for exactly as long as the bytes are accessed.
+            // External modification of the file while mapped is a documented
+            // TOCTOU risk inherent to mmap on all platforms; it is acceptable
+            // here because kreuzberg only reads user-supplied documents and
+            // makes no correctness guarantees about files modified concurrently.
+            let mmap = unsafe { memmap2::Mmap::map(&file) }.map_err(KreuzbergError::Io)?;
+            return Ok(FileBytes {
+                inner: FileBytesInner::Mapped(mmap),
+            });
+        }
+    }
+    // Small file or WASM: regular heap read.
+    let bytes = std::fs::read(path).map_err(KreuzbergError::Io)?;
+    Ok(FileBytes {
+        inner: FileBytesInner::Heap(bytes),
+    })
+}
+
+/// Read a file asynchronously.
+///
+/// # Arguments
+///
+/// * `path` - Path to the file to read
+///
+/// # Returns
+///
+/// The file contents as bytes.
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Io` for I/O errors (these always bubble up).
+#[cfg(feature = "tokio-runtime")]
+pub(crate) async fn read_file_async(path: impl AsRef<Path>) -> Result<Vec<u8>> {
+    tokio::fs::read(path.as_ref()).await.map_err(KreuzbergError::Io)
+}
+
+/// Read a file synchronously.
+///
+/// # Arguments
+///
+/// * `path` - Path to the file to read
+///
+/// # Returns
+///
+/// The file contents as bytes.
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Io` for I/O errors (these always bubble up).
+#[cfg(test)]
+pub(crate) fn read_file_sync(path: impl AsRef<Path>) -> Result<Vec<u8>> {
+    std::fs::read(path.as_ref()).map_err(KreuzbergError::Io)
+}
+
+/// Check if a file exists.
+///
+/// # Arguments
+///
+/// * `path` - Path to check
+///
+/// # Returns
+///
+/// `true` if the file exists, `false` otherwise.
+pub(crate) fn file_exists(path: impl AsRef<Path>) -> bool {
+    path.as_ref().exists()
+}
+
+/// Validate that a file exists.
+///
+/// # Arguments
+///
+/// * `path` - Path to validate
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Io` if file doesn't exist.
+pub(crate) fn validate_file_exists(path: impl AsRef<Path>) -> Result<()> {
+    if !file_exists(&path) {
+        return Err(KreuzbergError::from(std::io::Error::new(
+            std::io::ErrorKind::NotFound,
+            format!("File does not exist: {}", path.as_ref().display()),
+        )));
+    }
+    Ok(())
+}
+
+/// Traverse a directory and return all file paths matching a pattern.
+///
+/// # Arguments
+///
+/// * `dir` - Directory to traverse
+/// * `recursive` - Whether to recursively traverse subdirectories
+/// * `filter` - Optional filter function to match files
+///
+/// # Returns
+///
+/// Vector of file paths that match the criteria.
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Io` for I/O errors.
+#[cfg(test)]
+pub(crate) fn traverse_directory<F>(
+    dir: impl AsRef<Path>,
+    recursive: bool,
+    filter: Option<F>,
+) -> Result<Vec<std::path::PathBuf>>
+where
+    F: Fn(&Path) -> bool,
+{
+    let dir = dir.as_ref();
+    let mut files = Vec::new();
+
+    if !dir.is_dir() {
+        return Err(KreuzbergError::from(std::io::Error::new(
+            std::io::ErrorKind::NotADirectory,
+            format!("Path is not a directory: {}", dir.display()),
+        )));
+    }
+
+    traverse_directory_impl(dir, recursive, &filter, &mut files)?;
+    Ok(files)
+}
+
+#[cfg(test)]
+fn traverse_directory_impl<F>(
+    dir: &Path,
+    recursive: bool,
+    filter: &Option<F>,
+    files: &mut Vec<std::path::PathBuf>,
+) -> Result<()>
+where
+    F: Fn(&Path) -> bool,
+{
+    let entries = std::fs::read_dir(dir).map_err(KreuzbergError::Io)?;
+
+    for entry in entries {
+        let entry = entry.map_err(KreuzbergError::Io)?;
+        let path = entry.path();
+
+        if path.is_file() {
+            let should_include = match filter {
+                Some(f) => f(&path),
+                None => true,
+            };
+
+            if should_include {
+                files.push(path);
+            }
+        } else if path.is_dir() && recursive {
+            traverse_directory_impl(&path, recursive, filter, files)?;
+        }
+    }
+
+    Ok(())
+}
+
+/// Get all files in a directory with a specific extension.
+///
+/// # Arguments
+///
+/// * `dir` - Directory to search
+/// * `extension` - File extension to match (without the dot)
+/// * `recursive` - Whether to recursively search subdirectories
+///
+/// # Returns
+///
+/// Vector of file paths with the specified extension.
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Io` for I/O errors.
+#[cfg(test)]
+pub(crate) fn find_files_by_extension(
+    dir: impl AsRef<Path>,
+    extension: &str,
+    recursive: bool,
+) -> Result<Vec<std::path::PathBuf>> {
+    let ext = extension.to_lowercase();
+    traverse_directory(
+        dir,
+        recursive,
+        Some(|path: &Path| {
+            path.extension()
+                .and_then(|e| e.to_str())
+                .map(|e| e.to_lowercase() == ext)
+                .unwrap_or(false)
+        }),
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs::File;
+    use std::io::Write;
+    use tempfile::tempdir;
+
+    #[cfg(feature = "tokio-runtime")]
+    #[tokio::test]
+    async fn test_read_file_async() {
+        let dir = tempdir().unwrap();
+        let file_path = dir.path().join("test.txt");
+        let mut file = File::create(&file_path).unwrap();
+        file.write_all(b"test content").unwrap();
+
+        let content = read_file_async(&file_path).await.unwrap();
+        assert_eq!(content, b"test content");
+    }
+
+    #[test]
+    fn test_read_file_sync() {
+        let dir = tempdir().unwrap();
+        let file_path = dir.path().join("test.txt");
+        let mut file = File::create(&file_path).unwrap();
+        file.write_all(b"test content").unwrap();
+
+        let content = read_file_sync(&file_path).unwrap();
+        assert_eq!(content, b"test content");
+    }
+
+    #[test]
+    fn test_file_exists() {
+        let dir = tempdir().unwrap();
+        let file_path = dir.path().join("test.txt");
+        File::create(&file_path).unwrap();
+
+        assert!(file_exists(&file_path));
+        assert!(!file_exists(dir.path().join("nonexistent.txt")));
+    }
+
+    #[test]
+    fn test_validate_file_exists() {
+        let dir = tempdir().unwrap();
+        let file_path = dir.path().join("test.txt");
+        File::create(&file_path).unwrap();
+
+        assert!(validate_file_exists(&file_path).is_ok());
+        assert!(validate_file_exists(dir.path().join("nonexistent.txt")).is_err());
+    }
+
+    #[test]
+    fn test_traverse_directory_non_recursive() {
+        let dir = tempdir().unwrap();
+
+        File::create(dir.path().join("file1.txt")).unwrap();
+        File::create(dir.path().join("file2.pdf")).unwrap();
+        File::create(dir.path().join("file3.txt")).unwrap();
+
+        std::fs::create_dir(dir.path().join("subdir")).unwrap();
+        File::create(dir.path().join("subdir").join("file4.txt")).unwrap();
+
+        let files = traverse_directory(dir.path(), false, None::<fn(&Path) -> bool>).unwrap();
+        assert_eq!(files.len(), 3);
+    }
+
+    #[test]
+    fn test_traverse_directory_recursive() {
+        let dir = tempdir().unwrap();
+
+        File::create(dir.path().join("file1.txt")).unwrap();
+        File::create(dir.path().join("file2.pdf")).unwrap();
+
+        std::fs::create_dir(dir.path().join("subdir")).unwrap();
+        File::create(dir.path().join("subdir").join("file3.txt")).unwrap();
+        File::create(dir.path().join("subdir").join("file4.pdf")).unwrap();
+
+        let files = traverse_directory(dir.path(), true, None::<fn(&Path) -> bool>).unwrap();
+        assert_eq!(files.len(), 4);
+    }
+
+    #[test]
+    fn test_traverse_directory_with_filter() {
+        let dir = tempdir().unwrap();
+
+        File::create(dir.path().join("file1.txt")).unwrap();
+        File::create(dir.path().join("file2.pdf")).unwrap();
+        File::create(dir.path().join("file3.txt")).unwrap();
+
+        let files = traverse_directory(
+            dir.path(),
+            false,
+            Some(|path: &Path| {
+                path.extension()
+                    .and_then(|e| e.to_str())
+                    .map(|e| e == "txt")
+                    .unwrap_or(false)
+            }),
+        )
+        .unwrap();
+
+        assert_eq!(files.len(), 2);
+        assert!(files.iter().all(|p| p.extension().unwrap() == "txt"));
+    }
+
+    #[test]
+    fn test_find_files_by_extension() {
+        let dir = tempdir().unwrap();
+
+        File::create(dir.path().join("file1.txt")).unwrap();
+        File::create(dir.path().join("file2.pdf")).unwrap();
+        File::create(dir.path().join("file3.TXT")).unwrap();
+
+        std::fs::create_dir(dir.path().join("subdir")).unwrap();
+        File::create(dir.path().join("subdir").join("file4.txt")).unwrap();
+
+        let files = find_files_by_extension(dir.path(), "txt", false).unwrap();
+        assert_eq!(files.len(), 2);
+
+        let files_recursive = find_files_by_extension(dir.path(), "txt", true).unwrap();
+        assert_eq!(files_recursive.len(), 3);
+    }
+
+    #[test]
+    fn test_traverse_directory_invalid_path() {
+        let result = traverse_directory("/nonexistent/directory", false, None::<fn(&Path) -> bool>);
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_traverse_directory_file_not_dir() {
+        let dir = tempdir().unwrap();
+        let file_path = dir.path().join("test.txt");
+        File::create(&file_path).unwrap();
+
+        let result = traverse_directory(&file_path, false, None::<fn(&Path) -> bool>);
+        assert!(result.is_err());
+    }
+
+    #[cfg(feature = "tokio-runtime")]
+    #[tokio::test]
+    async fn test_read_file_async_io_error() {
+        let result = read_file_async("/nonexistent/file.txt").await;
+        assert!(result.is_err());
+        assert!(matches!(result.unwrap_err(), KreuzbergError::Io(_)));
+    }
+
+    #[test]
+    fn test_read_file_sync_io_error() {
+        let result = read_file_sync("/nonexistent/file.txt");
+        assert!(result.is_err());
+        assert!(matches!(result.unwrap_err(), KreuzbergError::Io(_)));
+    }
+}

crates/kreuzberg/src/core/mod.rs

@@ -0,0 +1,61 @@
+//! Core extraction orchestration module.
+//!
+//! This module contains the main extraction logic and orchestration layer for Kreuzberg.
+//! It provides the primary entry points for file and bytes extraction, manages the
+//! extractor registry, MIME type detection, configuration, and post-processing pipeline.
+//!
+//! # Architecture
+//!
+//! The core module is responsible for:
+//! - **Entry Points**: Main `extract_file()` and `extract_bytes()` functions
+//! - **Registry**: Mapping MIME types to extractors with priority-based selection
+//! - **MIME Detection**: Detecting and validating MIME types from files and extensions
+//! - **Pipeline**: Orchestrating post-processing steps (chunking, quality, etc.)
+//! - **Configuration**: Loading and managing extraction configuration
+//! - **I/O**: File reading and validation utilities
+//!
+//! # Example
+//!
+//! ```rust,no_run
+//! use kreuzberg::core::extractor::extract_file;
+//! use kreuzberg::core::config::ExtractionConfig;
+//!
+//! # async fn example() -> kreuzberg::Result<()> {
+//! let config = ExtractionConfig::default();
+//! let result = extract_file("document.pdf", None, &config).await?;
+//! println!("Extracted content: {}", result.content);
+//! # Ok(())
+//! # }
+//! ```
+
+#[cfg(feature = "tokio-runtime")]
+pub mod batch_mode;
+#[cfg(feature = "tokio-runtime")]
+pub mod batch_optimizations;
+pub mod config;
+pub mod config_validation;
+pub mod extractor;
+pub mod formats;
+pub mod io;
+pub mod mime;
+pub(crate) mod path_resolver;
+pub mod pipeline;
+#[cfg(feature = "api-types")]
+pub mod server_config;
+
+#[cfg(feature = "pdf")]
+pub use config::HierarchyConfig;
+pub use config::{
+    ChunkingConfig, EmbeddingConfig, EmbeddingModelType, ExtractionConfig, ImageExtractionConfig,
+    LanguageDetectionConfig, OcrConfig, OutputFormat, PageConfig, PostProcessorConfig, TokenReductionOptions,
+};
+#[cfg(feature = "api-types")]
+pub use server_config::ServerConfig;
+
+#[cfg(feature = "tokio-runtime")]
+pub use batch_optimizations::{BatchProcessor, BatchProcessorConfig};
+#[cfg(feature = "pdf")]
+pub use config::PdfConfig;
+#[cfg(feature = "tokio-runtime")]
+pub use extractor::{batch_extract_bytes, batch_extract_files};
+pub use extractor::{extract_bytes, extract_file};

crates/kreuzberg/src/core/path_resolver.rs

@@ -0,0 +1,282 @@
+//! Image path resolution for markup extractors.
+//!
+//! Resolves relative image paths found in markup documents (Markdown, LaTeX, RST,
+//! Org-mode, Typst, Djot, DocBook) to actual filesystem paths, reads the image data,
+//! and attaches them to the extraction result.
+
+use std::borrow::Cow;
+use std::path::{Path, PathBuf};
+
+use bytes::Bytes;
+
+use crate::core::config::ExtractionConfig;
+use crate::types::ExtractedImage;
+use crate::types::internal::InternalDocument;
+use crate::types::uri::UriKind;
+
+/// Maximum image file size: 50 MB.
+const MAX_IMAGE_SIZE: u64 = 50 * 1024 * 1024;
+
+/// Resolve a relative image reference against a base directory.
+///
+/// Returns `None` for URLs, absolute paths, and paths that escape `base_dir`
+/// via traversal (`..`). Returns `Some(resolved)` for safe relative paths.
+///
+/// This function performs no filesystem access — it only validates the
+/// structural safety of the path.
+pub(crate) fn resolve_image_path(base_dir: &Path, image_ref: &str) -> Option<PathBuf> {
+    let trimmed = image_ref.trim();
+
+    // Reject URLs
+    if trimmed.starts_with("http://")
+        || trimmed.starts_with("https://")
+        || trimmed.starts_with("data:")
+        || trimmed.starts_with("ftp://")
+        || trimmed.starts_with("mailto:")
+    {
+        return None;
+    }
+
+    // Strip file:// or file: prefix (org-mode uses file: without //)
+    let path_str = if let Some(stripped) = trimmed.strip_prefix("file://") {
+        stripped
+    } else if let Some(stripped) = trimmed.strip_prefix("file:") {
+        stripped
+    } else {
+        trimmed
+    };
+
+    // Reject absolute paths (Unix or Windows drive letter)
+    if path_str.starts_with('/')
+        || (path_str.len() >= 2 && path_str.as_bytes()[0].is_ascii_alphabetic() && path_str.as_bytes()[1] == b':')
+    {
+        return None;
+    }
+
+    let joined = base_dir.join(path_str);
+    let normalized = normalize_path(&joined);
+
+    // Path traversal prevention: resolved path must start with base_dir
+    let norm_base = normalize_path(base_dir);
+    if !normalized.starts_with(&norm_base) {
+        return None;
+    }
+
+    Some(normalized)
+}
+
+/// Read an image file and produce an `ExtractedImage`.
+///
+/// Returns `None` if the file does not exist, is not a regular file,
+/// exceeds the size limit, or has an unrecognised extension.
+pub(crate) fn read_image_file(path: &Path, image_index: u32) -> Option<ExtractedImage> {
+    let meta = std::fs::metadata(path).ok()?;
+    if !meta.is_file() {
+        return None;
+    }
+    if meta.len() > MAX_IMAGE_SIZE {
+        return None;
+    }
+
+    let ext = path
+        .extension()
+        .and_then(|e| e.to_str())
+        .map(|s| s.to_ascii_lowercase())?;
+
+    let format: Cow<'static, str> = match ext.as_str() {
+        "png" => Cow::Borrowed("png"),
+        "jpg" | "jpeg" => Cow::Borrowed("jpeg"),
+        "gif" => Cow::Borrowed("gif"),
+        "webp" => Cow::Borrowed("webp"),
+        "svg" => Cow::Borrowed("svg"),
+        "bmp" => Cow::Borrowed("bmp"),
+        "tiff" | "tif" => Cow::Borrowed("tiff"),
+        "avif" => Cow::Borrowed("avif"),
+        _ => return None,
+    };
+
+    let data = std::fs::read(path).ok()?;
+    let source_path = path.to_string_lossy().into_owned();
+
+    Some(ExtractedImage {
+        data: Bytes::from(data),
+        format,
+        image_index,
+        page_number: None,
+        width: None,
+        height: None,
+        colorspace: None,
+        bits_per_component: None,
+        is_mask: false,
+        description: None,
+        ocr_result: None,
+        bounding_box: None,
+        source_path: Some(source_path),
+        image_kind: None,
+        kind_confidence: None,
+        cluster_id: None,
+    })
+}
+
+/// Resolve image URIs in an `InternalDocument` to actual image data.
+///
+/// Iterates over all `UriKind::Image` entries, resolves them relative to
+/// `base_dir`, reads the file, and appends the result to `doc.images`.
+/// No-op when image extraction is disabled in `config`.
+pub(crate) fn resolve_image_uris(doc: &mut InternalDocument, base_dir: &Path, config: &ExtractionConfig) {
+    let image_extraction_enabled = config.images.as_ref().is_some_and(|img| img.extract_images);
+
+    if !image_extraction_enabled {
+        return;
+    }
+
+    let mut image_index = doc.images.len() as u32;
+
+    // Collect URI indices first to avoid borrow conflict (doc.uris vs doc.images).
+    let image_uri_indices: Vec<usize> = doc
+        .uris
+        .iter()
+        .enumerate()
+        .filter(|(_, uri)| uri.kind == UriKind::Image)
+        .map(|(i, _)| i)
+        .collect();
+
+    for idx in image_uri_indices {
+        if let Some(resolved) = resolve_image_path(base_dir, &doc.uris[idx].url)
+            && let Some(img) = read_image_file(&resolved, image_index)
+        {
+            doc.images.push(img);
+            image_index += 1;
+        }
+    }
+}
+
+/// Read a file, extract via `extract_bytes`, then resolve image URIs.
+///
+/// Shared helper for markup extractors (Markdown, LaTeX, RST, Org-mode, Typst,
+/// Djot, DocBook, MDX) that need to resolve relative image paths after extraction.
+pub(crate) async fn extract_file_with_image_resolution(
+    extractor: &(dyn crate::plugins::DocumentExtractor + Sync),
+    path: &Path,
+    mime_type: &str,
+    config: &ExtractionConfig,
+) -> crate::Result<InternalDocument> {
+    let bytes = crate::core::io::open_file_bytes(path)?;
+    let mut doc = extractor.extract_bytes(&bytes, mime_type, config).await?;
+    if let Some(base_dir) = path.parent() {
+        resolve_image_uris(&mut doc, base_dir, config);
+    }
+    Ok(doc)
+}
+
+/// Normalize a path by resolving `.` and `..` components without filesystem access.
+fn normalize_path(path: &Path) -> PathBuf {
+    let mut components = Vec::new();
+    for component in path.components() {
+        match component {
+            std::path::Component::ParentDir => {
+                components.pop();
+            }
+            std::path::Component::CurDir => {}
+            c => components.push(c),
+        }
+    }
+    components.iter().collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::path::Path;
+
+    #[test]
+    fn test_resolve_relative_path() {
+        let base = Path::new("/home/user/docs");
+        let result = resolve_image_path(base, "images/photo.png");
+        assert_eq!(result, Some(PathBuf::from("/home/user/docs/images/photo.png")));
+    }
+
+    #[test]
+    fn test_resolve_nested_relative() {
+        let base = Path::new("/project/content");
+        let result = resolve_image_path(base, "images/subfolder/nested.png");
+        assert_eq!(
+            result,
+            Some(PathBuf::from("/project/content/images/subfolder/nested.png"))
+        );
+    }
+
+    #[test]
+    fn test_reject_absolute_path() {
+        let base = Path::new("/home/user/docs");
+        assert_eq!(resolve_image_path(base, "/etc/passwd"), None);
+    }
+
+    #[test]
+    fn test_reject_traversal() {
+        let base = Path::new("/home/user/docs");
+        assert_eq!(resolve_image_path(base, "../../etc/passwd"), None);
+    }
+
+    #[test]
+    fn test_reject_http_url() {
+        let base = Path::new("/home/user/docs");
+        assert_eq!(resolve_image_path(base, "https://example.com/img.png"), None);
+    }
+
+    #[test]
+    fn test_reject_data_uri() {
+        let base = Path::new("/home/user/docs");
+        assert_eq!(resolve_image_path(base, "data:image/png;base64,abc"), None);
+    }
+
+    #[test]
+    fn test_nonexistent_file_still_resolves() {
+        // resolve_image_path only checks structure, not filesystem
+        let base = Path::new("/nonexistent/base");
+        let result = resolve_image_path(base, "sub/image.jpg");
+        assert_eq!(result, Some(PathBuf::from("/nonexistent/base/sub/image.jpg")));
+    }
+
+    #[test]
+    fn test_path_with_spaces() {
+        let base = Path::new("/home/user/my docs");
+        let result = resolve_image_path(base, "my images/photo.png");
+        assert_eq!(result, Some(PathBuf::from("/home/user/my docs/my images/photo.png")));
+    }
+
+    #[test]
+    fn test_windows_backslash() {
+        // On all platforms, std::path::Path::join handles separators correctly.
+        // On Unix, backslash is a valid filename char so it stays as-is in the component.
+        // The key point: the function does not panic and produces a usable path.
+        let base = Path::new("/home/user/docs");
+        let result = resolve_image_path(base, "images/photo.png");
+        assert!(result.is_some());
+    }
+
+    #[test]
+    fn test_reject_ftp_url() {
+        let base = Path::new("/home/user/docs");
+        assert_eq!(resolve_image_path(base, "ftp://server/img.png"), None);
+    }
+
+    #[test]
+    fn test_reject_mailto() {
+        let base = Path::new("/home/user/docs");
+        assert_eq!(resolve_image_path(base, "mailto:user@example.com"), None);
+    }
+
+    #[test]
+    fn test_file_uri_stripped() {
+        let base = Path::new("/home/user/docs");
+        let result = resolve_image_path(base, "file://images/photo.png");
+        assert_eq!(result, Some(PathBuf::from("/home/user/docs/images/photo.png")));
+    }
+
+    #[test]
+    fn test_reject_windows_absolute() {
+        let base = Path::new("/home/user/docs");
+        assert_eq!(resolve_image_path(base, "C:\\Windows\\img.png"), None);
+    }
+}

crates/kreuzberg/src/core/pipeline/cache.rs

@@ -0,0 +1,46 @@
+//! Processor caching to reduce lock contention.
+//!
+//! This module manages the caching of post-processors by processing stage,
+//! eliminating repeated registry lock acquisitions.
+
+use crate::Result;
+use crate::plugins::{PostProcessor, ProcessingStage};
+use parking_lot::RwLock;
+use std::sync::Arc;
+use std::sync::LazyLock;
+
+/// Cached post-processors for each stage to reduce lock contention.
+///
+/// This cache is populated once during the first pipeline run and reused
+/// for all subsequent extractions, eliminating 3 of 4 registry lock acquisitions
+/// per extraction.
+pub(super) struct ProcessorCache {
+    pub(super) early: Arc<Vec<Arc<dyn PostProcessor>>>,
+    pub(super) middle: Arc<Vec<Arc<dyn PostProcessor>>>,
+    pub(super) late: Arc<Vec<Arc<dyn PostProcessor>>>,
+}
+
+impl ProcessorCache {
+    /// Create a new processor cache by fetching from the registry.
+    pub(super) fn new() -> Result<Self> {
+        let processor_registry = crate::plugins::registry::get_post_processor_registry();
+        let registry = processor_registry.read();
+
+        Ok(Self {
+            early: Arc::new(registry.get_for_stage(ProcessingStage::Early)),
+            middle: Arc::new(registry.get_for_stage(ProcessingStage::Middle)),
+            late: Arc::new(registry.get_for_stage(ProcessingStage::Late)),
+        })
+    }
+}
+
+/// Lazy processor cache - initialized on first use, then cached.
+pub(super) static PROCESSOR_CACHE: LazyLock<RwLock<Option<ProcessorCache>>> = LazyLock::new(|| RwLock::new(None));
+
+/// Clear the processor cache (primarily for testing when registry changes).
+#[cfg_attr(alef, alef(skip))]
+pub fn clear_processor_cache() -> Result<()> {
+    let mut cache = PROCESSOR_CACHE.write();
+    *cache = None;
+    Ok(())
+}

crates/kreuzberg/src/core/pipeline/execution.rs

@@ -0,0 +1,128 @@
+//! Core processor execution logic.
+//!
+//! This module handles the execution of post-processors and validators
+//! in the correct order.
+
+use crate::core::config::ExtractionConfig;
+use crate::plugins::ProcessingStage;
+use crate::types::{ExtractionResult, ProcessingWarning};
+use crate::{KreuzbergError, Result};
+use std::borrow::Cow;
+#[cfg(feature = "otel")]
+use std::time::Instant;
+#[cfg(feature = "otel")]
+use tracing::Instrument;
+
+/// Execute all registered post-processors by stage.
+pub(super) async fn execute_processors(
+    result: &mut ExtractionResult,
+    config: &ExtractionConfig,
+    pp_config: &Option<&crate::core::config::PostProcessorConfig>,
+    early_processors: std::sync::Arc<Vec<std::sync::Arc<dyn crate::plugins::PostProcessor>>>,
+    middle_processors: std::sync::Arc<Vec<std::sync::Arc<dyn crate::plugins::PostProcessor>>>,
+    late_processors: std::sync::Arc<Vec<std::sync::Arc<dyn crate::plugins::PostProcessor>>>,
+) -> Result<()> {
+    for (_stage, processors_arc) in [
+        (ProcessingStage::Early, early_processors),
+        (ProcessingStage::Middle, middle_processors),
+        (ProcessingStage::Late, late_processors),
+    ] {
+        #[cfg(feature = "otel")]
+        let stage_name = match _stage {
+            ProcessingStage::Early => crate::telemetry::conventions::stages::POST_PROCESSING_EARLY,
+            ProcessingStage::Middle => crate::telemetry::conventions::stages::POST_PROCESSING_MIDDLE,
+            ProcessingStage::Late => crate::telemetry::conventions::stages::POST_PROCESSING_LATE,
+        };
+        #[cfg(feature = "otel")]
+        let stage_span = crate::telemetry::spans::pipeline_stage_span(stage_name);
+        #[cfg(feature = "otel")]
+        let stage_start = Instant::now();
+        #[cfg(feature = "otel")]
+        let _stage_guard = stage_span.enter();
+
+        for processor in processors_arc.iter() {
+            let processor_name = processor.name();
+
+            let should_run = should_processor_run(pp_config, processor_name);
+
+            if should_run && processor.should_process(result, config) {
+                #[cfg(feature = "otel")]
+                let processor_span = crate::telemetry::spans::pipeline_processor_span(stage_name, processor_name);
+
+                #[cfg(feature = "otel")]
+                let process_result = processor.process(result, config).instrument(processor_span).await;
+                #[cfg(not(feature = "otel"))]
+                let process_result = processor.process(result, config).await;
+
+                match process_result {
+                    Ok(_) => {}
+                    Err(err @ KreuzbergError::Io(_))
+                    | Err(err @ KreuzbergError::LockPoisoned(_))
+                    | Err(err @ KreuzbergError::Plugin { .. }) => {
+                        return Err(err);
+                    }
+                    Err(err) => {
+                        let error_msg = err.to_string();
+                        result.processing_warnings.push(ProcessingWarning {
+                            source: Cow::Owned(processor_name.to_string()),
+                            message: Cow::Owned(error_msg),
+                        });
+                    }
+                }
+            }
+        }
+
+        #[cfg(feature = "otel")]
+        {
+            let stage_ms = stage_start.elapsed().as_secs_f64() * 1000.0;
+            crate::telemetry::metrics::get_metrics().pipeline_duration_ms.record(
+                stage_ms,
+                &[opentelemetry::KeyValue::new(
+                    crate::telemetry::conventions::PIPELINE_STAGE,
+                    stage_name.to_string(),
+                )],
+            );
+            drop(_stage_guard);
+            drop(stage_span);
+        }
+    }
+    Ok(())
+}
+
+/// Determine if a processor should run based on configuration.
+fn should_processor_run(pp_config: &Option<&crate::core::config::PostProcessorConfig>, processor_name: &str) -> bool {
+    if let Some(config) = pp_config {
+        if let Some(ref enabled_set) = config.enabled_set {
+            enabled_set.contains(processor_name)
+        } else if let Some(ref disabled_set) = config.disabled_set {
+            !disabled_set.contains(processor_name)
+        } else if let Some(ref enabled) = config.enabled_processors {
+            enabled.iter().any(|name| name == processor_name)
+        } else if let Some(ref disabled) = config.disabled_processors {
+            !disabled.iter().any(|name| name == processor_name)
+        } else {
+            true
+        }
+    } else {
+        true
+    }
+}
+
+/// Execute all registered validators.
+pub(super) async fn execute_validators(result: &ExtractionResult, config: &ExtractionConfig) -> Result<()> {
+    let validator_registry = crate::plugins::registry::get_validator_registry();
+    let validators = {
+        let registry = validator_registry.read();
+        registry.get_all()
+    };
+
+    if !validators.is_empty() {
+        for validator in validators {
+            if validator.should_validate(result, config) {
+                validator.validate(result, config).await?;
+            }
+        }
+    }
+
+    Ok(())
+}

crates/kreuzberg/src/core/pipeline/features.rs

@@ -0,0 +1,461 @@
+//! Feature processing logic.
+//!
+//! This module handles feature-specific processing like chunking,
+//! embedding generation, and language detection.
+
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+#[cfg(feature = "chunking")]
+use crate::types::PageBoundary;
+use crate::types::{ExtractionResult, ProcessingWarning};
+use std::borrow::Cow;
+
+/// Recompute page boundaries against the rendered `content` string.
+///
+/// `PageBoundary` offsets produced during extraction are computed against raw
+/// rendered/source text, but `result.content` is produced by `render_plain` which
+/// trims trailing whitespace from each paragraph.  The raw page text therefore has
+/// different byte lengths for pages that contain trailing-space artifacts from PDF
+/// rendering.  This function re-derives the boundaries by locating each page's
+/// **paragraph-normalised** content (each `"\n\n"`-separated segment trimmed, then
+/// re-joined) inside the combined `content` string, so that the byte offsets passed
+/// to the chunker are valid indices into `result.content`.
+///
+/// Pages whose content cannot be found are silently skipped (the chunker will
+/// still produce output, just without page-range metadata for those pages).
+#[cfg(feature = "chunking")]
+fn recompute_boundaries_from_pages(content: &str, pages: &[crate::types::PageContent]) -> Vec<PageBoundary> {
+    let mut boundaries = Vec::with_capacity(pages.len());
+    let mut search_offset = 0usize;
+
+    for page in pages {
+        if page.content.trim().is_empty() {
+            boundaries.push(PageBoundary {
+                page_number: page.page_number,
+                byte_start: search_offset,
+                byte_end: search_offset,
+            });
+            continue;
+        }
+
+        // Normalise page content to match what render_plain produces: split on the
+        // paragraph separator, trim each segment (PDF pages often carry trailing
+        // spaces before "\n\n" that render_plain strips via paragraph.trim()), then
+        // re-join.  Using the normalised form means exact-match succeeds and the
+        // resulting byte_end is correct — avoiding cascading search_offset
+        // over-advance that would push past subsequent pages.
+        let normalized: String = page
+            .content
+            .split("\n\n")
+            .map(str::trim)
+            .filter(|s| !s.is_empty())
+            .collect::<Vec<_>>()
+            .join("\n\n");
+
+        // Try normalised-exact match (primary path — handles trailing-space pages).
+        if let Some(pos) = content[search_offset..].find(normalized.as_str()) {
+            let byte_start = search_offset + pos;
+            let byte_end = content.floor_char_boundary(byte_start + normalized.len());
+            boundaries.push(PageBoundary {
+                page_number: page.page_number,
+                byte_start,
+                byte_end,
+            });
+            search_offset = byte_end;
+            continue;
+        }
+
+        // Fallback: search for first non-empty line of page content.
+        // Use normalized.len() for byte_end so search_offset advances correctly.
+        if let Some(line) = page.content.lines().find(|l| !l.trim().is_empty()).map(|l| l.trim())
+            && let Some(pos) = content[search_offset..].find(line)
+        {
+            let byte_start = search_offset + pos;
+            let raw_end = (byte_start + normalized.len()).min(content.len());
+            let byte_end = content.floor_char_boundary(raw_end);
+            boundaries.push(PageBoundary {
+                page_number: page.page_number,
+                byte_start,
+                byte_end,
+            });
+            search_offset = byte_end;
+            continue;
+        }
+
+        // Last resort: skip this page
+        tracing::debug!(
+            page = page.page_number,
+            "Could not locate page content in rendered text — skipping page boundary"
+        );
+    }
+
+    boundaries
+}
+
+/// Map TSLP `CodeChunk`s directly to kreuzberg `Chunk`s, bypassing text-splitter.
+///
+/// When the extraction result contains code intelligence with non-empty chunks,
+/// those chunks already represent semantically meaningful code boundaries produced
+/// by tree-sitter. Using text-splitter would break these boundaries.
+#[cfg(feature = "tree-sitter")]
+fn try_code_chunks(result: &ExtractionResult) -> Option<Vec<crate::types::extraction::Chunk>> {
+    use crate::types::extraction::{Chunk, ChunkMetadata, ChunkType, HeadingContext, HeadingLevel};
+
+    let code_chunks = match &result.metadata.format {
+        Some(crate::types::metadata::FormatMetadata::Code(pr)) if !pr.chunks.is_empty() => &pr.chunks,
+        _ => return None,
+    };
+
+    let total_chunks = code_chunks.len();
+    let chunks: Vec<Chunk> = code_chunks
+        .iter()
+        .enumerate()
+        .map(|(i, cc)| {
+            // All code chunks are classified as CodeBlock regardless of node type.
+            let chunk_type = ChunkType::CodeBlock;
+
+            // Build heading context from context_path.
+            let heading_context = if cc.metadata.context_path.is_empty() {
+                None
+            } else {
+                Some(HeadingContext {
+                    headings: cc
+                        .metadata
+                        .context_path
+                        .iter()
+                        .enumerate()
+                        .map(|(depth, name)| HeadingLevel {
+                            level: (depth as u8).saturating_add(2).min(6),
+                            text: name.clone(),
+                        })
+                        .collect(),
+                })
+            };
+
+            Chunk {
+                content: cc.content.clone(),
+                chunk_type,
+                embedding: None,
+                metadata: ChunkMetadata {
+                    byte_start: cc.start_byte,
+                    byte_end: cc.end_byte,
+                    token_count: None,
+                    chunk_index: i,
+                    total_chunks,
+                    first_page: None,
+                    last_page: None,
+                    heading_context,
+                    image_indices: Vec::new(),
+                },
+            }
+        })
+        .collect();
+
+    Some(chunks)
+}
+
+/// Execute chunking if configured.
+pub(super) fn execute_chunking(result: &mut ExtractionResult, config: &ExtractionConfig) -> Result<()> {
+    #[cfg(feature = "chunking")]
+    if let Some(ref chunking_config) = config.chunking {
+        // For code extractions with TSLP chunks, bypass text-splitter and map directly.
+        #[cfg(feature = "tree-sitter")]
+        if let Some(code_chunks) = try_code_chunks(result) {
+            result.chunks = Some(code_chunks);
+
+            let resolved_config = chunking_config.resolve_preset();
+            #[cfg(feature = "embeddings")]
+            if let Some(ref embedding_config) = resolved_config.embedding
+                && let Some(ref mut chunks) = result.chunks
+                && let Err(e) = crate::embeddings::generate_embeddings_for_chunks(chunks, embedding_config)
+            {
+                tracing::warn!("Embedding generation failed: {e}. Check that ONNX Runtime is installed.");
+                result.processing_warnings.push(ProcessingWarning {
+                    source: Cow::Borrowed("embedding"),
+                    message: Cow::Owned(e.to_string()),
+                });
+            }
+
+            #[cfg(not(feature = "embeddings"))]
+            if resolved_config.embedding.is_some() {
+                tracing::warn!(
+                    "Embedding config provided but embeddings feature is not enabled. Recompile with --features embeddings."
+                );
+                result.processing_warnings.push(ProcessingWarning {
+                    source: Cow::Borrowed("embedding"),
+                    message: Cow::Borrowed("Embeddings feature not enabled"),
+                });
+            }
+
+            return Ok(());
+        }
+
+        let resolved_config = chunking_config.resolve_preset();
+        let chunking_config = &resolved_config;
+
+        // Recompute page boundaries against `result.content` (rendered by `render_plain`)
+        // if per-page content is available.  The boundaries stored in
+        // `result.metadata.pages.boundaries` were computed against the raw extractor text
+        // and may have different byte offsets than the rendered content.
+        let recomputed_boundaries: Option<Vec<PageBoundary>> = result
+            .pages
+            .as_deref()
+            .map(|pages| recompute_boundaries_from_pages(&result.content, pages));
+
+        let page_boundaries: Option<&[PageBoundary]> = recomputed_boundaries
+            .as_deref()
+            .filter(|s| !s.is_empty())
+            .or_else(|| result.metadata.pages.as_ref().and_then(|ps| ps.boundaries.as_deref()));
+
+        // Pass formatted_content (markdown) for heading context resolution when available.
+        // Plain-text rendering strips heading markers, but the markdown chunker needs them
+        // to build the heading hierarchy for chunk metadata.
+        let heading_source = result.formatted_content.as_deref();
+        match crate::chunking::chunk_text_with_heading_source(
+            &result.content,
+            chunking_config,
+            page_boundaries,
+            heading_source,
+        ) {
+            Ok(chunking_result) => {
+                result.chunks = Some(chunking_result.chunks);
+
+                // Populate image_indices on each chunk: collect indices of images whose
+                // page_number falls within the chunk's [first_page, last_page] range.
+                if let Some(ref images) = result.images
+                    && let Some(ref mut chunks) = result.chunks
+                {
+                    for chunk in chunks.iter_mut() {
+                        if let (Some(first), Some(last)) = (chunk.metadata.first_page, chunk.metadata.last_page) {
+                            chunk.metadata.image_indices = images
+                                .iter()
+                                .enumerate()
+                                .filter_map(|(idx, img)| {
+                                    let pg = img.page_number?;
+                                    (pg >= first && pg <= last).then_some(idx as u32)
+                                })
+                                .collect();
+                        }
+                    }
+                }
+
+                #[cfg(feature = "embeddings")]
+                if let Some(ref embedding_config) = chunking_config.embedding
+                    && let Some(ref mut chunks) = result.chunks
+                    && let Err(e) = crate::embeddings::generate_embeddings_for_chunks(chunks, embedding_config)
+                {
+                    tracing::warn!("Embedding generation failed: {e}. Check that ONNX Runtime is installed.");
+                    result.processing_warnings.push(ProcessingWarning {
+                        source: Cow::Borrowed("embedding"),
+                        message: Cow::Owned(e.to_string()),
+                    });
+                }
+
+                #[cfg(not(feature = "embeddings"))]
+                if chunking_config.embedding.is_some() {
+                    tracing::warn!(
+                        "Embedding config provided but embeddings feature is not enabled. Recompile with --features embeddings."
+                    );
+                    result.processing_warnings.push(ProcessingWarning {
+                        source: Cow::Borrowed("embedding"),
+                        message: Cow::Borrowed("Embeddings feature not enabled"),
+                    });
+                }
+            }
+            Err(e) => {
+                result.processing_warnings.push(ProcessingWarning {
+                    source: Cow::Borrowed("chunking"),
+                    message: Cow::Owned(e.to_string()),
+                });
+            }
+        }
+    }
+
+    #[cfg(not(feature = "chunking"))]
+    if config.chunking.is_some() {
+        result.processing_warnings.push(ProcessingWarning {
+            source: Cow::Borrowed("chunking"),
+            message: Cow::Borrowed("Chunking feature not enabled"),
+        });
+    }
+
+    Ok(())
+}
+
+/// Execute language detection if configured.
+pub(super) fn execute_language_detection(result: &mut ExtractionResult, config: &ExtractionConfig) -> Result<()> {
+    #[cfg(feature = "language-detection")]
+    if let Some(ref lang_config) = config.language_detection {
+        match crate::language_detection::detect_languages(&result.content, lang_config) {
+            Ok(detected) => {
+                result.detected_languages = detected;
+            }
+            Err(e) => {
+                result.processing_warnings.push(ProcessingWarning {
+                    source: Cow::Borrowed("language_detection"),
+                    message: Cow::Owned(e.to_string()),
+                });
+            }
+        }
+    }
+
+    #[cfg(not(feature = "language-detection"))]
+    if config.language_detection.is_some() {
+        result.processing_warnings.push(ProcessingWarning {
+            source: Cow::Borrowed("language_detection"),
+            message: Cow::Borrowed("Language detection feature not enabled"),
+        });
+    }
+
+    Ok(())
+}
+
+/// Execute token reduction if configured.
+pub(super) fn execute_token_reduction(result: &mut ExtractionResult, config: &ExtractionConfig) -> Result<()> {
+    #[cfg(feature = "quality")]
+    if let Some(ref tr_config) = config.token_reduction {
+        let level = crate::text::token_reduction::ReductionLevel::from(tr_config.mode.as_str());
+
+        if !matches!(level, crate::text::token_reduction::ReductionLevel::Off) {
+            let impl_config = crate::text::token_reduction::TokenReductionConfig {
+                level,
+                ..Default::default()
+            };
+
+            let lang_hint: Option<&str> = result
+                .detected_languages
+                .as_deref()
+                .and_then(|langs| langs.first().map(|s| s.as_str()));
+
+            match crate::text::token_reduction::reduce_tokens(&result.content, &impl_config, lang_hint) {
+                Ok(reduced) => {
+                    result.content = reduced;
+                }
+                Err(e) => {
+                    result.processing_warnings.push(ProcessingWarning {
+                        source: Cow::Borrowed("token_reduction"),
+                        message: Cow::Owned(e.to_string()),
+                    });
+                }
+            }
+        }
+    }
+
+    #[cfg(not(feature = "quality"))]
+    if config.token_reduction.is_some() {
+        result.processing_warnings.push(ProcessingWarning {
+            source: Cow::Borrowed("token_reduction"),
+            message: Cow::Borrowed("Token reduction requires the quality feature"),
+        });
+    }
+
+    Ok(())
+}
+
+#[cfg(all(test, feature = "chunking"))]
+mod tests {
+    use super::*;
+    use crate::types::PageContent;
+
+    fn make_page(page_number: u32, content: impl Into<String>) -> PageContent {
+        PageContent {
+            page_number,
+            content: content.into(),
+            tables: vec![],
+            image_indices: vec![],
+            hierarchy: None,
+            is_blank: None,
+            layout_regions: None,
+            section_name: None,
+            speaker_notes: None,
+            sheet_name: None,
+        }
+    }
+
+    // When PageContent.content matches result.content exactly, all boundaries succeed.
+    #[test]
+    fn recompute_boundaries_exact_match_produces_full_boundary_set() {
+        let p1 = "Hello world";
+        let p2 = "Second page text";
+        let p3 = "Third page here";
+        let content = format!("{p1}\n\n{p2}\n\n{p3}");
+
+        let pages = vec![make_page(1, p1), make_page(2, p2), make_page(3, p3)];
+        let boundaries = recompute_boundaries_from_pages(&content, &pages);
+
+        assert_eq!(boundaries.len(), 3, "all pages should resolve to boundaries");
+        assert_eq!(&content[boundaries[0].byte_start..boundaries[0].byte_end], p1);
+        assert_eq!(&content[boundaries[1].byte_start..boundaries[1].byte_end], p2);
+        assert_eq!(&content[boundaries[2].byte_start..boundaries[2].byte_end], p3);
+    }
+
+    // When PageContent.content is raw (control char present) but result.content has the
+    // cleaned version, the affected page is silently skipped — leaving fewer boundaries
+    // than pages. Documents the pre-fix failure mode.
+    #[test]
+    fn recompute_boundaries_raw_content_causes_skipped_pages() {
+        // U+0001 between word chars → fix_pdf_control_chars replaces with '-'
+        let p1_clean = "Hello world";
+        let p2_raw = "ab\x01cd"; // raw page text — control char present
+        let p2_clean = "ab-cd"; // what result.content contains after cleanup
+        let p3_clean = "Third page";
+        let content = format!("{p1_clean}\n\n{p2_clean}\n\n{p3_clean}");
+
+        // Pre-fix scenario: page.content = raw, result.content = cleaned → mismatch
+        let pages = vec![
+            make_page(1, p1_clean),
+            make_page(2, p2_raw), // intentionally stale raw content
+            make_page(3, p3_clean),
+        ];
+        let boundaries = recompute_boundaries_from_pages(&content, &pages);
+
+        // Page 2 is skipped: neither exact nor first-line search finds "ab\x01cd"
+        // inside content (which has "ab-cd"). Only pages 1 and 3 resolve.
+        assert_eq!(boundaries.len(), 2, "page with raw/cleaned mismatch should be skipped");
+        assert_eq!(boundaries[0].page_number, 1);
+        assert_eq!(boundaries[1].page_number, 3);
+    }
+
+    // When PageContent.content is the cleaned text (the fix), all pages resolve.
+    #[test]
+    fn recompute_boundaries_cleaned_content_resolves_all_pages() {
+        let p1_clean = "Hello world";
+        let p2_clean = "ab-cd"; // cleaned — matches result.content exactly
+        let p3_clean = "Third page";
+        let content = format!("{p1_clean}\n\n{p2_clean}\n\n{p3_clean}");
+
+        // Post-fix scenario: page.content = cleaned, result.content = cleaned → exact match
+        let pages = vec![make_page(1, p1_clean), make_page(2, p2_clean), make_page(3, p3_clean)];
+        let boundaries = recompute_boundaries_from_pages(&content, &pages);
+
+        assert_eq!(boundaries.len(), 3, "all pages should resolve after fix");
+        assert_eq!(&content[boundaries[1].byte_start..boundaries[1].byte_end], p2_clean);
+    }
+
+    // PDF pages often have trailing spaces before "\n\n" paragraph separators (PDF
+    // rendering artifact).  render_plain trims each paragraph via paragraph.trim(),
+    // so result.content lacks those trailing spaces while page.content retains them.
+    // The normalised-exact match must succeed and produce correct byte_end so that
+    // subsequent pages are found without cascading search_offset over-advance.
+    #[test]
+    fn recompute_boundaries_trailing_space_pages_all_resolve() {
+        // Simulate PDF page content with trailing spaces before "\n\n".
+        let p1_raw = "Heading \n\nBody paragraph one. ";
+        let p2_raw = "Second heading \n\nBody paragraph two. ";
+        let p3_raw = "Conclusion. ";
+
+        // result.content as render_plain produces it (each paragraph trimmed).
+        let p1_norm = "Heading\n\nBody paragraph one.";
+        let p2_norm = "Second heading\n\nBody paragraph two.";
+        let p3_norm = "Conclusion.";
+        let content = format!("{p1_norm}\n\n{p2_norm}\n\n{p3_norm}");
+
+        let pages = vec![make_page(1, p1_raw), make_page(2, p2_raw), make_page(3, p3_raw)];
+        let boundaries = recompute_boundaries_from_pages(&content, &pages);
+
+        assert_eq!(boundaries.len(), 3, "all pages must resolve despite trailing spaces");
+        assert_eq!(&content[boundaries[0].byte_start..boundaries[0].byte_end], p1_norm);
+        assert_eq!(&content[boundaries[1].byte_start..boundaries[1].byte_end], p2_norm);
+        assert_eq!(&content[boundaries[2].byte_start..boundaries[2].byte_end], p3_norm);
+    }
+}

crates/kreuzberg/src/core/pipeline/format.rs

@@ -0,0 +1,207 @@
+//! Output format conversion for extraction results.
+//!
+//! This module handles the final step of output format application: swapping
+//! pre-rendered content into the result and recording format metadata.
+//!
+//! The heavy rendering work (Markdown, Djot, HTML) is now done earlier in the
+//! pipeline inside `derive_extraction_result`, which populates
+//! `ExtractionResult::formatted_content`. This function simply swaps that
+//! pre-rendered content into the `content` field after post-processors have
+//! operated on the plain-text version.
+
+use crate::core::config::OutputFormat;
+use crate::types::ExtractionResult;
+#[cfg(test)]
+use std::borrow::Cow;
+
+/// Apply output format conversion to the extraction result.
+///
+/// Records the output format in metadata and swaps in pre-rendered content
+/// (produced during `derive_extraction_result`) if available.
+///
+/// This runs as the final pipeline step, after post-processors have operated
+/// on the plain-text `content` field.
+///
+/// # Arguments
+///
+/// * `result` - The extraction result to modify
+/// * `output_format` - The desired output format
+#[cfg_attr(alef, alef(skip))]
+pub fn apply_output_format(result: ExtractionResult, output_format: OutputFormat) -> ExtractionResult {
+    let mut result = result;
+    let format_name = match output_format {
+        OutputFormat::Plain => "plain",
+        OutputFormat::Markdown => "markdown",
+        OutputFormat::Djot => "djot",
+        OutputFormat::Html => "html",
+        OutputFormat::Json => "json",
+        OutputFormat::Structured => "structured",
+        OutputFormat::Custom(ref name) => name.as_str(),
+    };
+    result.metadata.output_format = Some(format_name.to_string());
+
+    // Swap in pre-rendered content if available (populated by derive_extraction_result).
+    if let Some(formatted) = result.formatted_content.take() {
+        result.content = formatted;
+    }
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::types::Metadata;
+
+    #[test]
+    fn test_apply_output_format_plain() {
+        let result = ExtractionResult {
+            content: "Hello World".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
+            ..Default::default()
+        };
+
+        let result = apply_output_format(result, OutputFormat::Plain);
+
+        assert_eq!(result.content, "Hello World");
+        assert_eq!(result.metadata.output_format, Some("plain".to_string()));
+    }
+
+    #[test]
+    fn test_apply_output_format_markdown_no_prerender() {
+        let result = ExtractionResult {
+            content: "Hello World".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
+            ..Default::default()
+        };
+
+        let result = apply_output_format(result, OutputFormat::Markdown);
+
+        // Without pre-rendered content, content stays as-is
+        assert_eq!(result.content, "Hello World");
+        assert_eq!(result.metadata.output_format, Some("markdown".to_string()));
+    }
+
+    #[test]
+    fn test_apply_output_format_swaps_formatted_content() {
+        let result = ExtractionResult {
+            content: "plain text".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
+            formatted_content: Some("# Heading\n\nFormatted markdown".to_string()),
+            ..Default::default()
+        };
+
+        let result = apply_output_format(result, OutputFormat::Markdown);
+
+        assert_eq!(result.content, "# Heading\n\nFormatted markdown");
+        assert!(result.formatted_content.is_none(), "formatted_content should be taken");
+        assert_eq!(result.metadata.output_format, Some("markdown".to_string()));
+    }
+
+    #[test]
+    fn test_apply_output_format_html_with_prerender() {
+        let result = ExtractionResult {
+            content: "plain text".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
+            formatted_content: Some("<p>Hello World</p>".to_string()),
+            ..Default::default()
+        };
+
+        let result = apply_output_format(result, OutputFormat::Html);
+
+        assert_eq!(result.content, "<p>Hello World</p>");
+        assert_eq!(result.metadata.output_format, Some("html".to_string()));
+    }
+
+    #[test]
+    fn test_apply_output_format_djot_with_prerender() {
+        let result = ExtractionResult {
+            content: "plain text".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
+            formatted_content: Some("# Djot heading".to_string()),
+            ..Default::default()
+        };
+
+        let result = apply_output_format(result, OutputFormat::Djot);
+
+        assert_eq!(result.content, "# Djot heading");
+        assert_eq!(result.metadata.output_format, Some("djot".to_string()));
+    }
+
+    #[test]
+    fn test_apply_output_format_structured() {
+        let result = ExtractionResult {
+            content: "Hello World".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
+            ..Default::default()
+        };
+
+        let result = apply_output_format(result, OutputFormat::Structured);
+
+        assert_eq!(result.content, "Hello World");
+        assert_eq!(result.metadata.output_format, Some("structured".to_string()));
+    }
+
+    #[test]
+    fn test_apply_output_format_preserves_metadata() {
+        use ahash::AHashMap;
+        let mut additional = AHashMap::new();
+        additional.insert(Cow::Borrowed("custom_key"), serde_json::json!("custom_value"));
+        let metadata = Metadata {
+            title: Some("Test Title".to_string()),
+            additional,
+            ..Default::default()
+        };
+
+        let result = ExtractionResult {
+            content: "Hello World".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
+            metadata,
+            ..Default::default()
+        };
+
+        let result = apply_output_format(result, OutputFormat::Markdown);
+
+        assert_eq!(result.metadata.title, Some("Test Title".to_string()));
+        assert_eq!(
+            result.metadata.additional.get("custom_key"),
+            Some(&serde_json::json!("custom_value"))
+        );
+    }
+
+    #[test]
+    fn test_apply_output_format_preserves_tables() {
+        use crate::types::Table;
+
+        let table = Table {
+            cells: vec![vec!["A".to_string(), "B".to_string()]],
+            markdown: "| A | B |".to_string(),
+            page_number: 1,
+            bounding_box: None,
+        };
+
+        let result = ExtractionResult {
+            content: "Hello World".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
+            tables: vec![table],
+            ..Default::default()
+        };
+
+        let result = apply_output_format(result, OutputFormat::Html);
+
+        assert_eq!(result.tables.len(), 1);
+        assert_eq!(result.tables[0].cells[0][0], "A");
+    }
+
+    #[test]
+    fn test_apply_output_format_sets_typed_field() {
+        let result = ExtractionResult {
+            content: "test".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
+            ..Default::default()
+        };
+
+        let result = apply_output_format(result, OutputFormat::Djot);
+
+        assert_eq!(result.metadata.output_format, Some("djot".to_string()));
+    }
+}

crates/kreuzberg/src/core/pipeline/initialization.rs

@@ -0,0 +1,67 @@
+//! Pipeline initialization and setup logic.
+//!
+//! This module handles the initialization of features and processor cache
+//! required for pipeline execution.
+
+use crate::Result;
+#[cfg(feature = "quality")]
+use std::sync::OnceLock;
+
+use super::cache::{PROCESSOR_CACHE, ProcessorCache};
+
+/// Type alias for processor stages tuple (Early, Middle, Late).
+type ProcessorStages = (
+    std::sync::Arc<Vec<std::sync::Arc<dyn crate::plugins::PostProcessor>>>,
+    std::sync::Arc<Vec<std::sync::Arc<dyn crate::plugins::PostProcessor>>>,
+    std::sync::Arc<Vec<std::sync::Arc<dyn crate::plugins::PostProcessor>>>,
+);
+
+/// Initialize feature-specific systems that may be needed during pipeline execution.
+pub(super) fn initialize_features() {
+    #[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
+    {
+        let _ = crate::keywords::ensure_initialized();
+    }
+
+    #[cfg(feature = "language-detection")]
+    {
+        let _ = crate::language_detection::ensure_initialized();
+    }
+
+    #[cfg(feature = "chunking")]
+    {
+        let _ = crate::chunking::ensure_initialized();
+    }
+
+    #[cfg(feature = "quality")]
+    {
+        static QUALITY_INIT: OnceLock<()> = OnceLock::new();
+        QUALITY_INIT.get_or_init(|| {
+            let registry = crate::plugins::registry::get_post_processor_registry();
+            let mut reg = registry.write();
+            let _ = reg.register(std::sync::Arc::new(crate::text::QualityProcessor));
+        });
+    }
+}
+
+/// Initialize the processor cache if not already initialized.
+pub(super) fn initialize_processor_cache() -> Result<()> {
+    let mut cache_lock = PROCESSOR_CACHE.write();
+    if cache_lock.is_none() {
+        *cache_lock = Some(ProcessorCache::new()?);
+    }
+    Ok(())
+}
+
+/// Get processors from the cache, organized by stage.
+pub(super) fn get_processors_from_cache() -> Result<ProcessorStages> {
+    let cache_lock = PROCESSOR_CACHE.read();
+    let cache = cache_lock
+        .as_ref()
+        .ok_or_else(|| crate::KreuzbergError::Other("Processor cache not initialized".to_string()))?;
+    Ok((
+        std::sync::Arc::clone(&cache.early),
+        std::sync::Arc::clone(&cache.middle),
+        std::sync::Arc::clone(&cache.late),
+    ))
+}

crates/kreuzberg/src/core/pipeline/mod.rs

@@ -0,0 +1,478 @@
+//! Post-processing pipeline orchestration.
+//!
+//! This module orchestrates the post-processing pipeline, executing validators,
+//! quality processing, chunking, and custom hooks in the correct order.
+
+mod cache;
+mod execution;
+mod features;
+mod format;
+mod initialization;
+
+#[cfg(test)]
+mod tests;
+
+pub use cache::clear_processor_cache;
+pub use format::apply_output_format;
+
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::types::ExtractionResult;
+use crate::types::internal::InternalDocument;
+
+use execution::{execute_processors, execute_validators};
+use features::{execute_chunking, execute_language_detection, execute_token_reduction};
+use initialization::{get_processors_from_cache, initialize_features, initialize_processor_cache};
+
+/// Run the post-processing pipeline on an `InternalDocument`.
+///
+/// Derives `ExtractionResult` from `InternalDocument` via the derivation pipeline,
+/// then executes post-processing in the following order:
+/// 1. Post-Processors - Execute by stage (Early, Middle, Late) to modify/enhance the result
+/// 2. Quality Processing - Text cleaning and quality scoring
+/// 3. Chunking - Text splitting if enabled
+/// 4. Validators - Run validation hooks on the processed result (can fail fast)
+///
+/// # Arguments
+///
+/// * `doc` - The internal document produced by the extractor
+/// * `config` - Extraction configuration
+///
+/// # Returns
+///
+/// The processed extraction result.
+///
+/// # Errors
+///
+/// - Validator errors bubble up immediately
+/// - Post-processor errors are caught and recorded in metadata
+/// - System errors (IO, RuntimeError equivalents) always bubble up
+#[cfg_attr(feature = "otel", tracing::instrument(
+    skip(doc, config),
+    fields(
+        pipeline.stage = "post_processing",
+        content.element_count = doc.elements.len(),
+    )
+))]
+#[cfg_attr(alef, alef(skip))]
+pub async fn run_pipeline(mut doc: InternalDocument, config: &ExtractionConfig) -> Result<ExtractionResult> {
+    // Propagate rendering preferences from config into the document.
+    doc.ocr_text_only = config.images.as_ref().map(|i| i.ocr_text_only).unwrap_or(false);
+    doc.append_ocr_text = config.images.as_ref().map(|i| i.append_ocr_text).unwrap_or(false);
+
+    // 1. Process extracted images with OCR if configured
+    #[cfg(all(feature = "ocr", feature = "tokio-runtime"))]
+    let image_ocr_enabled = config.images.as_ref().map(|i| i.run_ocr_on_images).unwrap_or(true);
+    #[cfg(all(feature = "ocr", feature = "tokio-runtime"))]
+    if image_ocr_enabled && config.ocr.is_some() && !doc.images.is_empty() {
+        let images_to_process = std::mem::take(&mut doc.images);
+        match crate::extraction::image_ocr::process_images_with_ocr(
+            images_to_process,
+            config,
+            &mut doc.processing_warnings,
+        )
+        .await
+        {
+            Ok(processed) => {
+                doc.images = processed;
+            }
+            Err(e) => {
+                doc.processing_warnings.push(crate::types::ProcessingWarning {
+                    source: std::borrow::Cow::Borrowed("image_ocr"),
+                    message: std::borrow::Cow::Owned(format!("Image OCR failed: {e}")),
+                });
+            }
+        }
+    }
+
+    replace_embedded_image_markdown_with_ocr(&mut doc);
+    append_embedded_image_ocr_text(&mut doc);
+
+    // Pre-render markdown for the chunker's heading context resolution when:
+    // - Markdown chunking is configured
+    // - Output format is not already Markdown (which would produce formatted_content anyway)
+    // Plain-text rendering strips heading markers, so the markdown chunker needs
+    // a separate markdown rendering to build the heading hierarchy for chunk metadata.
+    #[cfg(feature = "chunking")]
+    let chunker_heading_source = {
+        let needs_markdown = config.chunking.as_ref().is_some_and(|c| {
+            c.chunker_type == crate::core::config::ChunkerType::Markdown
+                || c.resolve_preset().chunker_type == crate::core::config::ChunkerType::Markdown
+        }) && config.output_format == crate::core::config::OutputFormat::Plain;
+        if needs_markdown {
+            Some(crate::rendering::render_markdown(&doc))
+        } else {
+            None
+        }
+    };
+
+    // Pre-render styled HTML before `doc` is consumed by `derive_extraction_result`.
+    // When `html` is active and the caller has configured `html_output`, we
+    // render the document here and inject the result after derivation.
+    #[cfg(feature = "html")]
+    let styled_html_prerender: Option<String> = {
+        use crate::plugins::Renderer as _;
+        if config.output_format == crate::core::config::OutputFormat::Html {
+            config.html_output.as_ref().and_then(|html_cfg| {
+                match crate::rendering::StyledHtmlRenderer::new(html_cfg.clone()) {
+                    Ok(renderer) => match renderer.render(&doc) {
+                        Ok(html) => Some(html),
+                        Err(e) => {
+                            tracing::warn!("StyledHtmlRenderer render failed, falling back to default HTML: {e}");
+                            None
+                        }
+                    },
+                    Err(e) => {
+                        tracing::warn!("StyledHtmlRenderer construction failed, falling back to default HTML: {e}");
+                        None
+                    }
+                }
+            })
+        } else {
+            None
+        }
+    };
+
+    // 2. Derive ExtractionResult from InternalDocument
+    let include_structure = config.include_document_structure;
+    let mut result =
+        crate::extraction::derive::derive_extraction_result(doc, include_structure, config.output_format.clone());
+
+    // Inject pre-rendered styled HTML (overrides the default render_html output).
+    #[cfg(feature = "html")]
+    if let Some(html) = styled_html_prerender {
+        result.formatted_content = Some(html);
+    }
+
+    // Temporarily store pre-rendered markdown for chunker heading context.
+    // Tracked separately so we can remove it after chunking — apply_output_format
+    // must not swap this into result.content when output_format is Plain.
+    #[cfg(feature = "chunking")]
+    let chunker_only_markdown = result.formatted_content.is_none();
+    #[cfg(feature = "chunking")]
+    if chunker_only_markdown && let Some(md) = chunker_heading_source {
+        result.formatted_content = Some(md);
+    }
+
+    // 2. Run post-processing pipeline
+    let pp_config = config.postprocessor.as_ref();
+    let postprocessing_enabled = pp_config.is_none_or(|c| c.enabled);
+
+    if postprocessing_enabled {
+        initialize_features();
+        initialize_processor_cache()?;
+
+        let (early_processors, middle_processors, late_processors) = get_processors_from_cache()?;
+
+        execute_processors(
+            &mut result,
+            config,
+            &pp_config,
+            early_processors,
+            middle_processors,
+            late_processors,
+        )
+        .await?;
+    }
+
+    execute_chunking(&mut result, config)?;
+
+    // Clear temporary markdown if it was only stored for chunker heading context.
+    // This prevents apply_output_format from swapping it into result.content.
+    #[cfg(feature = "chunking")]
+    if chunker_only_markdown {
+        result.formatted_content = None;
+    }
+
+    execute_language_detection(&mut result, config)?;
+    execute_token_reduction(&mut result, config)?;
+    execute_validators(&result, config).await?;
+
+    apply_element_transform(&mut result, config);
+    normalize_nfc(&mut result);
+
+    // Run LLM-based structured extraction BEFORE output formatting
+    // so extraction sees plain text, not markdown/HTML
+    // TODO(wasm-llm): hosted structured extraction should run on wasm through
+    // liter-llm's wasm-http backend once browser/runtime support is wired.
+    #[cfg(all(feature = "liter-llm", not(target_os = "windows"), not(target_arch = "wasm32")))]
+    if let Some(ref structured_config) = config.structured_extraction {
+        match crate::llm::structured::extract_structured(&result.content, structured_config).await {
+            Ok((output, usage)) => {
+                result.structured_output = Some(output);
+                crate::llm::usage::push_llm_usage(&mut result, usage);
+            }
+            Err(e) => {
+                tracing::warn!("Structured extraction failed: {e}");
+                result.processing_warnings.push(crate::types::ProcessingWarning {
+                    source: std::borrow::Cow::Borrowed("structured_extraction"),
+                    message: std::borrow::Cow::Owned(format!("Structured extraction failed: {e}")),
+                });
+            }
+        }
+    }
+
+    // TODO(wasm-llm): keep wasm in the fallback branch until structured
+    // extraction has an async wasm-compatible runtime path.
+    #[cfg(any(not(feature = "liter-llm"), target_os = "windows", target_arch = "wasm32"))]
+    if config.structured_extraction.is_some() {
+        result.processing_warnings.push(crate::types::ProcessingWarning {
+            source: std::borrow::Cow::Borrowed("structured_extraction"),
+            message: std::borrow::Cow::Borrowed("Structured extraction requires the 'liter-llm' feature"),
+        });
+    }
+
+    // Apply output format conversion as the final step
+    result = apply_output_format(result, config.output_format.clone());
+
+    Ok(result)
+}
+
+/// Run the post-processing pipeline synchronously (WASM-compatible version).
+///
+/// This is a synchronous implementation for WASM and non-async contexts.
+/// It performs a subset of the full async pipeline, excluding async post-processors
+/// and validators.
+///
+/// # Arguments
+///
+/// * `doc` - The internal document produced by the extractor
+/// * `config` - Extraction configuration
+///
+/// # Returns
+///
+/// The processed extraction result.
+///
+/// # Notes
+///
+/// This function is only available when the `tokio-runtime` feature is disabled.
+/// It handles:
+/// - Quality processing (if enabled)
+/// - Chunking (if enabled)
+/// - Language detection (if enabled)
+///
+/// It does NOT handle:
+/// - Async post-processors
+/// - Async validators
+#[cfg(not(feature = "tokio-runtime"))]
+#[cfg_attr(alef, alef(skip))]
+pub fn run_pipeline_sync(doc: InternalDocument, config: &ExtractionConfig) -> Result<ExtractionResult> {
+    // Pre-render markdown for chunker heading context (same logic as async path).
+    #[cfg(feature = "chunking")]
+    let chunker_heading_source = {
+        let needs_markdown = config.chunking.as_ref().is_some_and(|c| {
+            c.chunker_type == crate::core::config::ChunkerType::Markdown
+                || c.resolve_preset().chunker_type == crate::core::config::ChunkerType::Markdown
+        }) && config.output_format == crate::core::config::OutputFormat::Plain;
+        if needs_markdown {
+            Some(crate::rendering::render_markdown(&doc))
+        } else {
+            None
+        }
+    };
+
+    // Pre-render styled HTML before `doc` is consumed (mirrors async path).
+    #[cfg(feature = "html")]
+    let styled_html_prerender: Option<String> = {
+        use crate::plugins::Renderer as _;
+        if config.output_format == crate::core::config::OutputFormat::Html {
+            config.html_output.as_ref().and_then(|html_cfg| {
+                match crate::rendering::StyledHtmlRenderer::new(html_cfg.clone()) {
+                    Ok(renderer) => match renderer.render(&doc) {
+                        Ok(html) => Some(html),
+                        Err(e) => {
+                            tracing::warn!("StyledHtmlRenderer render failed, falling back to default HTML: {e}");
+                            None
+                        }
+                    },
+                    Err(e) => {
+                        tracing::warn!("StyledHtmlRenderer construction failed, falling back to default HTML: {e}");
+                        None
+                    }
+                }
+            })
+        } else {
+            None
+        }
+    };
+
+    // 1. Derive ExtractionResult from InternalDocument
+    let include_structure = config.include_document_structure;
+    let mut result =
+        crate::extraction::derive::derive_extraction_result(doc, include_structure, config.output_format.clone());
+
+    // Inject pre-rendered styled HTML.
+    #[cfg(feature = "html")]
+    if let Some(html) = styled_html_prerender {
+        result.formatted_content = Some(html);
+    }
+
+    #[cfg(feature = "chunking")]
+    let chunker_only_markdown = result.formatted_content.is_none();
+    #[cfg(feature = "chunking")]
+    if chunker_only_markdown && let Some(md) = chunker_heading_source {
+        result.formatted_content = Some(md);
+    }
+
+    // 2. Run synchronous post-processing
+    execute_chunking(&mut result, config)?;
+
+    #[cfg(feature = "chunking")]
+    if chunker_only_markdown {
+        result.formatted_content = None;
+    }
+
+    execute_language_detection(&mut result, config)?;
+    execute_token_reduction(&mut result, config)?;
+
+    apply_element_transform(&mut result, config);
+    normalize_nfc(&mut result);
+
+    // Apply output format conversion as the final step
+    result = apply_output_format(result, config.output_format.clone());
+
+    Ok(result)
+}
+
+/// Transform to element-based output if requested by the config.
+fn apply_element_transform(result: &mut ExtractionResult, config: &ExtractionConfig) {
+    if config.result_format == crate::types::ResultFormat::ElementBased {
+        result.elements = Some(crate::extraction::transform::transform_extraction_result_to_elements(
+            result,
+        ));
+    }
+}
+
+/// Replace inline markdown image references with OCR text for formats (e.g. PPTX)
+/// that bake placeholders into paragraph text rather than using `ElementKind::Image`.
+fn replace_embedded_image_markdown_with_ocr(doc: &mut InternalDocument) {
+    if !doc.ocr_text_only || doc.images.is_empty() {
+        return;
+    }
+
+    let mut image_idx = 0usize;
+
+    for elem in &mut doc.elements {
+        if !matches!(elem.kind, crate::types::internal::ElementKind::Paragraph) {
+            continue;
+        }
+        if !is_markdown_image_reference(&elem.text) {
+            continue;
+        }
+        if let Some(img) = doc.images.get(image_idx)
+            && let Some(ocr) = &img.ocr_result
+            && !ocr.content.is_empty()
+        {
+            elem.text = ocr.content.clone();
+            image_idx += 1;
+            continue;
+        }
+        image_idx += 1;
+    }
+
+    for table in &mut doc.tables {
+        for row in &mut table.cells {
+            for cell in row {
+                if !is_markdown_image_reference(cell) {
+                    continue;
+                }
+                if let Some(img) = doc.images.get(image_idx)
+                    && let Some(ocr) = &img.ocr_result
+                    && !ocr.content.is_empty()
+                {
+                    *cell = ocr.content.clone();
+                    image_idx += 1;
+                    continue;
+                }
+                image_idx += 1;
+            }
+        }
+    }
+}
+
+/// Append OCR text after inline markdown image references for formats (e.g. PPTX)
+/// that bake placeholders into paragraph text. Only runs when `append_ocr_text` is
+/// `true` and `ocr_text_only` is `false`.
+fn append_embedded_image_ocr_text(doc: &mut InternalDocument) {
+    if doc.ocr_text_only || !doc.append_ocr_text || doc.images.is_empty() {
+        return;
+    }
+
+    let mut image_idx = 0usize;
+    let mut new_elements = Vec::with_capacity(doc.elements.len() * 2);
+
+    for elem in &doc.elements {
+        new_elements.push(elem.clone());
+
+        if matches!(elem.kind, crate::types::internal::ElementKind::Paragraph)
+            && is_markdown_image_reference(&elem.text)
+        {
+            if let Some(img) = doc.images.get(image_idx)
+                && let Some(ocr) = &img.ocr_result
+                && !ocr.content.is_empty()
+            {
+                let ocr_elem = crate::types::internal::InternalElement::text(
+                    crate::types::internal::ElementKind::Paragraph,
+                    ocr.content.clone(),
+                    0,
+                );
+                new_elements.push(ocr_elem);
+            }
+            image_idx += 1;
+        }
+    }
+
+    doc.elements = new_elements;
+
+    for table in &mut doc.tables {
+        for row in &mut table.cells {
+            for cell in row {
+                if !is_markdown_image_reference(cell) {
+                    continue;
+                }
+                if let Some(img) = doc.images.get(image_idx)
+                    && let Some(ocr) = &img.ocr_result
+                    && !ocr.content.is_empty()
+                {
+                    *cell = format!("{}\n\n{}", cell.trim(), ocr.content);
+                }
+                image_idx += 1;
+            }
+        }
+    }
+}
+
+/// Returns `true` if `text` is exactly a markdown image reference (`![alt](url)`).
+fn is_markdown_image_reference(text: &str) -> bool {
+    let t = text.trim();
+    if !t.starts_with("![") {
+        return false;
+    }
+    let Some(bracket_end) = t.find("](") else {
+        return false;
+    };
+    if bracket_end < 2 {
+        return false;
+    }
+    let after = &t[bracket_end + 2..];
+    after.ends_with(')')
+}
+
+/// Apply NFC unicode normalization to all text content.
+///
+/// Ensures consistent representation of composed characters (e.g., é vs e+combining accent)
+/// across all extraction backends (PDF, OCR, DOCX, HTML, etc.).
+fn normalize_nfc(result: &mut ExtractionResult) {
+    #[cfg(feature = "quality")]
+    {
+        use unicode_normalization::UnicodeNormalization;
+        result.content = result.content.nfc().collect();
+        if let Some(pages) = result.pages.as_mut() {
+            for page in pages.iter_mut() {
+                page.content = page.content.nfc().collect();
+            }
+        }
+    }
+    // Suppress unused variable warning when quality feature is disabled
+    let _ = result;
+}

crates/kreuzberg/src/core/pipeline/tests.rs

@@ -0,0 +1,993 @@
+//! Pipeline orchestration tests.
+
+use super::*;
+use crate::core::config::OutputFormat;
+use crate::types::Metadata;
+use crate::types::internal::{ElementKind, InternalDocument, InternalElement};
+use serial_test::serial;
+use std::borrow::Cow;
+
+/// Build an `InternalDocument` with a single paragraph element for pipeline tests.
+fn make_doc(content: &str, mime: &str) -> InternalDocument {
+    let mut doc = InternalDocument::new("plain");
+    doc.mime_type = mime.to_string();
+    if !content.is_empty() {
+        doc.push_element(InternalElement::text(ElementKind::Paragraph, content, 0));
+    }
+    doc
+}
+
+/// Build an `InternalDocument` with content, mime, and custom metadata.
+fn make_doc_with_metadata(content: &str, mime: &str, metadata: Metadata) -> InternalDocument {
+    let mut doc = make_doc(content, mime);
+    doc.metadata = metadata;
+    doc
+}
+
+const VALIDATION_MARKER_KEY: &str = "registry_validation_marker";
+#[cfg(feature = "quality")]
+const QUALITY_VALIDATION_MARKER: &str = "quality_validation_test";
+const POSTPROCESSOR_VALIDATION_MARKER: &str = "postprocessor_validation_test";
+const ORDER_VALIDATION_MARKER: &str = "order_validation_test";
+
+/// Ensure the quality processor is registered and cache is fresh.
+/// Needed because other tests may call `shutdown_all()` on the registry,
+/// and the `OnceLock` in `initialize_features` prevents re-registration.
+#[cfg(feature = "quality")]
+fn ensure_quality_processor() {
+    let registry = crate::plugins::registry::get_post_processor_registry();
+    let mut reg = registry.write();
+    let _ = reg.register(std::sync::Arc::new(crate::text::QualityProcessor));
+    drop(reg);
+    let _ = clear_processor_cache();
+}
+
+#[tokio::test]
+#[serial]
+async fn test_run_pipeline_basic() {
+    let mut doc = make_doc("test", "text/plain");
+    doc.metadata.additional.insert(
+        Cow::Borrowed(VALIDATION_MARKER_KEY),
+        serde_json::json!(ORDER_VALIDATION_MARKER),
+    );
+    let config = ExtractionConfig {
+        postprocessor: Some(crate::core::config::PostProcessorConfig {
+            enabled: false,
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+    assert_eq!(processed.content, "test");
+}
+
+#[tokio::test]
+#[serial]
+#[cfg(feature = "quality")]
+async fn test_pipeline_with_quality_processing() {
+    ensure_quality_processor();
+    let doc = make_doc("This is a test document with some meaningful content.", "text/plain");
+    let config = ExtractionConfig {
+        enable_quality_processing: true,
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+    assert!(processed.quality_score.is_some());
+}
+
+#[tokio::test]
+#[serial]
+async fn test_pipeline_without_quality_processing() {
+    let doc = make_doc("test", "text/plain");
+    let config = ExtractionConfig {
+        enable_quality_processing: false,
+        postprocessor: Some(crate::core::config::PostProcessorConfig {
+            enabled: false,
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+    assert!(processed.quality_score.is_none());
+}
+
+#[tokio::test]
+#[serial]
+#[cfg(feature = "chunking")]
+async fn test_pipeline_with_chunking() {
+    let doc = make_doc(
+        &"This is a long text that should be chunked. ".repeat(100),
+        "text/plain",
+    );
+    let config = ExtractionConfig {
+        chunking: Some(crate::ChunkingConfig {
+            max_characters: 500,
+            overlap: 50,
+            trim: true,
+            chunker_type: crate::ChunkerType::Text,
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+    let chunks = processed.chunks.as_ref().expect("chunks should be present");
+    assert!(chunks.len() > 1);
+}
+
+#[tokio::test]
+#[serial]
+async fn test_pipeline_without_chunking() {
+    let doc = make_doc("test", "text/plain");
+    let config = ExtractionConfig {
+        chunking: None,
+        postprocessor: Some(crate::core::config::PostProcessorConfig {
+            enabled: false,
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+    assert!(processed.chunks.is_none());
+}
+
+#[tokio::test]
+#[serial]
+async fn test_pipeline_preserves_metadata() {
+    use ahash::AHashMap;
+    let mut additional = AHashMap::new();
+    additional.insert(Cow::Borrowed("source"), serde_json::json!("test"));
+    additional.insert(Cow::Borrowed("page"), serde_json::json!(1));
+
+    let doc = make_doc_with_metadata(
+        "test",
+        "text/plain",
+        Metadata {
+            additional,
+            ..Default::default()
+        },
+    );
+    let config = ExtractionConfig {
+        postprocessor: Some(crate::core::config::PostProcessorConfig {
+            enabled: false,
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+    assert_eq!(
+        processed.metadata.additional.get("source").unwrap(),
+        &serde_json::json!("test")
+    );
+    assert_eq!(
+        processed.metadata.additional.get("page").unwrap(),
+        &serde_json::json!(1)
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_pipeline_preserves_tables() {
+    use crate::types::Table;
+
+    let table = Table {
+        cells: vec![vec!["A".to_string(), "B".to_string()]],
+        markdown: "| A | B |".to_string(),
+        page_number: 0,
+        bounding_box: None,
+    };
+
+    let mut doc = make_doc("test", "text/plain");
+    doc.tables.push(table);
+    let config = ExtractionConfig {
+        postprocessor: Some(crate::core::config::PostProcessorConfig {
+            enabled: false,
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+    assert_eq!(processed.tables.len(), 1);
+    assert_eq!(processed.tables[0].cells.len(), 1);
+}
+
+#[tokio::test]
+#[serial]
+async fn test_pipeline_empty_content() {
+    {
+        let registry = crate::plugins::registry::get_post_processor_registry();
+        registry.write().shutdown_all().unwrap();
+    }
+    {
+        let registry = crate::plugins::registry::get_validator_registry();
+        registry.write().shutdown_all().unwrap();
+    }
+
+    let doc = make_doc("", "text/plain");
+    let config = ExtractionConfig::default();
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+    assert_eq!(processed.content, "");
+}
+
+#[tokio::test]
+#[serial]
+#[cfg(feature = "chunking")]
+async fn test_pipeline_with_all_features() {
+    #[cfg(feature = "quality")]
+    ensure_quality_processor();
+    let doc = make_doc(&"This is a comprehensive test document. ".repeat(50), "text/plain");
+    let config = ExtractionConfig {
+        enable_quality_processing: true,
+        chunking: Some(crate::ChunkingConfig {
+            max_characters: 500,
+            overlap: 50,
+            trim: true,
+            chunker_type: crate::ChunkerType::Text,
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+    #[cfg(feature = "quality")]
+    assert!(processed.quality_score.is_some());
+    assert!(processed.chunks.is_some());
+}
+
+#[tokio::test]
+#[serial]
+#[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
+async fn test_pipeline_with_keyword_extraction() {
+    crate::plugins::registry::get_validator_registry()
+        .write()
+        .shutdown_all()
+        .unwrap();
+    crate::plugins::registry::get_post_processor_registry()
+        .write()
+        .shutdown_all()
+        .unwrap();
+
+    // Register keyword processor directly (bypasses Lazy which only runs once per process)
+    let _ = crate::keywords::register_keyword_processor();
+    clear_processor_cache().unwrap();
+
+    let doc = make_doc(
+        r#"
+Machine learning is a branch of artificial intelligence that focuses on
+building systems that can learn from data. Deep learning is a subset of
+machine learning that uses neural networks with multiple layers.
+Natural language processing enables computers to understand human language.
+            "#,
+        "text/plain",
+    );
+    #[cfg(feature = "keywords-yake")]
+    let keyword_config = crate::keywords::KeywordConfig::yake();
+
+    #[cfg(all(feature = "keywords-rake", not(feature = "keywords-yake")))]
+    let keyword_config = crate::keywords::KeywordConfig::rake();
+
+    let config = ExtractionConfig {
+        keywords: Some(keyword_config),
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+
+    let keywords = processed
+        .extracted_keywords
+        .as_ref()
+        .expect("Should have extracted keywords");
+    assert!(!keywords.is_empty(), "Should have extracted keywords");
+
+    let first_keyword = &keywords[0];
+    assert!(!first_keyword.text.is_empty());
+    assert!(first_keyword.score > 0.0);
+}
+
+#[tokio::test]
+#[serial]
+#[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
+async fn test_pipeline_without_keyword_config() {
+    let doc = make_doc("Machine learning and artificial intelligence.", "text/plain");
+
+    let config = ExtractionConfig {
+        keywords: None,
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+
+    assert!(!processed.metadata.additional.contains_key("keywords"));
+}
+
+#[tokio::test]
+#[serial]
+#[cfg(any(feature = "keywords-yake", feature = "keywords-rake"))]
+async fn test_pipeline_keyword_extraction_short_content() {
+    crate::plugins::registry::get_validator_registry()
+        .write()
+        .shutdown_all()
+        .unwrap();
+    crate::plugins::registry::get_post_processor_registry()
+        .write()
+        .shutdown_all()
+        .unwrap();
+
+    let doc = make_doc("Short text", "text/plain");
+
+    #[cfg(feature = "keywords-yake")]
+    let keyword_config = crate::keywords::KeywordConfig::yake();
+
+    #[cfg(all(feature = "keywords-rake", not(feature = "keywords-yake")))]
+    let keyword_config = crate::keywords::KeywordConfig::rake();
+
+    let config = ExtractionConfig {
+        keywords: Some(keyword_config),
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+
+    assert!(!processed.metadata.additional.contains_key("keywords"));
+}
+
+#[tokio::test]
+#[serial]
+async fn test_postprocessor_runs_before_validator() {
+    use crate::plugins::{Plugin, PostProcessor, ProcessingStage, Validator};
+    use async_trait::async_trait;
+    use std::sync::Arc;
+
+    struct TestPostProcessor;
+    impl Plugin for TestPostProcessor {
+        fn name(&self) -> &str {
+            "test-processor"
+        }
+        fn version(&self) -> String {
+            "1.0.0".to_string()
+        }
+        fn initialize(&self) -> Result<()> {
+            Ok(())
+        }
+        fn shutdown(&self) -> Result<()> {
+            Ok(())
+        }
+    }
+
+    #[async_trait]
+    impl PostProcessor for TestPostProcessor {
+        async fn process(&self, result: &mut ExtractionResult, _config: &ExtractionConfig) -> Result<()> {
+            result
+                .metadata
+                .additional
+                .insert(Cow::Borrowed("processed"), serde_json::json!(true));
+            Ok(())
+        }
+
+        fn processing_stage(&self) -> ProcessingStage {
+            ProcessingStage::Middle
+        }
+    }
+
+    struct TestValidator;
+    impl Plugin for TestValidator {
+        fn name(&self) -> &str {
+            "test-validator"
+        }
+        fn version(&self) -> String {
+            "1.0.0".to_string()
+        }
+        fn initialize(&self) -> Result<()> {
+            Ok(())
+        }
+        fn shutdown(&self) -> Result<()> {
+            Ok(())
+        }
+    }
+
+    #[async_trait]
+    impl Validator for TestValidator {
+        async fn validate(&self, result: &ExtractionResult, _config: &ExtractionConfig) -> Result<()> {
+            let should_validate = result
+                .metadata
+                .additional
+                .get(VALIDATION_MARKER_KEY)
+                .and_then(|v| v.as_str())
+                == Some(POSTPROCESSOR_VALIDATION_MARKER);
+
+            if !should_validate {
+                return Ok(());
+            }
+
+            let processed = result
+                .metadata
+                .additional
+                .get("processed")
+                .and_then(|v| v.as_bool())
+                .unwrap_or(false);
+
+            if !processed {
+                return Err(crate::KreuzbergError::Validation {
+                    message: "Post-processor did not run before validator".to_string(),
+                    source: None,
+                });
+            }
+            Ok(())
+        }
+    }
+
+    let pp_registry = crate::plugins::registry::get_post_processor_registry();
+    let val_registry = crate::plugins::registry::get_validator_registry();
+
+    clear_processor_cache().unwrap();
+    pp_registry.write().shutdown_all().unwrap();
+    val_registry.write().shutdown_all().unwrap();
+    clear_processor_cache().unwrap();
+
+    {
+        let mut registry = pp_registry.write();
+        registry.register(Arc::new(TestPostProcessor)).unwrap();
+    }
+
+    {
+        let mut registry = val_registry.write();
+        registry.register(Arc::new(TestValidator)).unwrap();
+    }
+
+    clear_processor_cache().unwrap();
+
+    let mut doc = make_doc("test", "text/plain");
+    doc.metadata.additional.insert(
+        Cow::Borrowed(VALIDATION_MARKER_KEY),
+        serde_json::json!(POSTPROCESSOR_VALIDATION_MARKER),
+    );
+
+    let config = ExtractionConfig {
+        postprocessor: Some(crate::core::config::PostProcessorConfig {
+            enabled: true,
+            enabled_set: None,
+            disabled_set: None,
+            enabled_processors: None,
+            disabled_processors: None,
+        }),
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await;
+
+    pp_registry.write().shutdown_all().unwrap();
+    val_registry.write().shutdown_all().unwrap();
+
+    assert!(processed.is_ok(), "Validator should have seen post-processor metadata");
+    let processed = processed.unwrap();
+    assert_eq!(
+        processed.metadata.additional.get("processed"),
+        Some(&serde_json::json!(true)),
+        "Post-processor metadata should be present"
+    );
+}
+
+#[tokio::test]
+#[serial]
+#[cfg(feature = "quality")]
+async fn test_quality_processing_runs_before_validator() {
+    ensure_quality_processor();
+    use crate::plugins::{Plugin, Validator};
+    use async_trait::async_trait;
+    use std::sync::Arc;
+
+    struct QualityValidator;
+    impl Plugin for QualityValidator {
+        fn name(&self) -> &str {
+            "quality-validator"
+        }
+        fn version(&self) -> String {
+            "1.0.0".to_string()
+        }
+        fn initialize(&self) -> Result<()> {
+            Ok(())
+        }
+        fn shutdown(&self) -> Result<()> {
+            Ok(())
+        }
+    }
+
+    #[async_trait]
+    impl Validator for QualityValidator {
+        async fn validate(&self, result: &ExtractionResult, _config: &ExtractionConfig) -> Result<()> {
+            let should_validate = result
+                .metadata
+                .additional
+                .get(VALIDATION_MARKER_KEY)
+                .and_then(|v| v.as_str())
+                == Some(QUALITY_VALIDATION_MARKER);
+
+            if !should_validate {
+                return Ok(());
+            }
+
+            if result.quality_score.is_none() {
+                return Err(crate::KreuzbergError::Validation {
+                    message: "Quality processing did not run before validator".to_string(),
+                    source: None,
+                });
+            }
+            Ok(())
+        }
+    }
+
+    let val_registry = crate::plugins::registry::get_validator_registry();
+    {
+        let mut registry = val_registry.write();
+        registry.register(Arc::new(QualityValidator)).unwrap();
+    }
+
+    let mut doc = make_doc("This is meaningful test content for quality scoring.", "text/plain");
+    doc.metadata.additional.insert(
+        Cow::Borrowed(VALIDATION_MARKER_KEY),
+        serde_json::json!(QUALITY_VALIDATION_MARKER),
+    );
+
+    let config = ExtractionConfig {
+        enable_quality_processing: true,
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await;
+
+    {
+        let mut registry = val_registry.write();
+        registry.remove("quality-validator").unwrap();
+    }
+
+    assert!(processed.is_ok(), "Validator should have seen quality_score");
+}
+
+#[tokio::test]
+#[serial]
+async fn test_multiple_postprocessors_run_before_validator() {
+    use crate::plugins::{Plugin, PostProcessor, ProcessingStage, Validator};
+    use async_trait::async_trait;
+    use std::sync::Arc;
+
+    struct EarlyProcessor;
+    impl Plugin for EarlyProcessor {
+        fn name(&self) -> &str {
+            "early-proc"
+        }
+        fn version(&self) -> String {
+            "1.0.0".to_string()
+        }
+        fn initialize(&self) -> Result<()> {
+            Ok(())
+        }
+        fn shutdown(&self) -> Result<()> {
+            Ok(())
+        }
+    }
+
+    #[async_trait]
+    impl PostProcessor for EarlyProcessor {
+        async fn process(&self, result: &mut ExtractionResult, _config: &ExtractionConfig) -> Result<()> {
+            let mut order = result
+                .metadata
+                .additional
+                .get("execution_order")
+                .and_then(|v| v.as_array())
+                .cloned()
+                .unwrap_or_default();
+            order.push(serde_json::json!("early"));
+            result
+                .metadata
+                .additional
+                .insert(Cow::Borrowed("execution_order"), serde_json::json!(order));
+            Ok(())
+        }
+
+        fn processing_stage(&self) -> ProcessingStage {
+            ProcessingStage::Early
+        }
+    }
+
+    struct LateProcessor;
+    impl Plugin for LateProcessor {
+        fn name(&self) -> &str {
+            "late-proc"
+        }
+        fn version(&self) -> String {
+            "1.0.0".to_string()
+        }
+        fn initialize(&self) -> Result<()> {
+            Ok(())
+        }
+        fn shutdown(&self) -> Result<()> {
+            Ok(())
+        }
+    }
+
+    #[async_trait]
+    impl PostProcessor for LateProcessor {
+        async fn process(&self, result: &mut ExtractionResult, _config: &ExtractionConfig) -> Result<()> {
+            let mut order = result
+                .metadata
+                .additional
+                .get("execution_order")
+                .and_then(|v| v.as_array())
+                .cloned()
+                .unwrap_or_default();
+            order.push(serde_json::json!("late"));
+            result
+                .metadata
+                .additional
+                .insert(Cow::Borrowed("execution_order"), serde_json::json!(order));
+            Ok(())
+        }
+
+        fn processing_stage(&self) -> ProcessingStage {
+            ProcessingStage::Late
+        }
+    }
+
+    struct OrderValidator;
+    impl Plugin for OrderValidator {
+        fn name(&self) -> &str {
+            "order-validator"
+        }
+        fn version(&self) -> String {
+            "1.0.0".to_string()
+        }
+        fn initialize(&self) -> Result<()> {
+            Ok(())
+        }
+        fn shutdown(&self) -> Result<()> {
+            Ok(())
+        }
+    }
+
+    #[async_trait]
+    impl Validator for OrderValidator {
+        async fn validate(&self, result: &ExtractionResult, _config: &ExtractionConfig) -> Result<()> {
+            let should_validate = result
+                .metadata
+                .additional
+                .get(VALIDATION_MARKER_KEY)
+                .and_then(|v| v.as_str())
+                == Some(ORDER_VALIDATION_MARKER);
+
+            if !should_validate {
+                return Ok(());
+            }
+
+            let order = result
+                .metadata
+                .additional
+                .get("execution_order")
+                .and_then(|v| v.as_array())
+                .ok_or_else(|| crate::KreuzbergError::Validation {
+                    message: "No execution order found".to_string(),
+                    source: None,
+                })?;
+
+            if order.len() != 2 {
+                return Err(crate::KreuzbergError::Validation {
+                    message: format!("Expected 2 processors to run, got {}", order.len()),
+                    source: None,
+                });
+            }
+
+            if order[0] != "early" || order[1] != "late" {
+                return Err(crate::KreuzbergError::Validation {
+                    message: format!("Wrong execution order: {:?}", order),
+                    source: None,
+                });
+            }
+
+            Ok(())
+        }
+    }
+
+    let pp_registry = crate::plugins::registry::get_post_processor_registry();
+    let val_registry = crate::plugins::registry::get_validator_registry();
+
+    pp_registry.write().shutdown_all().unwrap();
+    val_registry.write().shutdown_all().unwrap();
+    clear_processor_cache().unwrap();
+
+    {
+        let mut registry = pp_registry.write();
+        registry.register(Arc::new(EarlyProcessor)).unwrap();
+        registry.register(Arc::new(LateProcessor)).unwrap();
+    }
+
+    {
+        let mut registry = val_registry.write();
+        registry.register(Arc::new(OrderValidator)).unwrap();
+    }
+
+    // Clear the cache after registering new processors so it rebuilds with the test processors
+    clear_processor_cache().unwrap();
+
+    let doc = make_doc("test", "text/plain");
+
+    let config = ExtractionConfig::default();
+
+    let processed = run_pipeline(doc, &config).await;
+
+    pp_registry.write().shutdown_all().unwrap();
+    val_registry.write().shutdown_all().unwrap();
+    clear_processor_cache().unwrap();
+
+    assert!(processed.is_ok(), "All processors should run before validator");
+}
+
+#[tokio::test]
+#[serial]
+async fn test_run_pipeline_with_output_format_plain() {
+    let doc = make_doc("test content", "text/plain");
+
+    let config = crate::core::config::ExtractionConfig {
+        output_format: OutputFormat::Plain,
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+    assert_eq!(processed.content, "test content");
+    assert_eq!(processed.metadata.output_format, Some("plain".to_string()));
+}
+
+#[tokio::test]
+#[serial]
+async fn test_run_pipeline_with_output_format_djot() {
+    let doc = make_doc("test content", "text/djot");
+
+    let config = crate::core::config::ExtractionConfig {
+        output_format: OutputFormat::Djot,
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+    // The content should still be present
+    assert!(!processed.content.is_empty());
+    assert_eq!(processed.metadata.output_format, Some("djot".to_string()));
+}
+
+#[tokio::test]
+#[serial]
+async fn test_run_pipeline_with_output_format_html() {
+    let doc = make_doc("test content", "text/plain");
+
+    let config = crate::core::config::ExtractionConfig {
+        output_format: OutputFormat::Html,
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+    // HTML renderer produces semantic tags from InternalDocument
+    assert!(processed.content.contains("test content"));
+    assert_eq!(processed.metadata.output_format, Some("html".to_string()));
+}
+
+#[tokio::test]
+#[serial]
+#[cfg(feature = "quality")]
+async fn test_nfc_normalization_decomposes_to_composed() {
+    // NFC normalization should convert decomposed characters to composed form.
+    // "e\u{0301}" (e + combining acute accent) → "\u{00e9}" (é precomposed)
+    let doc = make_doc("caf\u{0065}\u{0301}", "text/plain"); // "café" with decomposed é
+    let config = ExtractionConfig {
+        postprocessor: Some(crate::core::config::PostProcessorConfig {
+            enabled: false,
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+    assert_eq!(processed.content, "caf\u{00e9}"); // composed é
+    assert!(!processed.content.contains('\u{0301}')); // no combining accent
+}
+
+#[tokio::test]
+#[serial]
+#[cfg(feature = "quality")]
+async fn test_nfc_normalization_idempotent_on_ascii() {
+    // NFC on already-normalized/ASCII text should be a no-op.
+    let doc = make_doc("Hello, world! 123", "text/plain");
+    let config = ExtractionConfig {
+        postprocessor: Some(crate::core::config::PostProcessorConfig {
+            enabled: false,
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+    assert_eq!(processed.content, "Hello, world! 123");
+}
+
+#[tokio::test]
+#[serial]
+#[cfg(feature = "quality")]
+async fn test_nfc_normalization_applies_to_page_content() {
+    // Create a doc with a page-1 element containing decomposed characters
+    let mut doc = InternalDocument::new("plain");
+    doc.mime_type = "text/plain".to_string();
+    doc.push_element(InternalElement::text(ElementKind::Paragraph, "re\u{0301}sume\u{0301}", 0).with_page(1));
+    let config = ExtractionConfig {
+        postprocessor: Some(crate::core::config::PostProcessorConfig {
+            enabled: false,
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+    // Content derived from page element
+    assert!(processed.content.contains("r\u{00e9}sum\u{00e9}"));
+    let pages = processed.pages.unwrap();
+    assert_eq!(pages[0].content, "r\u{00e9}sum\u{00e9}");
+}
+
+#[tokio::test]
+#[serial]
+async fn test_run_pipeline_applies_output_format_last() {
+    // This test verifies that output format is applied after all other processing
+    let doc = make_doc("test", "text/plain");
+
+    let config = crate::core::config::ExtractionConfig {
+        output_format: OutputFormat::Djot,
+        // Disable other processing to ensure pipeline runs cleanly
+        enable_quality_processing: false,
+        ..Default::default()
+    };
+
+    let processed = run_pipeline(doc, &config).await.unwrap();
+    // The result should have gone through the pipeline successfully
+    assert_eq!(processed.metadata.output_format, Some("djot".to_string()));
+}
+
+#[tokio::test]
+#[serial]
+#[cfg(all(feature = "pdf", feature = "chunking"))]
+async fn test_chunking_populates_page_numbers_for_pdf() {
+    use crate::core::config::ChunkingConfig;
+
+    let pdf_path =
+        std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test_documents/pdf/issue-636-chunk-pages.pdf");
+
+    if !pdf_path.exists() {
+        // Skip if test document not available
+        return;
+    }
+
+    let pdf_bytes = std::fs::read(&pdf_path).unwrap();
+
+    // Configure chunking WITHOUT explicit pages config (the default user scenario)
+    let config = ExtractionConfig {
+        chunking: Some(ChunkingConfig {
+            max_characters: 500,
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+
+    let result = crate::core::extractor::extract_bytes(&pdf_bytes, "application/pdf", &config)
+        .await
+        .unwrap();
+
+    // Chunks should exist
+    assert!(result.chunks.is_some(), "Chunks should be produced");
+    let chunks = result.chunks.as_ref().unwrap();
+    assert!(!chunks.is_empty(), "Should have at least one chunk");
+
+    // At least some chunks should have page numbers
+    let chunks_with_pages = chunks.iter().filter(|c| c.metadata.first_page.is_some()).count();
+    assert!(
+        chunks_with_pages > 0,
+        "At least some chunks should have page numbers, but none do. Total chunks: {}",
+        chunks.len()
+    );
+}
+
+#[test]
+fn test_append_ocr_text_for_pptx_images() {
+    use crate::types::ExtractedImage;
+    use crate::types::internal::{ElementKind, InternalDocument, InternalElement};
+    use std::borrow::Cow;
+
+    let mut doc = InternalDocument::new("pptx");
+    doc.append_ocr_text = true;
+    doc.elements
+        .push(InternalElement::text(ElementKind::Paragraph, "Before image.", 0));
+    doc.elements.push(InternalElement::text(
+        ElementKind::Paragraph,
+        "![img](../media/image-1.jpeg)",
+        0,
+    ));
+    doc.elements
+        .push(InternalElement::text(ElementKind::Paragraph, "After image.", 0));
+
+    doc.images.push(ExtractedImage {
+        data: bytes::Bytes::new(),
+        format: Cow::Borrowed("jpeg"),
+        image_index: 0,
+        page_number: Some(1),
+        width: Some(100),
+        height: Some(100),
+        colorspace: None,
+        bits_per_component: None,
+        is_mask: false,
+        description: None,
+        ocr_result: Some(Box::new(crate::types::ExtractionResult {
+            content: "OCR text here".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
+            ..Default::default()
+        })),
+        bounding_box: None,
+        source_path: None,
+        image_kind: None,
+        kind_confidence: None,
+        cluster_id: None,
+    });
+
+    super::append_embedded_image_ocr_text(&mut doc);
+
+    assert_eq!(
+        doc.elements.len(),
+        4,
+        "should have 4 elements (original 3 + 1 OCR paragraph)"
+    );
+    assert_eq!(doc.elements[2].text, "OCR text here");
+
+    let rendered = crate::rendering::render_markdown(&doc);
+    assert!(rendered.contains("OCR text here"));
+}
+
+#[tokio::test]
+#[serial]
+async fn test_pdf_run_fallback_not_suppressed_without_images_config() {
+    // When config.images is None, run_ocr_on_images must default to false so
+    // the PDF document-level OCR fallback is NOT silently suppressed for
+    // existing callers that never configured ImageExtractionConfig.
+    use crate::core::config::ImageExtractionConfig;
+
+    let default_no_images = crate::core::config::ExtractionConfig::default();
+    assert!(
+        default_no_images.images.is_none(),
+        "baseline: default config has no images section"
+    );
+
+    let skip_fallback = default_no_images
+        .images
+        .as_ref()
+        .map(|i| i.run_ocr_on_images)
+        .unwrap_or(false);
+    assert!(
+        !skip_fallback,
+        "RunFallback must NOT be suppressed when config.images is None"
+    );
+
+    let with_images_opted_in = crate::core::config::ExtractionConfig {
+        images: Some(ImageExtractionConfig {
+            run_ocr_on_images: true,
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+    let skip_fallback_opted_in = with_images_opted_in
+        .images
+        .as_ref()
+        .map(|i| i.run_ocr_on_images)
+        .unwrap_or(false);
+    assert!(
+        skip_fallback_opted_in,
+        "RunFallback must be suppressed when images.run_ocr_on_images=true"
+    );
+}

crates/kreuzberg/src/core/server_config/env.rs

@@ -0,0 +1,76 @@
+//! Environment variable overrides for server configuration.
+//!
+//! This module provides functionality to override server configuration values
+//! using environment variables. All settings can be overridden at runtime.
+
+use crate::{KreuzbergError, Result};
+
+/// Apply environment variable overrides to a ServerConfig.
+///
+/// Reads the following environment variables and overrides config values if set:
+///
+/// - `KREUZBERG_HOST` - Server host address
+/// - `KREUZBERG_PORT` - Server port number (parsed as u16)
+/// - `KREUZBERG_CORS_ORIGINS` - Comma-separated list of allowed origins
+/// - `KREUZBERG_MAX_REQUEST_BODY_BYTES` - Max request body size in bytes
+/// - `KREUZBERG_MAX_MULTIPART_FIELD_BYTES` - Max multipart field size in bytes
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Validation` if:
+/// - `KREUZBERG_PORT` cannot be parsed as u16
+/// - `KREUZBERG_MAX_REQUEST_BODY_BYTES` cannot be parsed as usize
+/// - `KREUZBERG_MAX_MULTIPART_FIELD_BYTES` cannot be parsed as usize
+pub(crate) fn apply_env_overrides(
+    host: &mut String,
+    port: &mut u16,
+    cors_origins: &mut Vec<String>,
+    max_request_body_bytes: &mut usize,
+    max_multipart_field_bytes: &mut usize,
+) -> Result<()> {
+    // Host override
+    if let Ok(env_host) = std::env::var("KREUZBERG_HOST") {
+        *host = env_host;
+    }
+
+    // Port override
+    if let Ok(port_str) = std::env::var("KREUZBERG_PORT") {
+        *port = port_str.parse::<u16>().map_err(|e| {
+            KreuzbergError::validation(format!(
+                "KREUZBERG_PORT must be a valid u16 number, got '{}': {}",
+                port_str, e
+            ))
+        })?;
+    }
+
+    // CORS origins override (comma-separated)
+    if let Ok(origins_str) = std::env::var("KREUZBERG_CORS_ORIGINS") {
+        *cors_origins = origins_str
+            .split(',')
+            .map(|s| s.trim().to_string())
+            .filter(|s| !s.is_empty())
+            .collect();
+    }
+
+    // Max request body bytes override
+    if let Ok(bytes_str) = std::env::var("KREUZBERG_MAX_REQUEST_BODY_BYTES") {
+        *max_request_body_bytes = bytes_str.parse::<usize>().map_err(|e| {
+            KreuzbergError::validation(format!(
+                "KREUZBERG_MAX_REQUEST_BODY_BYTES must be a valid usize, got '{}': {}",
+                bytes_str, e
+            ))
+        })?;
+    }
+
+    // Max multipart field bytes override
+    if let Ok(bytes_str) = std::env::var("KREUZBERG_MAX_MULTIPART_FIELD_BYTES") {
+        *max_multipart_field_bytes = bytes_str.parse::<usize>().map_err(|e| {
+            KreuzbergError::validation(format!(
+                "KREUZBERG_MAX_MULTIPART_FIELD_BYTES must be a valid usize, got '{}': {}",
+                bytes_str, e
+            ))
+        })?;
+    }
+
+    Ok(())
+}

crates/kreuzberg/src/core/server_config/loader.rs

@@ -0,0 +1,193 @@
+//! File loading logic for server configuration.
+//!
+//! This module provides functionality to load server configuration from various
+//! file formats (TOML, YAML, JSON) with support for both flat and nested formats.
+
+use crate::{KreuzbergError, Result};
+use serde::Deserialize;
+use std::path::Path;
+
+use super::ServerConfig;
+
+/// Load server configuration from a file.
+///
+/// Automatically detects the file format based on extension:
+/// - `.toml` - TOML format
+/// - `.yaml` or `.yml` - YAML format
+/// - `.json` - JSON format
+///
+/// This function handles two config file formats:
+/// 1. Flat format: Server config at root level
+/// 2. Nested format: Server config under `[server]` section (combined with ExtractionConfig)
+///
+/// # Arguments
+///
+/// * `path` - Path to the configuration file
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Validation` if:
+/// - File doesn't exist or cannot be read
+/// - File extension is not recognized
+/// - File content is invalid for the detected format
+pub(crate) fn from_file(path: impl AsRef<Path>) -> Result<ServerConfig> {
+    let path = path.as_ref();
+
+    let content = std::fs::read_to_string(path)
+        .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
+
+    let extension = path.extension().and_then(|ext| ext.to_str()).ok_or_else(|| {
+        KreuzbergError::validation(format!(
+            "Cannot determine file format: no extension found in {}",
+            path.display()
+        ))
+    })?;
+
+    let config = match extension.to_lowercase().as_str() {
+        "toml" => from_toml_str(&content, path)?,
+        "yaml" | "yml" => from_yaml_str(&content, path)?,
+        "json" => from_json_str(&content, path)?,
+        _ => {
+            return Err(KreuzbergError::validation(format!(
+                "Unsupported config file format: .{}. Supported formats: .toml, .yaml, .yml, .json",
+                extension
+            )));
+        }
+    };
+
+    Ok(config)
+}
+
+/// Load server configuration from a TOML file.
+///
+/// # Arguments
+///
+/// * `path` - Path to the TOML file
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Validation` if the file doesn't exist or is invalid TOML.
+pub(crate) fn from_toml_file(path: impl AsRef<Path>) -> Result<ServerConfig> {
+    let path = path.as_ref();
+
+    let content = std::fs::read_to_string(path)
+        .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
+
+    let config: ServerConfig = toml::from_str(&content)
+        .map_err(|e| KreuzbergError::validation(format!("Invalid TOML in {}: {}", path.display(), e)))?;
+
+    Ok(config)
+}
+
+/// Load server configuration from a YAML file.
+///
+/// # Arguments
+///
+/// * `path` - Path to the YAML file
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Validation` if the file doesn't exist or is invalid YAML.
+pub(crate) fn from_yaml_file(path: impl AsRef<Path>) -> Result<ServerConfig> {
+    let path = path.as_ref();
+
+    let content = std::fs::read_to_string(path)
+        .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
+
+    let config: ServerConfig = serde_yaml_ng::from_str(&content)
+        .map_err(|e| KreuzbergError::validation(format!("Invalid YAML in {}: {}", path.display(), e)))?;
+
+    Ok(config)
+}
+
+/// Load server configuration from a JSON file.
+///
+/// # Arguments
+///
+/// * `path` - Path to the JSON file
+///
+/// # Errors
+///
+/// Returns `KreuzbergError::Validation` if the file doesn't exist or is invalid JSON.
+pub(crate) fn from_json_file(path: impl AsRef<Path>) -> Result<ServerConfig> {
+    let path = path.as_ref();
+
+    let content = std::fs::read_to_string(path)
+        .map_err(|e| KreuzbergError::validation(format!("Failed to read config file {}: {}", path.display(), e)))?;
+
+    let config: ServerConfig = serde_json::from_str(&content)
+        .map_err(|e| KreuzbergError::validation(format!("Invalid JSON in {}: {}", path.display(), e)))?;
+
+    Ok(config)
+}
+
+// Helper functions for parsing different formats
+
+fn from_toml_str(content: &str, path: &Path) -> Result<ServerConfig> {
+    // Try nested format first (with [server] section)
+    #[derive(Deserialize)]
+    struct RootConfig {
+        #[serde(default)]
+        server: Option<ServerConfig>,
+    }
+
+    if let Ok(root) = toml::from_str::<RootConfig>(content) {
+        if let Some(server) = root.server {
+            return Ok(server);
+        } else {
+            // No [server] section, try flat format
+            return toml::from_str::<ServerConfig>(content)
+                .map_err(|e| KreuzbergError::validation(format!("Invalid TOML in {}: {}", path.display(), e)));
+        }
+    }
+
+    // Fall back to flat format
+    toml::from_str::<ServerConfig>(content)
+        .map_err(|e| KreuzbergError::validation(format!("Invalid TOML in {}: {}", path.display(), e)))
+}
+
+fn from_yaml_str(content: &str, path: &Path) -> Result<ServerConfig> {
+    // Try nested format first (with server: section)
+    #[derive(Deserialize)]
+    struct RootConfig {
+        #[serde(default)]
+        server: Option<ServerConfig>,
+    }
+
+    if let Ok(root) = serde_yaml_ng::from_str::<RootConfig>(content) {
+        if let Some(server) = root.server {
+            return Ok(server);
+        } else {
+            // No server section, try flat format
+            return serde_yaml_ng::from_str::<ServerConfig>(content)
+                .map_err(|e| KreuzbergError::validation(format!("Invalid YAML in {}: {}", path.display(), e)));
+        }
+    }
+
+    // Fall back to flat format
+    serde_yaml_ng::from_str::<ServerConfig>(content)
+        .map_err(|e| KreuzbergError::validation(format!("Invalid YAML in {}: {}", path.display(), e)))
+}
+
+fn from_json_str(content: &str, path: &Path) -> Result<ServerConfig> {
+    // Try nested format first (with "server" key)
+    #[derive(Deserialize)]
+    struct RootConfig {
+        #[serde(default)]
+        server: Option<ServerConfig>,
+    }
+
+    if let Ok(root) = serde_json::from_str::<RootConfig>(content) {
+        if let Some(server) = root.server {
+            return Ok(server);
+        } else {
+            // No server key, try flat format
+            return serde_json::from_str::<ServerConfig>(content)
+                .map_err(|e| KreuzbergError::validation(format!("Invalid JSON in {}: {}", path.display(), e)));
+        }
+    }
+
+    // Fall back to flat format
+    serde_json::from_str::<ServerConfig>(content)
+        .map_err(|e| KreuzbergError::validation(format!("Invalid JSON in {}: {}", path.display(), e)))
+}

crates/kreuzberg/src/core/server_config/mod.rs

@@ -0,0 +1,357 @@
+//! Server configuration for the Kreuzberg API.
+//!
+//! This module provides the `ServerConfig` struct for managing API server settings
+//! including host, port, CORS, and upload size limits. Configuration can be loaded
+//! from TOML, YAML, or JSON files and can be overridden by environment variables.
+//!
+//! # Features
+//!
+//! - **Multi-format support**: Load configuration from TOML, YAML, or JSON files
+//! - **Environment overrides**: All settings can be overridden via environment variables
+//! - **Sensible defaults**: All fields have reasonable defaults matching current behavior
+//! - **Flexible CORS**: Support for all origins (default) or specific origin lists
+//!
+//! # Example
+//!
+//! ```rust,no_run
+//! use kreuzberg::core::ServerConfig;
+//!
+//! # fn example() -> kreuzberg::Result<()> {
+//! // Create with defaults
+//! let mut config = ServerConfig::default();
+//!
+//! // Or load from file
+//! let mut config = ServerConfig::from_file("kreuzberg.toml")?;
+//!
+//! // Apply environment variable overrides
+//! config.apply_env_overrides()?;
+//!
+//! # Ok(())
+//! # }
+//! ```
+
+use crate::Result;
+use serde::{Deserialize, Serialize};
+use std::path::Path;
+
+mod env;
+mod loader;
+mod validation;
+
+#[cfg(test)]
+mod tests;
+
+/// Default host address for API server
+const DEFAULT_HOST: &str = "127.0.0.1";
+
+/// Default port for API server
+const DEFAULT_PORT: u16 = 8000;
+
+/// Default maximum request body size: 100 MB
+const DEFAULT_MAX_REQUEST_BODY_BYTES: usize = 104_857_600;
+
+/// Default maximum multipart field size: 100 MB
+const DEFAULT_MAX_MULTIPART_FIELD_BYTES: usize = 104_857_600;
+
+/// API server configuration.
+///
+/// This struct holds all configuration options for the Kreuzberg API server,
+/// including host/port settings, CORS configuration, and upload limits.
+///
+/// # Defaults
+///
+/// - `host`: "127.0.0.1" (localhost only)
+/// - `port`: 8000
+/// - `cors_origins`: empty vector (allows all origins)
+/// - `max_request_body_bytes`: 104_857_600 (100 MB)
+/// - `max_multipart_field_bytes`: 104_857_600 (100 MB)
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default)]
+pub struct ServerConfig {
+    /// Server host address (e.g., "127.0.0.1", "0.0.0.0")
+    #[serde(default = "default_host")]
+    pub host: String,
+
+    /// Server port number
+    #[serde(default = "default_port")]
+    pub port: u16,
+
+    /// CORS allowed origins. Empty vector means allow all origins.
+    ///
+    /// If this is an empty vector, the server will accept requests from any origin.
+    /// If populated with specific origins (e.g., `"https://example.com"`), only
+    /// those origins will be allowed.
+    #[serde(default)]
+    pub cors_origins: Vec<String>,
+
+    /// Maximum size of request body in bytes (default: 100 MB)
+    #[serde(default = "default_max_request_body_bytes")]
+    pub max_request_body_bytes: usize,
+
+    /// Maximum size of multipart fields in bytes (default: 100 MB)
+    #[serde(default = "default_max_multipart_field_bytes")]
+    pub max_multipart_field_bytes: usize,
+}
+
+impl Default for ServerConfig {
+    fn default() -> Self {
+        Self {
+            host: default_host(),
+            port: default_port(),
+            cors_origins: Vec::new(),
+            max_request_body_bytes: default_max_request_body_bytes(),
+            max_multipart_field_bytes: default_max_multipart_field_bytes(),
+        }
+    }
+}
+
+// Default value functions for serde
+fn default_host() -> String {
+    DEFAULT_HOST.to_string()
+}
+
+fn default_port() -> u16 {
+    DEFAULT_PORT
+}
+
+fn default_max_request_body_bytes() -> usize {
+    DEFAULT_MAX_REQUEST_BODY_BYTES
+}
+
+fn default_max_multipart_field_bytes() -> usize {
+    DEFAULT_MAX_MULTIPART_FIELD_BYTES
+}
+
+impl ServerConfig {
+    /// Create a new `ServerConfig` with default values.
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Get the server listen address (host:port).
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// use kreuzberg::core::ServerConfig;
+    ///
+    /// let config = ServerConfig::default();
+    /// assert_eq!(config.listen_addr(), "127.0.0.1:8000");
+    /// ```
+    pub fn listen_addr(&self) -> String {
+        format!("{}:{}", self.host, self.port)
+    }
+
+    /// Check if CORS allows all origins.
+    ///
+    /// Returns `true` if the `cors_origins` vector is empty, meaning all origins
+    /// are allowed. Returns `false` if specific origins are configured.
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// use kreuzberg::core::ServerConfig;
+    ///
+    /// let mut config = ServerConfig::default();
+    /// assert!(config.cors_allows_all());
+    ///
+    /// config.cors_origins.push("https://example.com".to_string());
+    /// assert!(!config.cors_allows_all());
+    /// ```
+    pub fn cors_allows_all(&self) -> bool {
+        self.cors_origins.is_empty()
+    }
+
+    /// Check if a given origin is allowed by CORS configuration.
+    ///
+    /// Returns `true` if:
+    /// - CORS allows all origins (empty origins list), or
+    /// - The given origin is in the allowed origins list
+    ///
+    /// # Arguments
+    ///
+    /// * `origin` - The origin to check (e.g., "https://example.com")
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// use kreuzberg::core::ServerConfig;
+    ///
+    /// let mut config = ServerConfig::default();
+    /// assert!(config.is_origin_allowed("https://example.com"));
+    ///
+    /// config.cors_origins.push("https://allowed.com".to_string());
+    /// assert!(config.is_origin_allowed("https://allowed.com"));
+    /// assert!(!config.is_origin_allowed("https://denied.com"));
+    /// ```
+    pub fn is_origin_allowed(&self, origin: &str) -> bool {
+        self.cors_origins.is_empty() || self.cors_origins.contains(&origin.to_string())
+    }
+
+    /// Get maximum request body size in megabytes (rounded up).
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// use kreuzberg::core::ServerConfig;
+    ///
+    /// let mut config = ServerConfig::default();
+    /// assert_eq!(config.max_request_body_mb(), 100);
+    /// ```
+    pub fn max_request_body_mb(&self) -> usize {
+        self.max_request_body_bytes.div_ceil(1_048_576)
+    }
+
+    /// Get maximum multipart field size in megabytes (rounded up).
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// use kreuzberg::core::ServerConfig;
+    ///
+    /// let mut config = ServerConfig::default();
+    /// assert_eq!(config.max_multipart_field_mb(), 100);
+    /// ```
+    pub fn max_multipart_field_mb(&self) -> usize {
+        self.max_multipart_field_bytes.div_ceil(1_048_576)
+    }
+
+    /// Apply environment variable overrides to the configuration.
+    ///
+    /// Reads the following environment variables and overrides config values if set:
+    ///
+    /// - `KREUZBERG_HOST` - Server host address
+    /// - `KREUZBERG_PORT` - Server port number (parsed as u16)
+    /// - `KREUZBERG_CORS_ORIGINS` - Comma-separated list of allowed origins
+    /// - `KREUZBERG_MAX_REQUEST_BODY_BYTES` - Max request body size in bytes
+    /// - `KREUZBERG_MAX_MULTIPART_FIELD_BYTES` - Max multipart field size in bytes
+    ///
+    /// # Errors
+    ///
+    /// Returns `KreuzbergError::Validation` if:
+    /// - `KREUZBERG_PORT` cannot be parsed as u16
+    /// - `KREUZBERG_MAX_REQUEST_BODY_BYTES` cannot be parsed as usize
+    /// - `KREUZBERG_MAX_MULTIPART_FIELD_BYTES` cannot be parsed as usize
+    ///
+    /// # Example
+    ///
+    /// ```rust,no_run
+    /// use kreuzberg::core::ServerConfig;
+    ///
+    /// # fn example() -> kreuzberg::Result<()> {
+    /// unsafe {
+    ///     std::env::set_var("KREUZBERG_HOST", "0.0.0.0");
+    ///     std::env::set_var("KREUZBERG_PORT", "3000");
+    /// }
+    ///
+    /// let mut config = ServerConfig::default();
+    /// config.apply_env_overrides()?;
+    ///
+    /// assert_eq!(config.host, "0.0.0.0");
+    /// assert_eq!(config.port, 3000);
+    /// # Ok(())
+    /// # }
+    /// ```
+    #[cfg_attr(alef, alef(skip))]
+    pub fn apply_env_overrides(&mut self) -> Result<()> {
+        env::apply_env_overrides(
+            &mut self.host,
+            &mut self.port,
+            &mut self.cors_origins,
+            &mut self.max_request_body_bytes,
+            &mut self.max_multipart_field_bytes,
+        )?;
+
+        Ok(())
+    }
+
+    /// Load server configuration from a file.
+    ///
+    /// Automatically detects the file format based on extension:
+    /// - `.toml` - TOML format
+    /// - `.yaml` or `.yml` - YAML format
+    /// - `.json` - JSON format
+    ///
+    /// This function handles two config file formats:
+    /// 1. Flat format: Server config at root level
+    /// 2. Nested format: Server config under `[server]` section (combined with ExtractionConfig)
+    ///
+    /// # Arguments
+    ///
+    /// * `path` - Path to the configuration file
+    ///
+    /// # Errors
+    ///
+    /// Returns `KreuzbergError::Validation` if:
+    /// - File doesn't exist or cannot be read
+    /// - File extension is not recognized
+    /// - File content is invalid for the detected format
+    ///
+    /// # Example
+    ///
+    /// ```rust,no_run
+    /// use kreuzberg::core::ServerConfig;
+    ///
+    /// # fn example() -> kreuzberg::Result<()> {
+    /// let config = ServerConfig::from_file("kreuzberg.toml")?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    #[cfg_attr(alef, alef(skip))]
+    pub fn from_file(path: impl AsRef<Path>) -> Result<Self> {
+        loader::from_file(path)
+    }
+
+    /// Load server configuration from a TOML file.
+    ///
+    /// # Arguments
+    ///
+    /// * `path` - Path to the TOML file
+    ///
+    /// # Errors
+    ///
+    /// Returns `KreuzbergError::Validation` if the file doesn't exist or is invalid TOML.
+    ///
+    /// # Example
+    ///
+    /// ```rust,no_run
+    /// use kreuzberg::core::ServerConfig;
+    ///
+    /// # fn example() -> kreuzberg::Result<()> {
+    /// let config = ServerConfig::from_toml_file("kreuzberg.toml")?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    #[cfg_attr(alef, alef(skip))]
+    pub fn from_toml_file(path: impl AsRef<Path>) -> Result<Self> {
+        loader::from_toml_file(path)
+    }
+
+    /// Load server configuration from a YAML file.
+    ///
+    /// # Arguments
+    ///
+    /// * `path` - Path to the YAML file
+    ///
+    /// # Errors
+    ///
+    /// Returns `KreuzbergError::Validation` if the file doesn't exist or is invalid YAML.
+    #[cfg_attr(alef, alef(skip))]
+    pub fn from_yaml_file(path: impl AsRef<Path>) -> Result<Self> {
+        loader::from_yaml_file(path)
+    }
+
+    /// Load server configuration from a JSON file.
+    ///
+    /// # Arguments
+    ///
+    /// * `path` - Path to the JSON file
+    ///
+    /// # Errors
+    ///
+    /// Returns `KreuzbergError::Validation` if the file doesn't exist or is invalid JSON.
+    #[cfg_attr(alef, alef(skip))]
+    pub fn from_json_file(path: impl AsRef<Path>) -> Result<Self> {
+        loader::from_json_file(path)
+    }
+}

crates/kreuzberg/src/core/server_config/tests/basic_tests.rs

@@ -0,0 +1,87 @@
+//! Basic tests for ServerConfig functionality.
+
+use crate::core::ServerConfig;
+
+#[test]
+fn test_default_config() {
+    let config = ServerConfig::default();
+    assert_eq!(config.host, "127.0.0.1");
+    assert_eq!(config.port, 8000);
+    assert!(config.cors_origins.is_empty());
+    assert_eq!(config.max_request_body_bytes, 104_857_600);
+    assert_eq!(config.max_multipart_field_bytes, 104_857_600);
+}
+
+#[test]
+fn test_listen_addr() {
+    let config = ServerConfig::default();
+    assert_eq!(config.listen_addr(), "127.0.0.1:8000");
+}
+
+#[test]
+fn test_listen_addr_custom() {
+    let config = ServerConfig {
+        host: "0.0.0.0".to_string(),
+        port: 3000,
+        ..Default::default()
+    };
+    assert_eq!(config.listen_addr(), "0.0.0.0:3000");
+}
+
+#[test]
+fn test_cors_allows_all() {
+    let mut config = ServerConfig::default();
+    assert!(config.cors_allows_all());
+
+    config.cors_origins.push("https://example.com".to_string());
+    assert!(!config.cors_allows_all());
+}
+
+#[test]
+fn test_is_origin_allowed_all() {
+    let config = ServerConfig::default();
+    assert!(config.is_origin_allowed("https://example.com"));
+    assert!(config.is_origin_allowed("https://other.com"));
+}
+
+#[test]
+fn test_is_origin_allowed_specific() {
+    let mut config = ServerConfig::default();
+    config.cors_origins.push("https://allowed.com".to_string());
+
+    assert!(config.is_origin_allowed("https://allowed.com"));
+    assert!(!config.is_origin_allowed("https://denied.com"));
+}
+
+#[test]
+fn test_max_request_body_mb() {
+    let config = ServerConfig::default();
+    assert_eq!(config.max_request_body_mb(), 100);
+}
+
+#[test]
+fn test_max_multipart_field_mb() {
+    let config = ServerConfig::default();
+    assert_eq!(config.max_multipart_field_mb(), 100);
+}
+
+#[test]
+fn test_max_bytes_to_mb_rounding() {
+    let mut config = ServerConfig {
+        max_request_body_bytes: 1_048_576, // 1 MB
+        ..Default::default()
+    };
+    assert_eq!(config.max_request_body_mb(), 1);
+
+    config.max_request_body_bytes = 1_048_577; // 1 MB + 1 byte
+    assert_eq!(config.max_request_body_mb(), 2); // Rounds up
+}
+
+#[test]
+fn test_serde_default_serialization() {
+    let config = ServerConfig::default();
+    let json = serde_json::to_string(&config).unwrap();
+
+    assert!(json.contains("host"));
+    assert!(json.contains("port"));
+}

crates/kreuzberg/src/core/server_config/tests/env_tests.rs

@@ -0,0 +1,192 @@
+//! Tests for environment variable overrides.
+
+#![allow(unsafe_code)]
+
+use crate::core::ServerConfig;
+
+#[serial_test::serial]
+#[test]
+fn test_apply_env_host_override() {
+    let original = std::env::var("KREUZBERG_HOST").ok();
+    unsafe {
+        std::env::set_var("KREUZBERG_HOST", "192.168.1.1");
+    }
+
+    let mut config = ServerConfig::default();
+    config.apply_env_overrides().unwrap();
+
+    assert_eq!(config.host, "192.168.1.1");
+
+    // Cleanup
+    unsafe {
+        if let Some(orig) = original {
+            std::env::set_var("KREUZBERG_HOST", orig);
+        } else {
+            std::env::remove_var("KREUZBERG_HOST");
+        }
+    }
+}
+
+#[serial_test::serial]
+#[test]
+fn test_apply_env_port_override() {
+    let original = std::env::var("KREUZBERG_PORT").ok();
+    unsafe {
+        std::env::set_var("KREUZBERG_PORT", "5000");
+    }
+
+    let mut config = ServerConfig::default();
+    config.apply_env_overrides().unwrap();
+
+    assert_eq!(config.port, 5000);
+
+    // Cleanup
+    unsafe {
+        if let Some(orig) = original {
+            std::env::set_var("KREUZBERG_PORT", orig);
+        } else {
+            std::env::remove_var("KREUZBERG_PORT");
+        }
+    }
+}
+
+#[serial_test::serial]
+#[test]
+fn test_apply_env_port_invalid() {
+    let original = std::env::var("KREUZBERG_PORT").ok();
+    unsafe {
+        std::env::set_var("KREUZBERG_PORT", "not_a_number");
+    }
+
+    let mut config = ServerConfig::default();
+    let result = config.apply_env_overrides();
+
+    assert!(result.is_err());
+    assert!(
+        result
+            .unwrap_err()
+            .to_string()
+            .contains("KREUZBERG_PORT must be a valid u16")
+    );
+
+    // Cleanup
+    unsafe {
+        if let Some(orig) = original {
+            std::env::set_var("KREUZBERG_PORT", orig);
+        } else {
+            std::env::remove_var("KREUZBERG_PORT");
+        }
+    }
+}
+
+#[serial_test::serial]
+#[test]
+fn test_apply_env_cors_origins_override() {
+    let original = std::env::var("KREUZBERG_CORS_ORIGINS").ok();
+    unsafe {
+        std::env::set_var("KREUZBERG_CORS_ORIGINS", "https://example.com, https://other.com");
+    }
+
+    let mut config = ServerConfig::default();
+    config.apply_env_overrides().unwrap();
+
+    assert_eq!(config.cors_origins.len(), 2);
+    assert!(config.cors_origins.contains(&"https://example.com".to_string()));
+    assert!(config.cors_origins.contains(&"https://other.com".to_string()));
+
+    // Cleanup
+    unsafe {
+        if let Some(orig) = original {
+            std::env::set_var("KREUZBERG_CORS_ORIGINS", orig);
+        } else {
+            std::env::remove_var("KREUZBERG_CORS_ORIGINS");
+        }
+    }
+}
+
+#[serial_test::serial]
+#[test]
+fn test_apply_env_max_request_body_bytes_override() {
+    let original = std::env::var("KREUZBERG_MAX_REQUEST_BODY_BYTES").ok();
+    unsafe {
+        std::env::set_var("KREUZBERG_MAX_REQUEST_BODY_BYTES", "52428800");
+    }
+
+    let mut config = ServerConfig::default();
+    config.apply_env_overrides().unwrap();
+
+    assert_eq!(config.max_request_body_bytes, 52_428_800);
+
+    // Cleanup
+    unsafe {
+        if let Some(orig) = original {
+            std::env::set_var("KREUZBERG_MAX_REQUEST_BODY_BYTES", orig);
+        } else {
+            std::env::remove_var("KREUZBERG_MAX_REQUEST_BODY_BYTES");
+        }
+    }
+}
+
+#[serial_test::serial]
+#[test]
+fn test_apply_env_max_multipart_field_bytes_override() {
+    let original = std::env::var("KREUZBERG_MAX_MULTIPART_FIELD_BYTES").ok();
+    unsafe {
+        std::env::set_var("KREUZBERG_MAX_MULTIPART_FIELD_BYTES", "78643200");
+    }
+
+    let mut config = ServerConfig::default();
+    config.apply_env_overrides().unwrap();
+
+    assert_eq!(config.max_multipart_field_bytes, 78_643_200);
+
+    // Cleanup
+    unsafe {
+        if let Some(orig) = original {
+            std::env::set_var("KREUZBERG_MAX_MULTIPART_FIELD_BYTES", orig);
+        } else {
+            std::env::remove_var("KREUZBERG_MAX_MULTIPART_FIELD_BYTES");
+        }
+    }
+}
+
+#[serial_test::serial]
+#[test]
+fn test_apply_env_multiple_overrides() {
+    let host_orig = std::env::var("KREUZBERG_HOST").ok();
+    let port_orig = std::env::var("KREUZBERG_PORT").ok();
+    let cors_orig = std::env::var("KREUZBERG_CORS_ORIGINS").ok();
+
+    unsafe {
+        std::env::set_var("KREUZBERG_HOST", "0.0.0.0");
+        std::env::set_var("KREUZBERG_PORT", "4000");
+        std::env::set_var("KREUZBERG_CORS_ORIGINS", "https://api.example.com");
+    }
+
+    let mut config = ServerConfig::default();
+    config.apply_env_overrides().unwrap();
+
+    assert_eq!(config.host, "0.0.0.0");
+    assert_eq!(config.port, 4000);
+    assert_eq!(config.cors_origins.len(), 1);
+    assert_eq!(config.cors_origins[0], "https://api.example.com");
+
+    // Cleanup
+    unsafe {
+        if let Some(orig) = host_orig {
+            std::env::set_var("KREUZBERG_HOST", orig);
+        } else {
+            std::env::remove_var("KREUZBERG_HOST");
+        }
+        if let Some(orig) = port_orig {
+            std::env::set_var("KREUZBERG_PORT", orig);
+        } else {
+            std::env::remove_var("KREUZBERG_PORT");
+        }
+        if let Some(orig) = cors_orig {
+            std::env::set_var("KREUZBERG_CORS_ORIGINS", orig);
+        } else {
+            std::env::remove_var("KREUZBERG_CORS_ORIGINS");
+        }
+    }
+}

crates/kreuzberg/src/core/server_config/tests/file_loading_tests.rs

@@ -0,0 +1,321 @@
+//! Tests for file loading functionality.
+
+use crate::core::ServerConfig;
+use std::fs;
+use tempfile::tempdir;
+
+#[test]
+fn test_from_toml_file() {
+    let dir = tempdir().unwrap();
+    let config_path = dir.path().join("server.toml");
+
+    fs::write(
+        &config_path,
+        r#"
+host = "0.0.0.0"
+port = 3000
+cors_origins = ["https://example.com", "https://other.com"]
+max_request_body_bytes = 50000000
+max_multipart_field_bytes = 75000000
+        "#,
+    )
+    .unwrap();
+
+    let config = ServerConfig::from_toml_file(&config_path).unwrap();
+    assert_eq!(config.host, "0.0.0.0");
+    assert_eq!(config.port, 3000);
+    assert_eq!(config.cors_origins.len(), 2);
+    assert_eq!(config.max_request_body_bytes, 50_000_000);
+    assert_eq!(config.max_multipart_field_bytes, 75_000_000);
+}
+
+#[test]
+fn test_from_yaml_file() {
+    let dir = tempdir().unwrap();
+    let config_path = dir.path().join("server.yaml");
+
+    fs::write(
+        &config_path,
+        r#"
+host: 0.0.0.0
+port: 3000
+cors_origins:
+  - https://example.com
+  - https://other.com
+max_request_body_bytes: 50000000
+max_multipart_field_bytes: 75000000
+        "#,
+    )
+    .unwrap();
+
+    let config = ServerConfig::from_yaml_file(&config_path).unwrap();
+    assert_eq!(config.host, "0.0.0.0");
+    assert_eq!(config.port, 3000);
+    assert_eq!(config.cors_origins.len(), 2);
+    assert_eq!(config.max_request_body_bytes, 50_000_000);
+    assert_eq!(config.max_multipart_field_bytes, 75_000_000);
+}
+
+#[test]
+fn test_from_json_file() {
+    let dir = tempdir().unwrap();
+    let config_path = dir.path().join("server.json");
+
+    fs::write(
+        &config_path,
+        r#"{
+  "host": "0.0.0.0",
+  "port": 3000,
+  "cors_origins": ["https://example.com", "https://other.com"],
+  "max_request_body_bytes": 50000000,
+  "max_multipart_field_bytes": 75000000
+}
+        "#,
+    )
+    .unwrap();
+
+    let config = ServerConfig::from_json_file(&config_path).unwrap();
+    assert_eq!(config.host, "0.0.0.0");
+    assert_eq!(config.port, 3000);
+    assert_eq!(config.cors_origins.len(), 2);
+    assert_eq!(config.max_request_body_bytes, 50_000_000);
+    assert_eq!(config.max_multipart_field_bytes, 75_000_000);
+}
+
+#[test]
+fn test_from_file_auto_detects_toml() {
+    let dir = tempdir().unwrap();
+    let config_path = dir.path().join("server.toml");
+
+    fs::write(
+        &config_path,
+        r#"
+host = "0.0.0.0"
+port = 3000
+        "#,
+    )
+    .unwrap();
+
+    let config = ServerConfig::from_file(&config_path).unwrap();
+    assert_eq!(config.host, "0.0.0.0");
+    assert_eq!(config.port, 3000);
+}
+
+#[test]
+fn test_from_file_auto_detects_yaml() {
+    let dir = tempdir().unwrap();
+    let config_path = dir.path().join("server.yaml");
+
+    fs::write(
+        &config_path,
+        r#"
+host: 0.0.0.0
+port: 3000
+        "#,
+    )
+    .unwrap();
+
+    let config = ServerConfig::from_file(&config_path).unwrap();
+    assert_eq!(config.host, "0.0.0.0");
+    assert_eq!(config.port, 3000);
+}
+
+#[test]
+fn test_from_file_auto_detects_json() {
+    let dir = tempdir().unwrap();
+    let config_path = dir.path().join("server.json");
+
+    fs::write(&config_path, r#"{"host": "0.0.0.0", "port": 3000}"#).unwrap();
+
+    let config = ServerConfig::from_file(&config_path).unwrap();
+    assert_eq!(config.host, "0.0.0.0");
+    assert_eq!(config.port, 3000);
+}
+
+#[test]
+fn test_from_file_unsupported_extension() {
+    let dir = tempdir().unwrap();
+    let config_path = dir.path().join("server.txt");
+
+    fs::write(&config_path, "host = 0.0.0.0").unwrap();
+
+    let result = ServerConfig::from_file(&config_path);
+    assert!(result.is_err());
+    assert!(
+        result
+            .unwrap_err()
+            .to_string()
+            .contains("Unsupported config file format")
+    );
+}
+
+#[test]
+fn test_from_file_no_extension() {
+    let dir = tempdir().unwrap();
+    let config_path = dir.path().join("server");
+
+    fs::write(&config_path, "host = 0.0.0.0").unwrap();
+
+    let result = ServerConfig::from_file(&config_path);
+    assert!(result.is_err());
+    assert!(result.unwrap_err().to_string().contains("no extension found"));
+}
+
+#[test]
+fn test_cors_origins_empty_in_toml() {
+    let dir = tempdir().unwrap();
+    let config_path = dir.path().join("server.toml");
+
+    fs::write(
+        &config_path,
+        r#"
+host = "127.0.0.1"
+port = 8000
+        "#,
+    )
+    .unwrap();
+
+    let config = ServerConfig::from_toml_file(&config_path).unwrap();
+    assert!(config.cors_origins.is_empty());
+    assert!(config.cors_allows_all());
+}
+
+#[test]
+fn test_full_configuration_toml() {
+    let dir = tempdir().unwrap();
+    let config_path = dir.path().join("server.toml");
+
+    fs::write(
+        &config_path,
+        r#"
+host = "192.168.1.100"
+port = 9000
+cors_origins = ["https://app1.com", "https://app2.com", "https://app3.com"]
+max_request_body_bytes = 200000000
+max_multipart_field_bytes = 150000000
+        "#,
+    )
+    .unwrap();
+
+    let config = ServerConfig::from_toml_file(&config_path).unwrap();
+    assert_eq!(config.host, "192.168.1.100");
+    assert_eq!(config.port, 9000);
+    assert_eq!(config.listen_addr(), "192.168.1.100:9000");
+    assert_eq!(config.cors_origins.len(), 3);
+    assert!(!config.cors_allows_all());
+    assert!(config.is_origin_allowed("https://app1.com"));
+    assert!(!config.is_origin_allowed("https://app4.com"));
+    assert_eq!(config.max_request_body_bytes, 200_000_000);
+    assert_eq!(config.max_multipart_field_bytes, 150_000_000);
+    assert_eq!(config.max_request_body_mb(), 191);
+    assert_eq!(config.max_multipart_field_mb(), 144);
+}
+
+#[test]
+fn test_from_file_with_nested_server_section_toml() {
+    let dir = tempdir().unwrap();
+    let config_path = dir.path().join("kreuzberg.toml");
+
+    // Config file with [server] section and other sections (like ExtractionConfig)
+    fs::write(
+        &config_path,
+        r#"
+[server]
+host = "0.0.0.0"
+port = 3000
+cors_origins = ["https://example.com"]
+
+[ocr]
+backend = "tesseract"
+language = "eng"
+
+[extraction]
+enabled = true
+        "#,
+    )
+    .unwrap();
+
+    let config = ServerConfig::from_file(&config_path).unwrap();
+    assert_eq!(config.host, "0.0.0.0");
+    assert_eq!(config.port, 3000);
+    assert_eq!(config.cors_origins.len(), 1);
+    assert_eq!(config.cors_origins[0], "https://example.com");
+}
+
+#[test]
+fn test_from_file_with_nested_server_section_yaml() {
+    let dir = tempdir().unwrap();
+    let config_path = dir.path().join("kreuzberg.yaml");
+
+    // Config file with server: section and other sections
+    fs::write(
+        &config_path,
+        r#"
+server:
+  host: 0.0.0.0
+  port: 4000
+  cors_origins:
+    - https://example.com
+
+ocr:
+  backend: tesseract
+  language: eng
+        "#,
+    )
+    .unwrap();
+
+    let config = ServerConfig::from_file(&config_path).unwrap();
+    assert_eq!(config.host, "0.0.0.0");
+    assert_eq!(config.port, 4000);
+    assert_eq!(config.cors_origins.len(), 1);
+}
+
+#[test]
+fn test_from_file_with_nested_server_section_json() {
+    let dir = tempdir().unwrap();
+    let config_path = dir.path().join("kreuzberg.json");
+
+    // Config file with "server" key and other sections
+    fs::write(
+        &config_path,
+        r#"
+{
+  "server": {
+    "host": "0.0.0.0",
+    "port": 5000,
+    "cors_origins": ["https://example.com"]
+  },
+  "ocr": {
+    "backend": "tesseract",
+    "language": "eng"
+  }
+}
+        "#,
+    )
+    .unwrap();
+
+    let config = ServerConfig::from_file(&config_path).unwrap();
+    assert_eq!(config.host, "0.0.0.0");
+    assert_eq!(config.port, 5000);
+    assert_eq!(config.cors_origins.len(), 1);
+}
+
+#[test]
+fn test_from_file_flat_format_still_works() {
+    let dir = tempdir().unwrap();
+    let config_path = dir.path().join("server.toml");
+
+    // Old flat format without [server] section
+    fs::write(
+        &config_path,
+        r#"
+host = "192.168.1.1"
+port = 6000
+        "#,
+    )
+    .unwrap();
+
+    let config = ServerConfig::from_file(&config_path).unwrap();
+    assert_eq!(config.host, "192.168.1.1");
+    assert_eq!(config.port, 6000);
+}

crates/kreuzberg/src/core/server_config/tests/mod.rs

@@ -0,0 +1,5 @@
+//! Tests for server configuration module.
+
+mod basic_tests;
+mod env_tests;
+mod file_loading_tests;

crates/kreuzberg/src/core/server_config/validation.rs

@@ -0,0 +1,4 @@
+//! Validation and normalization for server configuration.
+//!
+//! This module provides functionality to validate and normalize server configuration
+//! values.

crates/kreuzberg/src/diff/mod.rs

@@ -0,0 +1,542 @@
+//! Diff two [`ExtractionResult`] values.
+//!
+//! This module is gated behind the `diff` Cargo feature. Enable it by adding
+//! `kreuzberg = { features = ["diff"] }` to your `Cargo.toml`.
+//!
+//! # Example
+//!
+//! ```rust,no_run
+//! use kreuzberg::{ExtractionResult, diff::{compare, DiffOptions}};
+//!
+//! # fn main() {
+//! let a = ExtractionResult::default();
+//! let b = ExtractionResult::default();
+//! let opts = DiffOptions::default();
+//! let result = compare(&a, &b, &opts);
+//! assert!(result.content_diff.is_empty());
+//! # }
+//! ```
+
+pub mod types;
+
+pub use types::{
+    CellChange, DiffHunk, DiffLine, DiffOptions, EmbeddedChanges, EmbeddedDiff, ExtractionDiff, TableDiff,
+};
+
+use similar::{ChangeTag, DiffOp, TextDiff};
+
+use crate::types::extraction::{ArchiveEntry, ExtractionResult};
+use crate::types::tables::Table;
+
+/// Default number of context lines on each side of a changed region.
+const CONTEXT_LINES: usize = 3;
+
+/// Compare two extraction results and return a structured diff.
+///
+/// The comparison is purely structural — no I/O, no side effects. All fields
+/// of [`ExtractionDiff`] are populated according to the provided [`DiffOptions`].
+///
+/// # Arguments
+///
+/// * `a` — the "before" extraction result
+/// * `b` — the "after" extraction result
+/// * `opts` — controls which sections are compared and optional truncation
+///
+/// # Example
+///
+/// ```rust,no_run
+/// use kreuzberg::{ExtractionResult, diff::{compare, DiffOptions}};
+///
+/// # fn main() {
+/// let mut a = ExtractionResult::default();
+/// let mut b = ExtractionResult::default();
+/// a.content = "Hello world".to_string();
+/// b.content = "Hello Rust".to_string();
+///
+/// let diff = compare(&a, &b, &DiffOptions::default());
+/// assert_eq!(diff.content_diff.len(), 1);
+/// # }
+/// ```
+pub fn compare(a: &ExtractionResult, b: &ExtractionResult, opts: &DiffOptions) -> ExtractionDiff {
+    let content_diff = diff_content(&a.content, &b.content, opts);
+    let (tables_added, tables_removed, tables_changed) = diff_tables(&a.tables, &b.tables);
+    let metadata_changed = if opts.include_metadata {
+        diff_metadata(&a.metadata, &b.metadata)
+    } else {
+        serde_json::Value::Null
+    };
+    let embedded_changes = if opts.include_embedded {
+        diff_embedded(a.children.as_deref(), b.children.as_deref(), opts)
+    } else {
+        EmbeddedChanges {
+            added: vec![],
+            removed: vec![],
+            changed: vec![],
+        }
+    };
+
+    ExtractionDiff {
+        content_diff,
+        tables_added,
+        tables_removed,
+        tables_changed,
+        metadata_changed,
+        embedded_changes,
+    }
+}
+
+// ── Content diff ─────────────────────────────────────────────────────────────
+
+fn diff_content(a: &str, b: &str, opts: &DiffOptions) -> Vec<DiffHunk> {
+    let a_text = apply_truncation(a, opts.max_content_chars);
+    let b_text = apply_truncation(b, opts.max_content_chars);
+    let a_ref: &str = a_text.as_deref().unwrap_or(a);
+    let b_ref: &str = b_text.as_deref().unwrap_or(b);
+
+    let text_diff = TextDiff::from_lines(a_ref, b_ref);
+
+    if text_diff.ratio() == 1.0 {
+        return vec![];
+    }
+
+    let mut hunks = Vec::new();
+    for group in text_diff.grouped_ops(CONTEXT_LINES) {
+        let hunk_from_line = hunk_old_start(&group);
+        let hunk_to_line = hunk_new_start(&group);
+        let hunk_from_count = hunk_old_len(&group);
+        let hunk_to_count = hunk_new_len(&group);
+        let mut lines = Vec::new();
+
+        for op in &group {
+            for change in text_diff.iter_changes(op) {
+                let text = change.value().trim_end_matches('\n').to_string();
+                let line = match change.tag() {
+                    ChangeTag::Equal => DiffLine::Context(text),
+                    ChangeTag::Insert => DiffLine::Added(text),
+                    ChangeTag::Delete => DiffLine::Removed(text),
+                };
+                lines.push(line);
+            }
+        }
+
+        if !lines.is_empty() {
+            hunks.push(DiffHunk {
+                from_line: hunk_from_line,
+                from_count: hunk_from_count,
+                to_line: hunk_to_line,
+                to_count: hunk_to_count,
+                lines,
+            });
+        }
+    }
+    hunks
+}
+
+fn hunk_old_start(ops: &[DiffOp]) -> usize {
+    ops.first().map_or(0, |op| op.old_range().start)
+}
+
+fn hunk_new_start(ops: &[DiffOp]) -> usize {
+    ops.first().map_or(0, |op| op.new_range().start)
+}
+
+fn hunk_old_len(ops: &[DiffOp]) -> usize {
+    let start = ops.first().map_or(0, |op| op.old_range().start);
+    let end = ops.last().map_or(0, |op| op.old_range().end);
+    end.saturating_sub(start)
+}
+
+fn hunk_new_len(ops: &[DiffOp]) -> usize {
+    let start = ops.first().map_or(0, |op| op.new_range().start);
+    let end = ops.last().map_or(0, |op| op.new_range().end);
+    end.saturating_sub(start)
+}
+
+fn apply_truncation(text: &str, limit: Option<usize>) -> Option<String> {
+    limit.map(|n| {
+        let mut boundary = n.min(text.len());
+        while !text.is_char_boundary(boundary) {
+            boundary -= 1;
+        }
+        text[..boundary].to_string()
+    })
+}
+
+// ── Table diff ────────────────────────────────────────────────────────────────
+
+fn diff_tables(a_tables: &[Table], b_tables: &[Table]) -> (Vec<Table>, Vec<Table>, Vec<TableDiff>) {
+    let min_len = a_tables.len().min(b_tables.len());
+    let mut tables_changed = Vec::new();
+
+    for idx in 0..min_len {
+        let a_t = &a_tables[idx];
+        let b_t = &b_tables[idx];
+
+        if tables_same_shape(a_t, b_t) {
+            let cell_changes = diff_cells(a_t, b_t);
+            if !cell_changes.is_empty() {
+                tables_changed.push(TableDiff {
+                    from_index: idx,
+                    to_index: idx,
+                    cell_changes,
+                });
+            }
+        } else {
+            // Different shape — treat the pair as remove + add.
+            // The "removed" side is reported in tables_removed and "added" in tables_added.
+            // We handle this by falling through to the asymmetric slice handling below.
+            // But we need to signal that these shouldn't be counted as "paired" — so we
+            // emit them as add + remove even though they share the same index.
+            tables_changed.push(TableDiff {
+                from_index: idx,
+                to_index: idx,
+                // No cell-level changes: shapes differ; report as a structural replacement.
+                cell_changes: vec![],
+            });
+        }
+    }
+
+    let tables_removed: Vec<Table> = if a_tables.len() > b_tables.len() {
+        a_tables[min_len..].to_vec()
+    } else {
+        vec![]
+    };
+    let tables_added: Vec<Table> = if b_tables.len() > a_tables.len() {
+        b_tables[min_len..].to_vec()
+    } else {
+        vec![]
+    };
+
+    (tables_added, tables_removed, tables_changed)
+}
+
+/// Two tables are considered the same shape if and only if their row and column counts match.
+///
+/// Header content is NOT compared — column reordering with the same dimensions will produce
+/// per-cell `CellChange` entries for every cell whose value differs, not a structural replacement.
+///
+/// TODO: smarter shape-matching that aligns tables by header names (instead of positional
+/// index) is a follow-up; for now dimensions-only is the v1 default.
+fn tables_same_shape(a: &Table, b: &Table) -> bool {
+    if a.cells.len() != b.cells.len() {
+        return false;
+    }
+    let a_cols = a.cells.first().map_or(0, Vec::len);
+    let b_cols = b.cells.first().map_or(0, Vec::len);
+    a_cols == b_cols
+}
+
+fn diff_cells(a: &Table, b: &Table) -> Vec<CellChange> {
+    let mut changes = Vec::new();
+    for (row_idx, (a_row, b_row)) in a.cells.iter().zip(b.cells.iter()).enumerate() {
+        for (col_idx, (a_cell, b_cell)) in a_row.iter().zip(b_row.iter()).enumerate() {
+            if a_cell != b_cell {
+                changes.push(CellChange {
+                    row: row_idx,
+                    col: col_idx,
+                    from: a_cell.clone(),
+                    to: b_cell.clone(),
+                });
+            }
+        }
+    }
+    changes
+}
+
+// ── Metadata diff ─────────────────────────────────────────────────────────────
+
+fn diff_metadata(a: &crate::types::metadata::Metadata, b: &crate::types::metadata::Metadata) -> serde_json::Value {
+    let a_val = serde_json::to_value(a).unwrap_or(serde_json::Value::Null);
+    let b_val = serde_json::to_value(b).unwrap_or(serde_json::Value::Null);
+
+    let a_obj = a_val.as_object().cloned().unwrap_or_default();
+    let b_obj = b_val.as_object().cloned().unwrap_or_default();
+
+    let mut added = serde_json::Map::new();
+    let mut removed = serde_json::Map::new();
+    let mut changed = serde_json::Map::new();
+
+    for (key, b_value) in &b_obj {
+        match a_obj.get(key) {
+            None => {
+                added.insert(key.clone(), b_value.clone());
+            }
+            Some(a_value) if a_value != b_value => {
+                changed.insert(key.clone(), serde_json::json!({ "from": a_value, "to": b_value }));
+            }
+            _ => {}
+        }
+    }
+    for (key, a_value) in &a_obj {
+        if !b_obj.contains_key(key) {
+            removed.insert(key.clone(), a_value.clone());
+        }
+    }
+
+    serde_json::json!({ "added": added, "removed": removed, "changed": changed })
+}
+
+// ── Embedded diff ─────────────────────────────────────────────────────────────
+
+fn diff_embedded(
+    a_children: Option<&[ArchiveEntry]>,
+    b_children: Option<&[ArchiveEntry]>,
+    opts: &DiffOptions,
+) -> EmbeddedChanges {
+    let a_entries = a_children.unwrap_or(&[]);
+    let b_entries = b_children.unwrap_or(&[]);
+
+    let mut added = Vec::new();
+    let mut removed = Vec::new();
+    let mut changed = Vec::new();
+
+    for b_entry in b_entries {
+        match a_entries.iter().find(|e| e.path == b_entry.path) {
+            None => added.push(b_entry.clone()),
+            Some(a_entry) => {
+                let child_diff = compare(&a_entry.result, &b_entry.result, opts);
+                if is_nonempty_diff(&child_diff) {
+                    changed.push(EmbeddedDiff {
+                        path: b_entry.path.clone(),
+                        diff: Box::new(child_diff),
+                    });
+                }
+            }
+        }
+    }
+    for a_entry in a_entries {
+        if !b_entries.iter().any(|e| e.path == a_entry.path) {
+            removed.push(a_entry.clone());
+        }
+    }
+
+    EmbeddedChanges {
+        added,
+        removed,
+        changed,
+    }
+}
+
+fn is_nonempty_diff(diff: &ExtractionDiff) -> bool {
+    !diff.content_diff.is_empty()
+        || !diff.tables_added.is_empty()
+        || !diff.tables_removed.is_empty()
+        || !diff.tables_changed.is_empty()
+        || !diff.embedded_changes.added.is_empty()
+        || !diff.embedded_changes.removed.is_empty()
+        || !diff.embedded_changes.changed.is_empty()
+        || is_nonempty_metadata_diff(&diff.metadata_changed)
+}
+
+fn is_nonempty_metadata_diff(val: &serde_json::Value) -> bool {
+    if val.is_null() {
+        return false;
+    }
+    let empty_obj = serde_json::json!({ "added": {}, "removed": {}, "changed": {} });
+    val != &empty_obj
+}
+
+// ── Tests ─────────────────────────────────────────────────────────────────────
+
+#[cfg(all(test, feature = "diff"))]
+mod tests {
+    use super::*;
+    use crate::types::{extraction::ExtractionResult, tables::Table};
+
+    fn empty_result() -> ExtractionResult {
+        ExtractionResult::default()
+    }
+
+    fn result_with_content(content: &str) -> ExtractionResult {
+        ExtractionResult {
+            content: content.to_string(),
+            ..Default::default()
+        }
+    }
+
+    fn result_with_tables(tables: Vec<Table>) -> ExtractionResult {
+        ExtractionResult {
+            tables,
+            ..Default::default()
+        }
+    }
+
+    fn simple_table(cells: Vec<Vec<&str>>) -> Table {
+        Table {
+            cells: cells
+                .into_iter()
+                .map(|row| row.into_iter().map(str::to_string).collect())
+                .collect(),
+            markdown: String::new(),
+            page_number: 1,
+            bounding_box: None,
+        }
+    }
+
+    // ── identical inputs ──────────────────────────────────────────────────────
+
+    #[test]
+    fn should_produce_empty_diff_for_identical_inputs() {
+        let a = empty_result();
+        let b = empty_result();
+        let diff = compare(&a, &b, &DiffOptions::default());
+
+        assert!(diff.content_diff.is_empty());
+        assert!(diff.tables_added.is_empty());
+        assert!(diff.tables_removed.is_empty());
+        assert!(diff.tables_changed.is_empty());
+        assert!(diff.embedded_changes.added.is_empty());
+        assert!(diff.embedded_changes.removed.is_empty());
+        assert!(diff.embedded_changes.changed.is_empty());
+    }
+
+    #[test]
+    fn should_produce_empty_diff_for_both_empty_results() {
+        let diff = compare(
+            &ExtractionResult::default(),
+            &ExtractionResult::default(),
+            &DiffOptions::default(),
+        );
+        assert!(!is_nonempty_diff(&diff));
+    }
+
+    // ── content diff ─────────────────────────────────────────────────────────
+
+    #[test]
+    fn should_produce_one_hunk_for_single_line_change() {
+        let a = result_with_content("Hello world");
+        let b = result_with_content("Hello Rust");
+        let diff = compare(&a, &b, &DiffOptions::default());
+
+        assert_eq!(diff.content_diff.len(), 1, "expected exactly one hunk");
+        let hunk = &diff.content_diff[0];
+        let has_removed = hunk
+            .lines
+            .iter()
+            .any(|l| matches!(l, DiffLine::Removed(t) if t == "Hello world"));
+        let has_added = hunk
+            .lines
+            .iter()
+            .any(|l| matches!(l, DiffLine::Added(t) if t == "Hello Rust"));
+        assert!(has_removed, "expected 'Hello world' as Removed line");
+        assert!(has_added, "expected 'Hello Rust' as Added line");
+    }
+
+    #[test]
+    fn should_report_correct_line_numbers_for_single_line_change() {
+        let a = result_with_content("line one\nline two\nline three");
+        let b = result_with_content("line one\nline TWO\nline three");
+        let diff = compare(&a, &b, &DiffOptions::default());
+
+        assert_eq!(diff.content_diff.len(), 1);
+        let hunk = &diff.content_diff[0];
+        // With 3-line context the hunk expands to include surrounding lines.
+        // Three-line text with change at line 1 (0-indexed): context pulls the
+        // hunk start back to line 0 (beginning of file).
+        assert_eq!(hunk.from_line, 0);
+        assert_eq!(hunk.to_line, 0);
+        // All 3 lines appear: one context, one changed, one context.
+        assert_eq!(hunk.from_count, 3);
+        assert_eq!(hunk.to_count, 3);
+        // The hunk must contain the changed lines.
+        let has_removed = hunk
+            .lines
+            .iter()
+            .any(|l| matches!(l, DiffLine::Removed(t) if t == "line two"));
+        let has_added = hunk
+            .lines
+            .iter()
+            .any(|l| matches!(l, DiffLine::Added(t) if t == "line TWO"));
+        assert!(has_removed, "expected 'line two' as Removed line");
+        assert!(has_added, "expected 'line TWO' as Added line");
+    }
+
+    #[test]
+    fn should_produce_empty_content_diff_when_content_identical_but_tables_differ() {
+        let mut a = result_with_tables(vec![simple_table(vec![vec!["A", "B"]])]);
+        a.content = "same text".to_string();
+        let mut b = result_with_tables(vec![simple_table(vec![vec!["A", "C"]])]);
+        b.content = "same text".to_string();
+
+        let diff = compare(&a, &b, &DiffOptions::default());
+        assert!(
+            diff.content_diff.is_empty(),
+            "content is identical; no content hunks expected"
+        );
+        assert!(!diff.tables_changed.is_empty(), "table change expected");
+    }
+
+    // ── table diff ───────────────────────────────────────────────────────────
+
+    #[test]
+    fn should_detect_single_cell_change_in_same_table() {
+        let a = result_with_tables(vec![simple_table(vec![vec!["A", "B"], vec!["C", "D"]])]);
+        let b = result_with_tables(vec![simple_table(vec![vec!["A", "B"], vec!["C", "X"]])]);
+        let diff = compare(&a, &b, &DiffOptions::default());
+
+        assert_eq!(diff.tables_changed.len(), 1);
+        let table_diff = &diff.tables_changed[0];
+        assert_eq!(table_diff.cell_changes.len(), 1);
+        let change = &table_diff.cell_changes[0];
+        assert_eq!(change.row, 1);
+        assert_eq!(change.col, 1);
+        assert_eq!(change.from, "D");
+        assert_eq!(change.to, "X");
+    }
+
+    #[test]
+    fn should_put_extra_table_in_tables_added() {
+        let a = result_with_tables(vec![simple_table(vec![vec!["A"]])]);
+        let b = result_with_tables(vec![simple_table(vec![vec!["A"]]), simple_table(vec![vec!["NEW"]])]);
+        let diff = compare(&a, &b, &DiffOptions::default());
+
+        assert_eq!(diff.tables_added.len(), 1);
+        assert_eq!(diff.tables_added[0].cells[0][0], "NEW");
+        assert!(diff.tables_removed.is_empty());
+    }
+
+    #[test]
+    fn should_put_missing_table_in_tables_removed() {
+        let a = result_with_tables(vec![simple_table(vec![vec!["A"]]), simple_table(vec![vec!["OLD"]])]);
+        let b = result_with_tables(vec![simple_table(vec![vec!["A"]])]);
+        let diff = compare(&a, &b, &DiffOptions::default());
+
+        assert_eq!(diff.tables_removed.len(), 1);
+        assert_eq!(diff.tables_removed[0].cells[0][0], "OLD");
+        assert!(diff.tables_added.is_empty());
+    }
+
+    // ── embedded diff ─────────────────────────────────────────────────────────
+
+    #[test]
+    fn should_detect_added_embedded_child() {
+        let a = empty_result();
+        let mut b = empty_result();
+        b.children = Some(vec![ArchiveEntry {
+            path: "doc.txt".to_string(),
+            mime_type: "text/plain".to_string(),
+            result: Box::new(result_with_content("hello")),
+        }]);
+
+        let diff = compare(&a, &b, &DiffOptions::default());
+        assert_eq!(diff.embedded_changes.added.len(), 1);
+        assert_eq!(diff.embedded_changes.added[0].path, "doc.txt");
+        assert!(diff.embedded_changes.removed.is_empty());
+    }
+
+    #[test]
+    fn should_detect_removed_embedded_child() {
+        let mut a = empty_result();
+        a.children = Some(vec![ArchiveEntry {
+            path: "old.txt".to_string(),
+            mime_type: "text/plain".to_string(),
+            result: Box::new(result_with_content("old")),
+        }]);
+        let b = empty_result();
+
+        let diff = compare(&a, &b, &DiffOptions::default());
+        assert_eq!(diff.embedded_changes.removed.len(), 1);
+        assert_eq!(diff.embedded_changes.removed[0].path, "old.txt");
+        assert!(diff.embedded_changes.added.is_empty());
+    }
+}

crates/kreuzberg/src/diff/types.rs

@@ -0,0 +1,127 @@
+//! Types for extraction result diffs.
+//!
+//! `DiffLine` and `CellChange` are canonical definitions live in
+//! `crate::types::revisions` so that `RevisionDelta` can reference them
+//! unconditionally without the `diff` feature gate. They are re-exported
+//! here so the `crate::diff::DiffLine` path continues to work for callers
+//! who import them through the `diff` feature.
+
+use serde::{Deserialize, Serialize};
+
+use crate::types::{extraction::ArchiveEntry, tables::Table};
+
+// Re-export from the unconditional types module so the `diff` feature's
+// public path (`kreuzberg::diff::DiffLine`, `kreuzberg::diff::CellChange`)
+// remains stable. The canonical definitions are in `crate::types::revisions`.
+pub use crate::types::revisions::{CellChange, DiffLine};
+
+/// Options controlling how two `ExtractionResult` values are compared.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct DiffOptions {
+    /// Include metadata changes in the diff. Default: `true`.
+    pub include_metadata: bool,
+    /// Include embedded-children changes in the diff. Default: `true`.
+    pub include_embedded: bool,
+    /// Truncate content to this many characters before diffing.
+    ///
+    /// Useful for very large documents where only the first N characters matter.
+    /// `None` means no truncation.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub max_content_chars: Option<usize>,
+}
+
+impl Default for DiffOptions {
+    fn default() -> Self {
+        Self {
+            include_metadata: true,
+            include_embedded: true,
+            max_content_chars: None,
+        }
+    }
+}
+
+/// The complete diff between two `ExtractionResult` values.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct ExtractionDiff {
+    /// Unified-diff hunks for the `content` field.
+    ///
+    /// Empty when the content is identical.
+    pub content_diff: Vec<DiffHunk>,
+
+    /// Tables present in `b` but not in `a` (by index position, excess right-side tables).
+    pub tables_added: Vec<Table>,
+
+    /// Tables present in `a` but not in `b` (by index position, excess left-side tables).
+    pub tables_removed: Vec<Table>,
+
+    /// Cell-level changes for table pairs that share the same index and dimensions.
+    pub tables_changed: Vec<TableDiff>,
+
+    /// Metadata difference, encoded as a JSON object with three top-level keys:
+    /// `added` (keys present in `b` but not `a`), `removed` (keys present in `a`
+    /// but not `b`), and `changed` (keys whose values differ — each entry is
+    /// `{ "from": <value-in-a>, "to": <value-in-b> }`).
+    ///
+    /// This is NOT RFC 6902 JSON Patch — we deliberately chose a flatter shape
+    /// to avoid pulling in a json-patch crate. If you need RFC 6902 semantics
+    /// (with JSON Pointer paths) feed `a.metadata` and `b.metadata` to your
+    /// preferred json-patch impl directly.
+    pub metadata_changed: serde_json::Value,
+
+    /// Changes to embedded archive children.
+    pub embedded_changes: EmbeddedChanges,
+}
+
+/// A single contiguous hunk in a unified diff.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct DiffHunk {
+    /// Starting line number in the old content (0-indexed).
+    pub from_line: usize,
+    /// Number of lines from the old content in this hunk.
+    pub from_count: usize,
+    /// Starting line number in the new content (0-indexed).
+    pub to_line: usize,
+    /// Number of lines from the new content in this hunk.
+    pub to_count: usize,
+    /// Lines that make up this hunk.
+    pub lines: Vec<DiffLine>,
+}
+
+/// Cell-level changes for a pair of tables that share the same index.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct TableDiff {
+    /// Zero-based index of the table in both `a.tables` and `b.tables`.
+    pub from_index: usize,
+    /// Zero-based index in `b.tables` (equal to `from_index` for same-dimension tables).
+    pub to_index: usize,
+    /// Cell-level changes within the table.
+    pub cell_changes: Vec<CellChange>,
+}
+
+/// Changes to embedded archive children between two results.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct EmbeddedChanges {
+    /// Children present in `b` but not in `a` (matched by `path`).
+    pub added: Vec<ArchiveEntry>,
+    /// Children present in `a` but not in `b` (matched by `path`).
+    pub removed: Vec<ArchiveEntry>,
+    /// Children present in both but with differing content (matched by `path`).
+    ///
+    /// Each entry holds the diff of the nested `ExtractionResult`.
+    pub changed: Vec<EmbeddedDiff>,
+}
+
+/// Diff for a single embedded archive entry that appears in both results.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
+pub struct EmbeddedDiff {
+    /// Archive-relative path identifying this entry.
+    pub path: String,
+    /// The recursive diff of the entry's extraction result.
+    pub diff: Box<ExtractionDiff>,
+}

crates/kreuzberg/src/doc_orientation/detector.rs

@@ -0,0 +1,310 @@
+//! Document orientation detection implementation using PP-LCNet_x1_0_doc_ori.
+//!
+//! Detects page-level orientation (0°, 90°, 180°, 270°) for scanned documents
+//! and images. Requires the `auto-rotate` feature (ONNX Runtime).
+//!
+//! Used by ALL OCR backends when `auto_rotate` is enabled in `OcrConfig`.
+//! More reliable than Tesseract's `DetectOrientationScript` which crashes
+//! on raw images without DPI metadata.
+
+use std::fs;
+use std::path::PathBuf;
+
+use image::RgbImage;
+use ort::session::Session;
+use ort::session::builder::{GraphOptimizationLevel, SessionBuilder};
+use ort::value::Tensor;
+
+use crate::Result;
+use crate::error::KreuzbergError;
+
+use super::types::OrientationResult;
+
+/// HuggingFace repository containing the model.
+const HF_REPO_ID: &str = "Kreuzberg/paddleocr-onnx-models";
+const REMOTE_FILENAME: &str = "v2/classifiers/PP-LCNet_x1_0_doc_ori.onnx";
+const SHA256: &str = "6b742aebce6f0f7f71f747931ac7becfc7c96c51641e14943b291eeb334e7947";
+
+// PP-LCNet preprocessing constants.
+// Input: resize short side to 256, center crop 224×224, ImageNet normalize (BGR).
+const INPUT_SIZE: u32 = 224;
+const RESIZE_SHORT: u32 = 256;
+
+/// Output labels: index -> degrees.
+const ORIENTATION_LABELS: [u32; 4] = [0, 90, 180, 270];
+
+/// PP-LCNet doc_ori outputs ~45% confidence for correct class in a 4-class problem.
+/// Uniform baseline is 25%. A threshold of 0.35 provides good discrimination.
+pub const MIN_CONFIDENCE: f32 = 0.35;
+
+/// Detects document page orientation using the PP-LCNet model.
+///
+/// Thread-safe: uses unsafe pointer cast for ONNX session (same pattern as embedding engine).
+/// The model is downloaded from HuggingFace on first use and cached locally.
+#[cfg_attr(alef, alef(skip))]
+pub struct DocOrientationDetector {
+    session: once_cell::sync::OnceCell<Session>,
+    cache_dir: PathBuf,
+    acceleration: Option<crate::core::config::acceleration::AccelerationConfig>,
+}
+
+impl DocOrientationDetector {
+    /// Creates a new detector with the given cache directory and acceleration config.
+    pub(crate) fn with_acceleration(
+        cache_dir: PathBuf,
+        accel: Option<crate::core::config::acceleration::AccelerationConfig>,
+    ) -> Self {
+        Self {
+            session: once_cell::sync::OnceCell::new(),
+            cache_dir,
+            acceleration: accel,
+        }
+    }
+
+    /// Detect document page orientation.
+    ///
+    /// Returns the detected orientation (0°, 90°, 180°, 270°) and confidence.
+    /// Thread-safe: can be called concurrently from multiple pages.
+    pub(crate) fn detect(&self, image: &RgbImage) -> Result<OrientationResult> {
+        let session = self.get_or_init_session()?;
+
+        // Preprocess: resize short side to 256, center crop 224×224
+        let preprocessed = preprocess(image);
+
+        // Build input tensor: [1, 3, 224, 224]
+        let input_tensor = normalize(&preprocessed);
+        let tensor = Tensor::from_array(input_tensor).map_err(|e| KreuzbergError::Ocr {
+            message: format!("Failed to create doc_ori input tensor: {e}"),
+            source: None,
+        })?;
+
+        // SAFETY: ONNX Runtime C API is thread-safe for concurrent inference.
+        // The ort crate's &mut self on Session::run is overly conservative.
+        #[allow(unsafe_code)]
+        let outputs = unsafe {
+            let session_ptr = session as *const Session as *mut Session;
+            (*session_ptr).run(ort::inputs!["x" => tensor])
+        }
+        .map_err(|e| KreuzbergError::Ocr {
+            message: format!("Doc orientation inference failed: {e}"),
+            source: None,
+        })?;
+
+        // Parse output: argmax over 4 orientation classes
+        let (_, output_value) = outputs.iter().next().ok_or_else(|| KreuzbergError::Ocr {
+            message: "No output from doc orientation model".to_string(),
+            source: None,
+        })?;
+
+        let scores: Vec<f32> = output_value
+            .try_extract_tensor::<f32>()
+            .map_err(|e| KreuzbergError::Ocr {
+                message: format!("Failed to extract doc_ori output: {e}"),
+                source: None,
+            })?
+            .1
+            .to_vec();
+
+        // Softmax + argmax
+        let max_score = scores.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
+        let exp_scores: Vec<f32> = scores.iter().map(|&s| (s - max_score).exp()).collect();
+        let sum_exp: f32 = exp_scores.iter().sum();
+        let probabilities: Vec<f32> = exp_scores.iter().map(|&e| e / sum_exp).collect();
+
+        let (best_idx, &best_prob) = probabilities
+            .iter()
+            .enumerate()
+            .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
+            .unwrap_or((0, &0.0));
+
+        let degrees = ORIENTATION_LABELS.get(best_idx).copied().unwrap_or(0);
+
+        Ok(OrientationResult {
+            degrees,
+            confidence: best_prob,
+        })
+    }
+
+    /// Ensure the model is downloaded and return the ONNX file path.
+    fn ensure_model(&self) -> Result<PathBuf> {
+        let model_dir = self.cache_dir.join("doc-orientation");
+        let model_file = model_dir.join("model.onnx");
+
+        if model_file.exists() {
+            return Ok(model_file);
+        }
+
+        tracing::info!("Downloading document orientation model...");
+        fs::create_dir_all(&model_dir)?;
+
+        let cached_path =
+            crate::model_download::hf_download(HF_REPO_ID, REMOTE_FILENAME).map_err(|e| KreuzbergError::Plugin {
+                message: e,
+                plugin_name: "auto-rotate".to_string(),
+            })?;
+
+        crate::model_download::verify_sha256(&cached_path, SHA256, "doc_ori").map_err(|e| {
+            KreuzbergError::Validation {
+                message: e,
+                source: None,
+            }
+        })?;
+
+        fs::copy(&cached_path, &model_file).map_err(|e| KreuzbergError::Plugin {
+            message: format!("Failed to copy doc_ori model: {e}"),
+            plugin_name: "auto-rotate".to_string(),
+        })?;
+
+        tracing::info!("Document orientation model saved");
+        Ok(model_file)
+    }
+
+    /// Get or initialize the ONNX session (lazy, thread-safe via OnceCell).
+    fn get_or_init_session(&self) -> Result<&Session> {
+        self.session.get_or_try_init(|| {
+            let model_path = self.ensure_model()?;
+
+            crate::ort_discovery::ensure_ort_available();
+
+            let num_threads = crate::core::config::concurrency::resolve_thread_budget(None);
+            let builder = SessionBuilder::new()
+                .map_err(|e| KreuzbergError::Ocr {
+                    message: format!("Failed to create doc_ori session builder: {e}"),
+                    source: None,
+                })?
+                .with_optimization_level(GraphOptimizationLevel::All)
+                .map_err(|e| KreuzbergError::Ocr {
+                    message: format!("Failed to set doc_ori optimization level: {e}"),
+                    source: None,
+                })?
+                .with_intra_threads(num_threads)
+                .map_err(|e| KreuzbergError::Ocr {
+                    message: format!("Failed to set doc_ori thread count: {e}"),
+                    source: None,
+                })?
+                .with_inter_threads(1)
+                .map_err(|e| KreuzbergError::Ocr {
+                    message: format!("Failed to set doc_ori inter threads: {e}"),
+                    source: None,
+                })?;
+            let mut builder = crate::ort_discovery::apply_execution_providers(builder, self.acceleration.as_ref())
+                .map_err(|e| KreuzbergError::Ocr {
+                    message: format!("Failed to set doc_ori execution providers: {e}"),
+                    source: None,
+                })?;
+            let session = builder.commit_from_file(&model_path).map_err(|e| KreuzbergError::Ocr {
+                message: format!("Failed to load doc_ori model: {e}"),
+                source: None,
+            })?;
+
+            tracing::info!("Doc orientation model loaded");
+            Ok(session)
+        })
+    }
+}
+
+/// Resolve the cache directory for the auto-rotate model.
+pub(crate) fn resolve_cache_dir() -> PathBuf {
+    crate::cache_dir::resolve_cache_dir("auto-rotate")
+}
+
+/// Detect orientation and return a corrected image if rotation is needed.
+///
+/// Returns `Ok(Some(rotated_bytes))` if rotation was applied,
+/// `Ok(None)` if no rotation needed (0° or low confidence).
+#[cfg_attr(alef, alef(skip))]
+pub(crate) fn detect_and_rotate(detector: &DocOrientationDetector, image_bytes: &[u8]) -> Result<Option<Vec<u8>>> {
+    let img = image::load_from_memory(image_bytes)
+        .map_err(|e| KreuzbergError::Ocr {
+            message: format!("Failed to load image for orientation detection: {e}"),
+            source: None,
+        })?
+        .to_rgb8();
+
+    let result = detector.detect(&img)?;
+
+    tracing::debug!(
+        degrees = result.degrees,
+        confidence = result.confidence,
+        "Document orientation detected"
+    );
+
+    if result.degrees == 0 || result.confidence < MIN_CONFIDENCE {
+        return Ok(None);
+    }
+
+    // Rotate the image back to upright (opposite direction of detected orientation).
+    let rotated = match result.degrees {
+        90 => image::imageops::rotate270(&img),
+        180 => image::imageops::rotate180(&img),
+        270 => image::imageops::rotate90(&img),
+        _ => return Ok(None),
+    };
+
+    // Encode back to PNG bytes
+    let mut buf = std::io::Cursor::new(Vec::new());
+    rotated
+        .write_to(&mut buf, image::ImageFormat::Png)
+        .map_err(|e| KreuzbergError::Ocr {
+            message: format!("Failed to encode rotated image: {e}"),
+            source: None,
+        })?;
+
+    tracing::info!(
+        degrees = result.degrees,
+        confidence = result.confidence,
+        "Auto-rotated document page"
+    );
+
+    Ok(Some(buf.into_inner()))
+}
+
+/// Resize short side to 256, then center crop to 224×224.
+fn preprocess(image: &RgbImage) -> RgbImage {
+    let (w, h) = (image.width(), image.height());
+
+    // Resize: scale so short side = RESIZE_SHORT
+    let (new_w, new_h) = if w < h {
+        let scale = RESIZE_SHORT as f32 / w as f32;
+        (RESIZE_SHORT, (h as f32 * scale).round() as u32)
+    } else {
+        let scale = RESIZE_SHORT as f32 / h as f32;
+        ((w as f32 * scale).round() as u32, RESIZE_SHORT)
+    };
+
+    let resized = image::imageops::resize(image, new_w, new_h, image::imageops::FilterType::Triangle);
+
+    // Center crop to INPUT_SIZE × INPUT_SIZE
+    let x_offset = (new_w.saturating_sub(INPUT_SIZE)) / 2;
+    let y_offset = (new_h.saturating_sub(INPUT_SIZE)) / 2;
+    let crop_w = INPUT_SIZE.min(new_w);
+    let crop_h = INPUT_SIZE.min(new_h);
+
+    image::imageops::crop_imm(&resized, x_offset, y_offset, crop_w, crop_h).to_image()
+}
+
+/// Normalize image to [1, 3, H, W] tensor with ImageNet mean/std in BGR order.
+/// PP-LCNet expects BGR input: channel 0=Blue, 1=Green, 2=Red.
+fn normalize(image: &RgbImage) -> ndarray::Array4<f32> {
+    let (w, h) = (image.width() as usize, image.height() as usize);
+    let mut tensor = ndarray::Array4::<f32>::zeros((1, 3, h, w));
+
+    // ImageNet mean/std for BGR order (swap R and B)
+    const BGR_MEAN: [f32; 3] = [0.406 * 255.0, 0.456 * 255.0, 0.485 * 255.0];
+    const BGR_NORM: [f32; 3] = [1.0 / (0.225 * 255.0), 1.0 / (0.224 * 255.0), 1.0 / (0.229 * 255.0)];
+
+    for y in 0..h {
+        for x in 0..w {
+            let pixel = image.get_pixel(x as u32, y as u32);
+            let r = pixel[0] as f32;
+            let g = pixel[1] as f32;
+            let b = pixel[2] as f32;
+            // BGR order: channel 0 = Blue, channel 1 = Green, channel 2 = Red
+            tensor[[0, 0, y, x]] = (b - BGR_MEAN[0]) * BGR_NORM[0];
+            tensor[[0, 1, y, x]] = (g - BGR_MEAN[1]) * BGR_NORM[1];
+            tensor[[0, 2, y, x]] = (r - BGR_MEAN[2]) * BGR_NORM[2];
+        }
+    }
+
+    tensor
+}

crates/kreuzberg/src/doc_orientation/mod.rs

@@ -0,0 +1,15 @@
+//! Document orientation detection using PP-LCNet_x1_0_doc_ori.
+//!
+//! The `types` submodule is always available under the `auto-rotate-types` feature
+//! (pure-Rust, no ORT dependency). The full detection implementation requires the
+//! `auto-rotate` feature and ONNX Runtime.
+
+pub mod types;
+pub use types::OrientationResult;
+
+#[cfg(feature = "auto-rotate")]
+pub(crate) mod detector;
+#[cfg(feature = "auto-rotate")]
+pub use detector::{DocOrientationDetector, MIN_CONFIDENCE};
+#[cfg(feature = "auto-rotate")]
+pub(crate) use detector::{detect_and_rotate, resolve_cache_dir};

crates/kreuzberg/src/doc_orientation/types.rs

@@ -0,0 +1,10 @@
+//! Pure-Rust orientation result types — available without ONNX Runtime.
+
+/// Document orientation detection result.
+#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)]
+pub struct OrientationResult {
+    /// Detected orientation in degrees (0, 90, 180, or 270).
+    pub degrees: u32,
+    /// Confidence score (0.0-1.0).
+    pub confidence: f32,
+}

crates/kreuzberg/src/embeddings/engine.rs

@@ -0,0 +1,456 @@
+//! Vendored text embedding engine.
+//!
+//! Core inference pipeline for ONNX-based text embedding generation.
+//! Key design: `embed()` takes `&self` instead of `&mut self`, enabling
+//! concurrent inference from multiple threads without mutex contention.
+//!
+//! This is safe because `ort::Session::run()` takes `&mut self` purely as
+//! an API constraint — its internal `run_inner()` takes `&self`, and the
+//! ONNX Runtime C API (`OrtApi::Run`) is documented as thread-safe for
+//! concurrent calls on the same session.
+//!
+//! See ATTRIBUTIONS.md for original source attribution.
+
+use ndarray::{Array2, ArrayView, Dim, Dimension, IxDynImpl, s};
+use ort::session::Session;
+use ort::value::Value;
+use tokenizers::Tokenizer;
+#[cfg_attr(alef, alef(skip))]
+/// Pooling strategy for extracting a single vector from token embeddings.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Pooling {
+    /// Use the [CLS] token embedding (first token).
+    Cls,
+    /// Mean of all token embeddings, weighted by attention mask.
+    Mean,
+}
+#[cfg_attr(alef, alef(skip))]
+/// Text embedding model with thread-safe inference.
+///
+/// The `embed()` method takes `&self` instead of `&mut self`, allowing it to
+/// be shared across threads via `Arc<EmbeddingEngine>` without mutex contention.
+pub struct EmbeddingEngine {
+    tokenizer: Tokenizer,
+    session: Session,
+    pooling: Pooling,
+    need_token_type_ids: bool,
+}
+
+impl EmbeddingEngine {
+    /// Create a new embedding engine from a pre-built session and tokenizer.
+    pub(crate) fn new(tokenizer: Tokenizer, session: Session, pooling: Pooling) -> Self {
+        let need_token_type_ids = session.inputs().iter().any(|input| input.name() == "token_type_ids");
+
+        Self {
+            tokenizer,
+            session,
+            pooling,
+            need_token_type_ids,
+        }
+    }
+
+    /// Generate embeddings for a batch of texts.
+    ///
+    /// This method is **thread-safe** — multiple threads can call `embed()`
+    /// concurrently on the same `EmbeddingEngine` instance.
+    ///
+    /// # Safety note
+    ///
+    /// Uses an internal unsafe cast because `ort::Session::run()` takes
+    /// `&mut self` despite performing no mutation (its `run_inner()` takes
+    /// `&self`). The ONNX Runtime C API is documented as thread-safe for
+    /// concurrent `Run()` calls on the same session.
+    pub(crate) fn embed<S: AsRef<str>>(&self, texts: &[S], batch_size: usize) -> Result<Vec<Vec<f32>>, EmbedError> {
+        if texts.is_empty() {
+            return Ok(Vec::new());
+        }
+
+        // Defensive: callers from polyglot bindings may pass batch_size=0 when the
+        // host-side `EmbeddingConfig` mirror omits the serde default. `chunks(0)`
+        // would panic — fall back to the documented default rather than aborting
+        // the entire interpreter.
+        let batch_size = if batch_size == 0 { 32 } else { batch_size };
+
+        let mut all_embeddings = Vec::with_capacity(texts.len());
+
+        for batch in texts.chunks(batch_size) {
+            let batch_embeddings = self.embed_batch(batch)?;
+            all_embeddings.extend(batch_embeddings);
+        }
+
+        Ok(all_embeddings)
+    }
+
+    /// Embed a single batch of texts.
+    fn embed_batch<S: AsRef<str>>(&self, batch: &[S]) -> Result<Vec<Vec<f32>>, EmbedError> {
+        // Tokenize
+        let inputs: Vec<&str> = batch.iter().map(|t| t.as_ref()).collect();
+        let encodings = self
+            .tokenizer
+            .encode_batch(inputs, true)
+            .map_err(|e| EmbedError::Tokenizer(e.to_string()))?;
+
+        let encoding_length = encodings
+            .first()
+            .ok_or_else(|| EmbedError::Tokenizer("Empty encodings".to_string()))?
+            .len();
+        let batch_size = batch.len();
+        let max_size = encoding_length * batch_size;
+
+        // Build input tensors
+        let mut ids_array = Vec::with_capacity(max_size);
+        let mut mask_array = Vec::with_capacity(max_size);
+        let mut type_ids_array = Vec::with_capacity(max_size);
+
+        for encoding in &encodings {
+            ids_array.extend(encoding.get_ids().iter().map(|&x| x as i64));
+            mask_array.extend(encoding.get_attention_mask().iter().map(|&x| x as i64));
+            type_ids_array.extend(encoding.get_type_ids().iter().map(|&x| x as i64));
+        }
+
+        let ids_tensor = ndarray::Array::from_shape_vec((batch_size, encoding_length), ids_array)
+            .map_err(|e| EmbedError::Shape(e.to_string()))?;
+        let type_ids_tensor = ndarray::Array::from_shape_vec((batch_size, encoding_length), type_ids_array)
+            .map_err(|e| EmbedError::Shape(e.to_string()))?;
+
+        let mask_nd = ndarray::Array::from_shape_vec((batch_size, encoding_length), mask_array)
+            .map_err(|e| EmbedError::Shape(e.to_string()))?;
+        // Clone mask only when mean pooling needs it for post-processing.
+        let attention_mask_for_pooling = if self.pooling == Pooling::Mean {
+            Some(mask_nd.clone())
+        } else {
+            None
+        };
+        let mask_tensor = Value::from_array(mask_nd)?;
+
+        let mut session_inputs = ort::inputs![
+            "input_ids" => Value::from_array(ids_tensor)?,
+            "attention_mask" => mask_tensor,
+        ];
+
+        if self.need_token_type_ids {
+            session_inputs.push(("token_type_ids".into(), Value::from_array(type_ids_tensor)?.into()));
+        }
+
+        // Run inference — thread-safe despite &mut self signature on Session::run()
+        //
+        // SAFETY: ort::Session::run() takes &mut self but delegates to run_inner(&self)
+        // with zero actual mutation. The ONNX Runtime C API (OrtApi::Run) is documented
+        // as thread-safe for concurrent Run() calls on the same session.
+        #[allow(unsafe_code)]
+        let outputs = unsafe {
+            let session_ptr = &self.session as *const Session as *mut Session;
+            (*session_ptr).run(session_inputs)
+        }
+        .map_err(EmbedError::Ort)?;
+
+        // Find the embedding output tensor
+        let (_, output_value) = outputs.iter().next().ok_or(EmbedError::NoOutput)?;
+
+        let tensor: ArrayView<f32, Dim<IxDynImpl>> = output_value.try_extract_array().map_err(EmbedError::Ort)?;
+
+        // Pool (without normalization — caller controls normalization)
+        let pooled = match attention_mask_for_pooling {
+            Some(mask) => mean_pool(&tensor, mask)?,
+            None => cls_pool(&tensor)?,
+        };
+
+        let embeddings: Vec<Vec<f32>> = pooled
+            .rows()
+            .into_iter()
+            .map(|row| row.as_slice().unwrap_or(&[]).to_vec())
+            .collect();
+
+        Ok(embeddings)
+    }
+}
+
+// SAFETY: EmbeddingEngine is Send + Sync because:
+// 1. Tokenizer is Send + Sync (confirmed in tokenizers crate)
+// 2. Session: we only call run() which is internally thread-safe
+// 3. All other fields are immutable after construction
+#[allow(unsafe_code)]
+unsafe impl Send for EmbeddingEngine {}
+#[allow(unsafe_code)]
+unsafe impl Sync for EmbeddingEngine {}
+
+/// CLS pooling — extract the first token's embedding.
+fn cls_pool(tensor: &ArrayView<f32, Dim<IxDynImpl>>) -> Result<Array2<f32>, EmbedError> {
+    match tensor.dim().ndim() {
+        2 => Ok(tensor.slice(s![.., ..]).to_owned()),
+        3 => Ok(tensor.slice(s![.., 0, ..]).to_owned()),
+        _ => Err(EmbedError::Shape(format!(
+            "Expected 2D or 3D tensor, got {:?}",
+            tensor.dim()
+        ))),
+    }
+}
+
+/// Mean pooling — average token embeddings weighted by attention mask.
+fn mean_pool(tensor: &ArrayView<f32, Dim<IxDynImpl>>, attention_mask: Array2<i64>) -> Result<Array2<f32>, EmbedError> {
+    if tensor.dim().ndim() == 2 {
+        return Ok(tensor.slice(s![.., ..]).to_owned());
+    }
+    if tensor.dim().ndim() != 3 {
+        return Err(EmbedError::Shape(format!(
+            "Expected 2D or 3D tensor, got {:?}",
+            tensor.dim()
+        )));
+    }
+
+    let token_embeddings = tensor.slice(s![.., .., ..]);
+    let mask_dim = attention_mask.dim();
+    let mask_expanded = attention_mask
+        .insert_axis(ndarray::Axis(2))
+        .broadcast(token_embeddings.dim())
+        .ok_or_else(|| {
+            EmbedError::Shape(format!(
+                "Cannot broadcast attention mask {:?} to {:?}",
+                mask_dim,
+                token_embeddings.dim()
+            ))
+        })?
+        .mapv(|x| x as f32);
+
+    let masked = &mask_expanded * &token_embeddings;
+    let sum = masked.sum_axis(ndarray::Axis(1));
+    let mask_sum = mask_expanded.sum_axis(ndarray::Axis(1));
+    let mask_sum = mask_sum.mapv(|x| if x == 0.0 { 1.0 } else { x });
+
+    Ok(&sum / &mask_sum)
+}
+
+/// L2-normalize a vector.
+#[cfg(test)]
+pub(crate) fn normalize(v: &[f32]) -> Vec<f32> {
+    let norm = v.iter().map(|x| x * x).sum::<f32>().sqrt();
+    if norm > f32::EPSILON {
+        let inv = 1.0 / norm;
+        v.iter().map(|&x| x * inv).collect()
+    } else {
+        v.to_vec()
+    }
+}
+#[cfg_attr(alef, alef(skip))]
+/// Embedding engine errors.
+#[derive(Debug)]
+pub enum EmbedError {
+    Tokenizer(String),
+    Ort(ort::Error),
+    Shape(String),
+    NoOutput,
+}
+
+impl From<ort::Error> for EmbedError {
+    fn from(e: ort::Error) -> Self {
+        Self::Ort(e)
+    }
+}
+
+impl std::fmt::Display for EmbedError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Tokenizer(e) => write!(f, "Tokenizer error: {e}"),
+            Self::Ort(e) => write!(f, "ONNX Runtime error: {e}"),
+            Self::Shape(e) => write!(f, "Tensor shape error: {e}"),
+            Self::NoOutput => write!(f, "Model produced no output tensors"),
+        }
+    }
+}
+
+impl std::error::Error for EmbedError {}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Test normalization of a known vector produces unit vector (L2 norm ≈ 1.0).
+    #[test]
+    fn test_normalize_unit_vector() {
+        let v = vec![3.0, 4.0]; // 3-4-5 triangle
+        let normalized = normalize(&v);
+
+        assert_eq!(normalized.len(), 2);
+        assert!(
+            (normalized[0] - 0.6).abs() < 1e-6,
+            "Expected ~0.6, got {}",
+            normalized[0]
+        );
+        assert!(
+            (normalized[1] - 0.8).abs() < 1e-6,
+            "Expected ~0.8, got {}",
+            normalized[1]
+        );
+
+        // Verify L2 norm is approximately 1.0
+        let norm: f32 = normalized.iter().map(|x| x * x).sum::<f32>().sqrt();
+        assert!((norm - 1.0).abs() < 1e-6, "L2 norm should be ~1.0, got {}", norm);
+    }
+
+    /// Test normalization of all-zeros vector produces zeros without NaN/panic.
+    #[test]
+    fn test_normalize_zero_vector() {
+        let v = vec![0.0, 0.0, 0.0];
+        let normalized = normalize(&v);
+
+        assert_eq!(normalized.len(), 3);
+        assert!(!normalized.iter().any(|x| x.is_nan()), "No NaN values expected");
+        assert!(
+            !normalized.iter().any(|x| x.is_infinite()),
+            "No infinite values expected"
+        );
+        for &val in &normalized {
+            assert_eq!(val, 0.0, "Zero vector should remain zero");
+        }
+    }
+
+    /// Test normalization of single-element vector.
+    #[test]
+    fn test_normalize_single_element() {
+        let v = vec![5.0];
+        let normalized = normalize(&v);
+
+        assert_eq!(normalized.len(), 1);
+        assert!(
+            (normalized[0] - 1.0).abs() < 1e-6,
+            "Expected 1.0, got {}",
+            normalized[0]
+        );
+    }
+
+    /// Test that all EmbedError variants have Display impl without panicking.
+    #[test]
+    fn test_embed_error_display() {
+        // Test Tokenizer variant
+        let tokenizer_err = EmbedError::Tokenizer("test error".to_string());
+        let display = format!("{}", tokenizer_err);
+        assert!(display.contains("Tokenizer error"), "Tokenizer display: {}", display);
+
+        // Test Shape variant
+        let shape_err = EmbedError::Shape("invalid shape".to_string());
+        let display = format!("{}", shape_err);
+        assert!(display.contains("Tensor shape error"), "Shape display: {}", display);
+
+        // Test NoOutput variant
+        let no_output_err = EmbedError::NoOutput;
+        let display = format!("{}", no_output_err);
+        assert!(display.contains("no output"), "NoOutput display: {}", display);
+    }
+
+    /// Test that Pooling variants are distinct and comparable.
+    #[test]
+    fn test_pooling_variants() {
+        let cls = Pooling::Cls;
+        let mean = Pooling::Mean;
+
+        // Different variants should not be equal
+        assert_ne!(cls, mean, "Pooling::Cls and Pooling::Mean should be different");
+
+        // Same variants should be equal
+        assert_eq!(cls, Pooling::Cls);
+        assert_eq!(mean, Pooling::Mean);
+
+        // Pooling should be cloneable
+        let cls_clone = cls.clone();
+        assert_eq!(cls, cls_clone);
+
+        // Pooling should be debuggable
+        let debug_output = format!("{:?}", cls);
+        assert!(debug_output.contains("Cls"), "Debug output: {}", debug_output);
+    }
+
+    /// Test normalization preserves input length.
+    #[test]
+    fn test_normalize_preserves_length() {
+        let test_cases = vec![vec![1.0, 2.0, 3.0], vec![-1.0, -2.0], vec![0.1, 0.2, 0.3, 0.4, 0.5]];
+
+        for v in test_cases {
+            let original_len = v.len();
+            let normalized = normalize(&v);
+            assert_eq!(
+                normalized.len(),
+                original_len,
+                "Normalization should preserve vector length"
+            );
+        }
+    }
+
+    /// Test normalization handles negative values.
+    #[test]
+    fn test_normalize_negative_values() {
+        let v = vec![-3.0, -4.0]; // Same magnitude as [3.0, 4.0]
+        let normalized = normalize(&v);
+
+        assert!((normalized[0] - (-0.6)).abs() < 1e-6, "Expected ~-0.6");
+        assert!((normalized[1] - (-0.8)).abs() < 1e-6, "Expected ~-0.8");
+
+        // Verify L2 norm is still 1.0
+        let norm: f32 = normalized.iter().map(|x| x * x).sum::<f32>().sqrt();
+        assert!((norm - 1.0).abs() < 1e-6);
+    }
+
+    /// Test normalization with very small non-zero values (below epsilon threshold).
+    #[test]
+    fn test_normalize_very_small_values() {
+        let v = vec![f32::EPSILON / 2.0, f32::EPSILON / 2.0];
+        let normalized = normalize(&v);
+
+        // Values below epsilon threshold should be returned as-is
+        assert_eq!(normalized, v, "Very small vectors (< epsilon) returned unchanged");
+    }
+
+    /// Test EmbedError implements Error trait.
+    #[test]
+    fn test_embed_error_is_error_type() {
+        let err = EmbedError::Shape("test".to_string());
+        let _: &dyn std::error::Error = &err;
+        // If this compiles, the trait is properly implemented
+    }
+
+    /// Test Pooling enum Clone and Debug traits.
+    #[test]
+    fn test_pooling_traits() {
+        let cls = Pooling::Cls;
+        let mean = Pooling::Mean;
+
+        // Test Clone
+        let cls_clone = cls.clone();
+        let mean_clone = mean.clone();
+        assert_eq!(cls, cls_clone);
+        assert_eq!(mean, mean_clone);
+
+        // Test Debug produces valid output
+        let cls_debug = format!("{:?}", cls);
+        let mean_debug = format!("{:?}", mean);
+        assert!(!cls_debug.is_empty());
+        assert!(!mean_debug.is_empty());
+
+        // Test PartialEq and Eq
+        assert_eq!(cls, cls);
+        assert_eq!(mean, mean);
+        assert_ne!(cls, mean);
+    }
+
+    /// Test normalization with large magnitude values.
+    #[test]
+    fn test_normalize_large_values() {
+        let v = vec![1e6, 1e6];
+        let normalized = normalize(&v);
+
+        // Should normalize without overflow
+        assert!(!normalized.iter().any(|x| x.is_infinite()), "No overflow");
+        let norm: f32 = normalized.iter().map(|x| x * x).sum::<f32>().sqrt();
+        assert!((norm - 1.0).abs() < 1e-5, "L2 norm should be ~1.0");
+    }
+
+    /// Test normalization with mixed positive and negative values.
+    #[test]
+    fn test_normalize_mixed_signs() {
+        let v = vec![1.0, -1.0, 0.0];
+        let normalized = normalize(&v);
+
+        assert_eq!(normalized.len(), 3);
+        let norm: f32 = normalized.iter().map(|x| x * x).sum::<f32>().sqrt();
+        assert!((norm - 1.0).abs() < 1e-6, "L2 norm should be ~1.0");
+    }
+}