Nomad changes

docs/snippets/zig/advanced/chunk_page_mapping.md

@@ -0,0 +1,63 @@
+```zig title="Zig"
+const std = @import("std");
+const kreuzberg = @import("kreuzberg");
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    defer _ = gpa.deinit();
+    const allocator = gpa.allocator();
+
+    const config_json =
+        \\{
+        \\  "chunking": {
+        \\    "max_characters": 500,
+        \\    "overlap": 50
+        \\  },
+        \\  "pages": {
+        \\    "extract_pages": true,
+        \\    "insert_page_markers": false,
+        \\    "marker_format": ""
+        \\  }
+        \\}
+    ;
+
+    const result_json = try kreuzberg.extract_file_sync("document.pdf", null, config_json);
+    defer std.heap.c_allocator.free(result_json);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, result_json, .{});
+    defer parsed.deinit();
+
+    const root = parsed.value;
+    if (root != .object) return;
+
+    const stdout = std.io.getStdOut().writer();
+
+    const chunks_val = root.object.get("chunks") orelse return;
+    if (chunks_val != .array) return;
+
+    for (chunks_val.array.items) |chunk| {
+        if (chunk != .object) continue;
+
+        const metadata_val = chunk.object.get("metadata") orelse continue;
+        if (metadata_val != .object) continue;
+
+        const first_page_val = metadata_val.object.get("first_page") orelse continue;
+        const last_page_val = metadata_val.object.get("last_page") orelse continue;
+        if (first_page_val != .integer or last_page_val != .integer) continue;
+
+        const first = first_page_val.integer;
+        const last = last_page_val.integer;
+
+        if (chunk.object.get("content")) |content_val| {
+            if (content_val == .string) {
+                const preview_len = @min(50, content_val.string.len);
+                if (first == last) {
+                    try stdout.print("Chunk: {s}... (Page {d})\n", .{ content_val.string[0..preview_len], first });
+                } else {
+                    try stdout.print("Chunk: {s}... (Pages {d}-{d})\n", .{ content_val.string[0..preview_len], first, last });
+                }
+            }
+        }
+    }
+}
+```

docs/snippets/zig/advanced/chunking_config.md

@@ -0,0 +1,54 @@
+```zig title="Zig"
+const std = @import("std");
+const kreuzberg = @import("kreuzberg");
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    defer _ = gpa.deinit();
+    const allocator = gpa.allocator();
+
+    const config_json =
+        \\{
+        \\  "chunking": {
+        \\    "max_characters": 1000,
+        \\    "overlap": 200,
+        \\    "chunker_type": "markdown",
+        \\    "prepend_heading_context": true
+        \\  }
+        \\}
+    ;
+
+    const result_json = try kreuzberg.extract_file_sync("document.pdf", null, config_json);
+    defer std.heap.c_allocator.free(result_json);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, result_json, .{});
+    defer parsed.deinit();
+}
+```
+
+```zig title="Zig - Semantic"
+const std = @import("std");
+const kreuzberg = @import("kreuzberg");
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    defer _ = gpa.deinit();
+    const allocator = gpa.allocator();
+
+    const config_json =
+        \\{
+        \\  "chunking": {
+        \\    "max_characters": 1000,
+        \\    "overlap": 200,
+        \\    "chunker_type": "semantic"
+        \\  }
+        \\}
+    ;
+
+    const result_json = try kreuzberg.extract_file_sync("document.pdf", null, config_json);
+    defer std.heap.c_allocator.free(result_json);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, result_json, .{});
+    defer parsed.deinit();
+}
+```

docs/snippets/zig/advanced/chunking_rag.md

@@ -0,0 +1,54 @@
+```zig title="Zig"
+const std = @import("std");
+const kreuzberg = @import("kreuzberg");
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    defer _ = gpa.deinit();
+    const allocator = gpa.allocator();
+
+    const config_json =
+        \\{
+        \\  "chunking": {
+        \\    "max_characters": 500,
+        \\    "overlap": 50,
+        \\    "embedding": {
+        \\      "model": {"type": "preset", "name": "balanced"},
+        \\      "normalize": true
+        \\    }
+        \\  }
+        \\}
+    ;
+
+    const result_json = try kreuzberg.extract_file_sync("research_paper.pdf", null, config_json);
+    defer std.heap.c_allocator.free(result_json);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, result_json, .{});
+    defer parsed.deinit();
+
+    const root = parsed.value;
+    if (root != .object) return;
+
+    const stdout = std.io.getStdOut().writer();
+
+    const chunks_val = root.object.get("chunks") orelse return;
+    if (chunks_val != .array) return;
+
+    for (chunks_val.array.items, 0..) |chunk, index| {
+        if (chunk != .object) continue;
+
+        if (chunk.object.get("content")) |content_val| {
+            if (content_val == .string) {
+                const preview_len = @min(100, content_val.string.len);
+                try stdout.print("Chunk {d}: {s}...\n", .{ index, content_val.string[0..preview_len] });
+            }
+        }
+
+        if (chunk.object.get("embedding")) |embedding_val| {
+            if (embedding_val == .array) {
+                try stdout.print("  Embedding: {d} dimensions\n", .{embedding_val.array.items.len});
+            }
+        }
+    }
+}
+```

docs/snippets/zig/advanced/embedding_with_chunking.md

@@ -0,0 +1,31 @@
+```zig title="Zig"
+const std = @import("std");
+const kreuzberg = @import("kreuzberg");
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    defer _ = gpa.deinit();
+    const allocator = gpa.allocator();
+
+    const config_json =
+        \\{
+        \\  "chunking": {
+        \\    "max_characters": 1024,
+        \\    "overlap": 100,
+        \\    "embedding": {
+        \\      "model": {"type": "preset", "name": "balanced"},
+        \\      "normalize": true,
+        \\      "batch_size": 32,
+        \\      "show_download_progress": false
+        \\    }
+        \\  }
+        \\}
+    ;
+
+    const result_json = try kreuzberg.extract_file_sync("document.pdf", null, config_json);
+    defer std.heap.c_allocator.free(result_json);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, result_json, .{});
+    defer parsed.deinit();
+}
+```

docs/snippets/zig/advanced/keyword_extraction_config.md

@@ -0,0 +1,28 @@
+```zig title="Zig"
+const std = @import("std");
+const kreuzberg = @import("kreuzberg");
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    defer _ = gpa.deinit();
+    const allocator = gpa.allocator();
+
+    const config_json =
+        \\{
+        \\  "keywords": {
+        \\    "algorithm": "yake",
+        \\    "max_keywords": 10,
+        \\    "min_score": 0.3,
+        \\    "ngram_range": [1, 3],
+        \\    "language": "en"
+        \\  }
+        \\}
+    ;
+
+    const result_json = try kreuzberg.extract_file_sync("document.pdf", null, config_json);
+    defer std.heap.c_allocator.free(result_json);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, result_json, .{});
+    defer parsed.deinit();
+}
+```

docs/snippets/zig/advanced/keyword_extraction_example.md

@@ -0,0 +1,51 @@
+```zig title="Zig"
+const std = @import("std");
+const kreuzberg = @import("kreuzberg");
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    defer _ = gpa.deinit();
+    const allocator = gpa.allocator();
+
+    const config_json =
+        \\{
+        \\  "keywords": {
+        \\    "algorithm": "yake",
+        \\    "max_keywords": 10,
+        \\    "min_score": 0.3,
+        \\    "ngram_range": [1, 3]
+        \\  }
+        \\}
+    ;
+
+    const result_json = try kreuzberg.extract_file_sync("research_paper.pdf", null, config_json);
+    defer std.heap.c_allocator.free(result_json);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, result_json, .{});
+    defer parsed.deinit();
+
+    const root = parsed.value;
+    if (root != .object) return;
+
+    const stdout = std.io.getStdOut().writer();
+
+    const keywords_val = root.object.get("extracted_keywords") orelse return;
+    if (keywords_val != .array) return;
+
+    for (keywords_val.array.items) |keyword| {
+        if (keyword != .object) continue;
+
+        const text_val = keyword.object.get("text") orelse continue;
+        const score_val = keyword.object.get("score") orelse continue;
+        if (text_val != .string) continue;
+
+        const score: f64 = switch (score_val) {
+            .float => |f| f,
+            .integer => |i| @floatFromInt(i),
+            else => continue,
+        };
+
+        try stdout.print("{s}: {d:.4}\n", .{ text_val.string, score });
+    }
+}
+```

docs/snippets/zig/advanced/language_detection_config.md

@@ -0,0 +1,26 @@
+```zig title="Zig"
+const std = @import("std");
+const kreuzberg = @import("kreuzberg");
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    defer _ = gpa.deinit();
+    const allocator = gpa.allocator();
+
+    const config_json =
+        \\{
+        \\  "language_detection": {
+        \\    "enabled": true,
+        \\    "min_confidence": 0.8,
+        \\    "detect_multiple": false
+        \\  }
+        \\}
+    ;
+
+    const result_json = try kreuzberg.extract_file_sync("document.pdf", null, config_json);
+    defer std.heap.c_allocator.free(result_json);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, result_json, .{});
+    defer parsed.deinit();
+}
+```

docs/snippets/zig/advanced/language_detection_multilingual.md

@@ -0,0 +1,43 @@
+```zig title="Zig"
+const std = @import("std");
+const kreuzberg = @import("kreuzberg");
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    defer _ = gpa.deinit();
+    const allocator = gpa.allocator();
+
+    const config_json =
+        \\{
+        \\  "language_detection": {
+        \\    "enabled": true,
+        \\    "min_confidence": 0.8,
+        \\    "detect_multiple": true
+        \\  }
+        \\}
+    ;
+
+    const result_json = try kreuzberg.extract_file_sync("multilingual_document.pdf", null, config_json);
+    defer std.heap.c_allocator.free(result_json);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, result_json, .{});
+    defer parsed.deinit();
+
+    const root = parsed.value;
+    if (root != .object) return;
+
+    const stdout = std.io.getStdOut().writer();
+
+    try stdout.print("Detected languages:", .{});
+    if (root.object.get("detected_languages")) |languages_val| {
+        if (languages_val == .array) {
+            for (languages_val.array.items) |lang_val| {
+                if (lang_val == .string) {
+                    try stdout.print(" {s}", .{lang_val.string});
+                }
+            }
+        }
+    }
+    try stdout.print("\n", .{});
+}
+```

docs/snippets/zig/advanced/quality_processing_config.md

@@ -0,0 +1,22 @@
+```zig title="Zig"
+const std = @import("std");
+const kreuzberg = @import("kreuzberg");
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    defer _ = gpa.deinit();
+    const allocator = gpa.allocator();
+
+    const config_json =
+        \\{
+        \\  "enable_quality_processing": true
+        \\}
+    ;
+
+    const result_json = try kreuzberg.extract_file_sync("document.pdf", null, config_json);
+    defer std.heap.c_allocator.free(result_json);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, result_json, .{});
+    defer parsed.deinit();
+}
+```

docs/snippets/zig/advanced/quality_processing_example.md

@@ -0,0 +1,41 @@
+```zig title="Zig"
+const std = @import("std");
+const kreuzberg = @import("kreuzberg");
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    defer _ = gpa.deinit();
+    const allocator = gpa.allocator();
+
+    const config_json =
+        \\{
+        \\  "enable_quality_processing": true
+        \\}
+    ;
+
+    const result_json = try kreuzberg.extract_file_sync("scanned_document.pdf", null, config_json);
+    defer std.heap.c_allocator.free(result_json);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, result_json, .{});
+    defer parsed.deinit();
+
+    const root = parsed.value;
+    if (root != .object) return;
+
+    const stdout = std.io.getStdOut().writer();
+
+    if (root.object.get("quality_score")) |score_val| {
+        const score: f64 = switch (score_val) {
+            .float => |f| f,
+            .integer => |i| @floatFromInt(i),
+            else => return,
+        };
+
+        if (score < 0.5) {
+            try stdout.print("Warning: Low quality extraction ({d:.2})\n", .{score});
+        } else {
+            try stdout.print("Quality score: {d:.2}\n", .{score});
+        }
+    }
+}
+```

docs/snippets/zig/advanced/token_reduction_config.md

@@ -0,0 +1,25 @@
+```zig title="Zig"
+const std = @import("std");
+const kreuzberg = @import("kreuzberg");
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    defer _ = gpa.deinit();
+    const allocator = gpa.allocator();
+
+    const config_json =
+        \\{
+        \\  "token_reduction": {
+        \\    "mode": "moderate",
+        \\    "preserve_important_words": true
+        \\  }
+        \\}
+    ;
+
+    const result_json = try kreuzberg.extract_file_sync("document.pdf", null, config_json);
+    defer std.heap.c_allocator.free(result_json);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, result_json, .{});
+    defer parsed.deinit();
+}
+```

docs/snippets/zig/advanced/token_reduction_example.md

@@ -0,0 +1,42 @@
+```zig title="Zig"
+const std = @import("std");
+const kreuzberg = @import("kreuzberg");
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    defer _ = gpa.deinit();
+    const allocator = gpa.allocator();
+
+    const config_json =
+        \\{
+        \\  "token_reduction": {
+        \\    "mode": "moderate",
+        \\    "preserve_important_words": true
+        \\  }
+        \\}
+    ;
+
+    const result_json = try kreuzberg.extract_file_sync("verbose_document.pdf", null, config_json);
+    defer std.heap.c_allocator.free(result_json);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, result_json, .{});
+    defer parsed.deinit();
+
+    const root = parsed.value;
+    if (root != .object) return;
+
+    const stdout = std.io.getStdOut().writer();
+
+    if (root.object.get("original_token_count")) |val| {
+        if (val == .integer) {
+            try stdout.print("Original tokens: {d}\n", .{val.integer});
+        }
+    }
+
+    if (root.object.get("reduced_token_count")) |val| {
+        if (val == .integer) {
+            try stdout.print("Reduced tokens: {d}\n", .{val.integer});
+        }
+    }
+}
+```

docs/snippets/zig/advanced/vector_database_integration.md

@@ -0,0 +1,60 @@
+```zig title="Zig"
+const std = @import("std");
+const kreuzberg = @import("kreuzberg");
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    defer _ = gpa.deinit();
+    const allocator = gpa.allocator();
+
+    const document_id = "doc_001";
+
+    const config_json =
+        \\{
+        \\  "chunking": {
+        \\    "max_characters": 512,
+        \\    "overlap": 50,
+        \\    "embedding": {
+        \\      "model": {"type": "preset", "name": "balanced"},
+        \\      "normalize": true,
+        \\      "batch_size": 32
+        \\    }
+        \\  }
+        \\}
+    ;
+
+    const result_json = try kreuzberg.extract_file_sync("document.pdf", null, config_json);
+    defer std.heap.c_allocator.free(result_json);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, result_json, .{});
+    defer parsed.deinit();
+
+    const root = parsed.value;
+    if (root != .object) return;
+
+    const stdout = std.io.getStdOut().writer();
+
+    const chunks_val = root.object.get("chunks") orelse return;
+    if (chunks_val != .array) return;
+
+    for (chunks_val.array.items, 0..) |chunk, index| {
+        if (chunk != .object) continue;
+
+        const embedding_val = chunk.object.get("embedding") orelse continue;
+        if (embedding_val != .array) continue;
+
+        const content_val = chunk.object.get("content") orelse continue;
+        if (content_val != .string) continue;
+
+        const record_id = try std.fmt.allocPrint(allocator, "{s}_chunk_{d}", .{ document_id, index });
+        defer allocator.free(record_id);
+
+        try stdout.print("id={s} dims={d} content_length={d}\n", .{
+            record_id,
+            embedding_val.array.items.len,
+            content_val.string.len,
+        });
+        // Persist record_id, content_val.string, and embedding_val.array.items in a vector database.
+    }
+}
+```