Nomad changes

crates/kreuzberg-ffi/tests/email_attachment_data_len.rs

@@ -0,0 +1,177 @@
+/// Regression test for GitHub #1059.
+///
+/// `kreuzberg_email_attachment_data` was the only byte-buffer accessor on a public
+/// FFI-exposed DTO that did not follow the established `*_data(ptr, out_len: *mut usize)`
+/// protocol used by `kreuzberg_extracted_image_data`, `kreuzberg_embedded_file_data`,
+/// and `kreuzberg_batch_bytes_item_content`.
+///
+/// Because `EmailAttachment.data` is `Option<Bytes>` (the only optional byte buffer among
+/// public types), alef's heuristic for emitting the two-parameter form did not trigger.
+/// Callers had no way to know the valid length of the returned pointer, making any read
+/// past the first byte undefined behaviour (especially for payloads containing 0x00).
+///
+/// The alef fix shipped with the 2-parameter form (`ptr`, `out_len`). These tests
+/// lock in the correct 2-param ABI and verify the full-length contract for payloads
+/// that contain embedded NUL bytes.
+///
+/// Per project rules: every unsafe block has a SAFETY comment.
+use std::ffi::{c_char, CString};
+use std::fs;
+use std::path::Path;
+
+use kreuzberg_ffi::{kreuzberg_email_attachment_free, kreuzberg_email_attachment_from_json, kreuzberg_last_error_code};
+
+/// Construct a minimal EmailAttachment JSON with a data payload that contains
+/// an embedded NUL and a trailing high byte (0xEF). This defeats any strlen-based
+/// or "read first byte only" implementations.
+fn attachment_json_with_nuls() -> CString {
+    // 8 bytes: JPEG-ish magic + NUL in the middle + high byte at the end.
+    // Length is authoritative and known.
+    let data: Vec<u8> = vec![0xFF, 0xD8, 0xFF, 0x00, 0xDE, 0xAD, 0xBE, 0xEF];
+    let json = format!(
+        r#"{{
+            "name": "test.bin",
+            "filename": "test.bin",
+            "mime_type": "application/octet-stream",
+            "size": {},
+            "is_image": false,
+            "data": {}
+        }}"#,
+        data.len(),
+        serde_json::to_string(&data).unwrap()
+    );
+    CString::new(json).expect("valid UTF-8 JSON for test attachment")
+}
+
+/// The committed C header must declare the 2-parameter form for
+/// `kreuzberg_email_attachment_data` (with `out_len`). This locks in the fix
+/// for GitHub #1059 so a future regeneration cannot silently revert to the
+/// 1-parameter form.
+#[test]
+fn email_attachment_data_accessor_must_provide_out_len_in_header() {
+    let header_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("include/kreuzberg.h");
+    let header = fs::read_to_string(&header_path).expect("committed kreuzberg.h must be readable by the test");
+
+    // Simple and robust: the declaration for this specific function must mention out_len.
+    let has_out_len = header.contains("kreuzberg_email_attachment_data") && header.contains("out_len");
+
+    assert!(
+        has_out_len,
+        "GitHub #1059 regression: the declaration of kreuzberg_email_attachment_data \
+         in crates/kreuzberg-ffi/include/kreuzberg.h does not contain the required \
+         `out_len` parameter.\n\n\
+         Expected something like:\n    uint8_t *kreuzberg_email_attachment_data(..., uintptr_t *out_len);\n\n\
+         Found the old 1-parameter form. Fix requires `task alef:generate` with an \
+         updated alef that handles Option<Bytes> fields for the FFI byte accessor heuristic.\n\n\
+         This is the lock-in test for #1059."
+    );
+}
+
+/// When an attachment has no data payload the accessor must return a null pointer
+/// and write 0 to out_len.
+#[test]
+fn email_attachment_data_none_returns_null_pointer() {
+    let json = CString::new(
+        r#"{"name":"empty","filename":"empty","mime_type":null,"size":null,"is_image":false,"data":null}"#,
+    )
+    .unwrap();
+
+    // SAFETY: json is valid null-terminated UTF-8.
+    let handle = unsafe { kreuzberg_email_attachment_from_json(json.as_ptr() as *const c_char) };
+    assert!(
+        !handle.is_null(),
+        "from_json should succeed (last_error_code={})",
+        // SAFETY: no precondition; reads a thread-local.
+        unsafe { kreuzberg_last_error_code() }
+    );
+
+    let mut out_len: usize = usize::MAX;
+    // SAFETY: handle is a valid non-null pointer returned by from_json;
+    // out_len is a valid stack-allocated usize.
+    let data_ptr = unsafe { kreuzberg_ffi::kreuzberg_email_attachment_data(handle, &mut out_len) };
+
+    assert!(
+        data_ptr.is_null(),
+        "data must be null when the attachment has no payload"
+    );
+    assert_eq!(out_len, 0, "out_len must be 0 when data is None");
+
+    // SAFETY: handle came from from_json; we are the sole owner.
+    unsafe { kreuzberg_email_attachment_free(handle) };
+}
+
+/// When an attachment carries a binary payload the accessor must return a non-null
+/// pointer and write the exact byte count — including bytes past any embedded NUL —
+/// to out_len. This is the core contract broken by the 1-parameter bug (#1059).
+#[test]
+fn email_attachment_data_with_out_len_returns_full_buffer_including_embedded_nuls() {
+    let json = attachment_json_with_nuls();
+    // SAFETY: json is a valid null-terminated CString we just created.
+    let handle = unsafe { kreuzberg_email_attachment_from_json(json.as_ptr() as *const c_char) };
+    assert!(
+        !handle.is_null(),
+        "from_json should succeed for our well-formed test attachment (last_error_code={})",
+        // SAFETY: no precondition; reads a thread-local.
+        unsafe { kreuzberg_last_error_code() }
+    );
+
+    let mut out_len: usize = 0;
+
+    // SAFETY: handle is non-null and freshly allocated by from_json;
+    // out_len is a valid stack-allocated usize. The returned pointer must not
+    // be freed by us — it borrows the internal Bytes of the handle.
+    let data_ptr = unsafe { kreuzberg_ffi::kreuzberg_email_attachment_data(handle, &mut out_len) };
+
+    assert!(
+        !data_ptr.is_null(),
+        "data pointer must be non-null for an attachment we created with a Some(data) payload"
+    );
+    assert_eq!(
+        out_len, 8,
+        "out_len must report the exact length of the Bytes payload (not 0, not guessed, not truncated at NUL)"
+    );
+
+    // SAFETY: data_ptr is valid for [0..out_len] because:
+    // - it came from the handle's internal Bytes (which we control),
+    // - out_len was written by the accessor,
+    // - the handle is still alive (we have not called free yet).
+    let slice = unsafe { std::slice::from_raw_parts(data_ptr, out_len) };
+
+    assert_eq!(slice.len(), 8);
+    assert_eq!(slice[0], 0xFF);
+    assert_eq!(slice[3], 0x00, "must be able to read the embedded NUL");
+    assert_eq!(
+        slice[7], 0xEF,
+        "must be able to read bytes after the NUL (no truncation)"
+    );
+
+    // Cleanup
+    // SAFETY: handle came from from_json; we are the owner.
+    unsafe { kreuzberg_email_attachment_free(handle) };
+}
+
+/// Verify that passing a null out_len pointer is safe: the accessor must not
+/// segfault, and the data pointer itself must still be returned.
+#[test]
+fn email_attachment_data_null_out_len_is_safe() {
+    let json = CString::new(
+        r#"{"name":"hasdata.bin","filename":"hasdata.bin","mime_type":"application/octet-stream","size":4,"is_image":false,"data":[65,0,66,67]}"#,
+    )
+    .unwrap();
+
+    // SAFETY: json is valid.
+    let handle = unsafe { kreuzberg_email_attachment_from_json(json.as_ptr() as *const c_char) };
+    assert!(!handle.is_null());
+
+    // SAFETY: handle is valid; passing null for out_len is a defined contract
+    // (the accessor null-checks before writing).
+    let data_ptr = unsafe { kreuzberg_ffi::kreuzberg_email_attachment_data(handle, std::ptr::null_mut()) };
+
+    assert!(
+        !data_ptr.is_null(),
+        "data pointer should be non-null when the attachment carries a payload"
+    );
+
+    // SAFETY: handle from from_json; we are the owner.
+    unsafe { kreuzberg_email_attachment_free(handle) };
+}

crates/kreuzberg-ffi/tests/vtable_bytes_len.rs

@@ -0,0 +1,204 @@
+/// Regression tests: vtable Bytes params carry companion length
+///
+/// The alef vtable generator previously emitted only `*const u8` for `&[u8]`
+/// trait-method parameters without a companion `{name}_len: usize`. Binary
+/// payloads contain embedded NUL bytes; read-until-NUL semantics silently
+/// truncated every real image or document buffer at the first `0x00`.
+///
+/// Fix shipped in alef ≥ v0.19.21 and is present in the generated FFI shim.
+/// These tests construct a vtable bridge directly, pass a buffer with an
+/// embedded NUL at a known offset, and assert the full buffer is received.
+///
+/// Per-test state is passed via `user_data` — no global statics — so tests
+/// are independent and can run in parallel without interfering.
+use kreuzberg_ffi::{
+    KreuzbergDocumentExtractorBridge, KreuzbergDocumentExtractorVTable, KreuzbergOcrBackendBridge,
+    KreuzbergOcrBackendVTable,
+};
+use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering};
+
+// ── Per-test callback state ───────────────────────────────────────────────
+
+struct CallbackState {
+    received_len: AtomicUsize,
+    received_last_byte: AtomicU8,
+}
+
+impl CallbackState {
+    fn new() -> Self {
+        Self {
+            received_len: AtomicUsize::new(0),
+            received_last_byte: AtomicU8::new(0),
+        }
+    }
+}
+
+// ── C callback stubs ─────────────────────────────────────────────────────
+
+unsafe extern "C" fn ocr_process_image(
+    user_data: *const std::ffi::c_void,
+    image_bytes: *const u8,
+    image_bytes_len: usize,
+    _config: *const std::ffi::c_char,
+    out_result: *mut *mut std::ffi::c_char,
+    out_error: *mut *mut std::ffi::c_char,
+) -> i32 {
+    // SAFETY: user_data points to a CallbackState that the calling test keeps alive.
+    let state = unsafe { &*(user_data as *const CallbackState) };
+    state.received_len.store(image_bytes_len, Ordering::SeqCst);
+    if image_bytes_len > 0 {
+        // SAFETY: caller guarantees image_bytes[0..image_bytes_len] is valid.
+        let last = unsafe { *image_bytes.add(image_bytes_len - 1) };
+        state.received_last_byte.store(last, Ordering::SeqCst);
+    }
+    unsafe { *out_result = std::ptr::null_mut() };
+    let msg = std::ffi::CString::new("stub").unwrap();
+    // SAFETY: caller owns out_error and will free it via kreuzberg_free_string.
+    unsafe { *out_error = msg.into_raw() };
+    1
+}
+
+unsafe extern "C" fn extractor_extract_bytes(
+    user_data: *const std::ffi::c_void,
+    content: *const u8,
+    content_len: usize,
+    _mime_type: *const std::ffi::c_char,
+    _config: *const std::ffi::c_char,
+    out_result: *mut *mut std::ffi::c_char,
+    out_error: *mut *mut std::ffi::c_char,
+) -> i32 {
+    // SAFETY: user_data points to a CallbackState that the calling test keeps alive.
+    let state = unsafe { &*(user_data as *const CallbackState) };
+    state.received_len.store(content_len, Ordering::SeqCst);
+    if content_len > 0 {
+        // SAFETY: caller guarantees content[0..content_len] is valid.
+        let last = unsafe { *content.add(content_len - 1) };
+        state.received_last_byte.store(last, Ordering::SeqCst);
+    }
+    unsafe { *out_result = std::ptr::null_mut() };
+    let msg = std::ffi::CString::new("stub").unwrap();
+    unsafe { *out_error = msg.into_raw() };
+    1
+}
+
+// ── Tests ─────────────────────────────────────────────────────────────────
+
+/// OcrBackend.process_image must pass the full buffer length even when
+/// the payload contains embedded NUL bytes.
+#[tokio::test]
+async fn ocr_backend_vtable_process_image_passes_full_length_with_embedded_nuls() {
+    // 8-byte buffer; NUL at index 3. strlen-style reads would stop at 3.
+    let image_bytes: Vec<u8> = vec![0xFF, 0xD8, 0xFF, 0x00, 0xDE, 0xAD, 0xBE, 0xEF];
+
+    let state = Box::new(CallbackState::new());
+    let state_ptr = state.as_ref() as *const CallbackState as *const std::ffi::c_void;
+
+    let vtable = KreuzbergOcrBackendVTable {
+        process_image: Some(ocr_process_image),
+        process_image_file: None,
+        name_fn: None,
+        version_fn: None,
+        initialize_fn: None,
+        shutdown_fn: None,
+        supports_language: None,
+        backend_type: None,
+        supported_languages: None,
+        supports_table_detection: None,
+        supports_document_processing: None,
+        process_document: None,
+        free_user_data: None,
+    };
+
+    // SAFETY: state lives for the duration of this test and outlives the bridge.
+    let bridge = unsafe { KreuzbergOcrBackendBridge::new("test-ocr-stub".to_string(), vtable, state_ptr) };
+
+    use kreuzberg::OcrBackend;
+    let _ = bridge
+        .process_image(&image_bytes, &kreuzberg::OcrConfig::default())
+        .await;
+
+    assert_eq!(
+        state.received_len.load(Ordering::SeqCst),
+        8,
+        "process_image vtable received wrong length (truncated at embedded NUL?)"
+    );
+    assert_eq!(
+        state.received_last_byte.load(Ordering::SeqCst),
+        0xEF,
+        "process_image vtable could not read past the embedded NUL"
+    );
+}
+
+/// DocumentExtractor.extract_bytes must pass the full buffer length even when
+/// the document bytes contain embedded NUL bytes.
+#[tokio::test]
+async fn document_extractor_vtable_extract_bytes_passes_full_length_with_embedded_nuls() {
+    // 8-byte buffer; NUL at index 2.
+    let content: Vec<u8> = vec![0x50, 0x4B, 0x00, 0x03, 0x14, 0x00, 0x00, 0x02];
+
+    let state = Box::new(CallbackState::new());
+    let state_ptr = state.as_ref() as *const CallbackState as *const std::ffi::c_void;
+
+    let vtable = KreuzbergDocumentExtractorVTable {
+        extract_bytes: Some(extractor_extract_bytes),
+        extract_file: None,
+        name_fn: None,
+        version_fn: None,
+        initialize_fn: None,
+        shutdown_fn: None,
+        supported_mime_types: None,
+        priority: None,
+        can_handle: None,
+        free_user_data: None,
+    };
+
+    // SAFETY: state lives for the duration of this test and outlives the bridge.
+    let bridge = unsafe { KreuzbergDocumentExtractorBridge::new("test-extractor-stub".to_string(), vtable, state_ptr) };
+
+    use kreuzberg::DocumentExtractor;
+    let _ = bridge
+        .extract_bytes(
+            &content,
+            "application/octet-stream",
+            &kreuzberg::ExtractionConfig::default(),
+        )
+        .await;
+
+    assert_eq!(
+        state.received_len.load(Ordering::SeqCst),
+        8,
+        "extract_bytes vtable received wrong length (truncated at embedded NUL?)"
+    );
+    assert_eq!(
+        state.received_last_byte.load(Ordering::SeqCst),
+        0x02,
+        "extract_bytes vtable could not read past the embedded NUL"
+    );
+}
+
+/// ImageKind numeric values: PageRaster must be 10 and Unknown must be 11.
+///
+/// alef ≥ v0.19.21 added PageRaster between Mask (9) and Unknown, bumping
+/// Unknown from 10 → 11. Any C/Go/Java/C# code that hardcoded Unknown = 10
+/// must be updated; this test pins the new ordinals so the renumbering is
+/// visible to CI.
+#[test]
+fn image_kind_page_raster_is_10_and_unknown_is_11() {
+    // SAFETY: pure integer dispatch, no pointers.
+    assert_eq!(
+        unsafe { kreuzberg_ffi::kreuzberg_image_kind_from_i32(10) },
+        10,
+        "PageRaster == 10"
+    );
+    assert_eq!(
+        unsafe { kreuzberg_ffi::kreuzberg_image_kind_from_i32(11) },
+        11,
+        "Unknown == 11"
+    );
+    // Old Unknown value must now resolve to PageRaster, not Unknown.
+    assert_ne!(
+        unsafe { kreuzberg_ffi::kreuzberg_image_kind_from_i32(10) },
+        -1,
+        "10 must be valid"
+    );
+}