Nomad changes

packages/elixir/lib/kreuzberg/chunk_sizing.ex

@@ -0,0 +1,25 @@
+# This file is auto-generated by alef — DO NOT EDIT.
+# alef:hash:4e15143f4af1ae8bafbdb1506ef057da924484c66a19483966333558ad437e75
+# To regenerate: alef generate
+# To verify freshness: alef verify --exit-code
+# Issues & docs: https://github.com/kreuzberg-dev/alef
+defmodule Kreuzberg.ChunkSizing do
+  @moduledoc """
+  How chunk size is measured.
+
+  Defaults to `Characters` (Unicode character count). When using token-based sizing,
+  chunks are sized by token count according to the specified tokenizer.
+
+  Token-based sizing uses HuggingFace tokenizers loaded at runtime. Any tokenizer
+  available on HuggingFace Hub can be used, including OpenAI-compatible tokenizers
+  (e.g., `Xenova/gpt-4o`, `Xenova/cl100k_base`).
+  """
+
+  @typedoc "How chunk size is measured."
+  @type t :: term()
+
+  @typedoc "Size measured in Unicode characters (default)."
+  @type characters :: :characters
+  @typedoc "Size measured in tokens from a HuggingFace tokenizer."
+  @type tokenizer :: %{type: :tokenizer, model: String.t(), cache_dir: String.t()}
+end