46 lines
1.4 KiB
Elixir
46 lines
1.4 KiB
Elixir
|
# This file is auto-generated by alef — DO NOT EDIT.
|
||
|
|
# alef:hash:4e15143f4af1ae8bafbdb1506ef057da924484c66a19483966333558ad437e75
|
||
|
|
# To regenerate: alef generate
|
||
|
|
# To verify freshness: alef verify --exit-code
|
||
|
|
# Issues & docs: https://github.com/kreuzberg-dev/alef
|
||
|
|
defmodule Kreuzberg.TokenReductionConfig do
|
||
|
|
@moduledoc false
|
||
|
|
|
||
|
|
@type t :: %__MODULE__{
|
||
|
|
level: String.t() | nil,
|
||
|
|
language_hint: String.t() | nil,
|
||
|
|
preserve_markdown: boolean(),
|
||
|
|
preserve_code: boolean(),
|
||
|
|
semantic_threshold: float(),
|
||
|
|
enable_parallel: boolean(),
|
||
|
|
use_simd: boolean(),
|
||
|
|
custom_stopwords: map() | nil,
|
||
|
|
preserve_patterns: [String.t()],
|
||
|
|
target_reduction: float() | nil,
|
||
|
|
enable_semantic_clustering: boolean()
|
||
|
|
}
|
||
|
|
|
||
|
|
defstruct level: :moderate,
|
||
|
|
language_hint: nil,
|
||
|
|
preserve_markdown: false,
|
||
|
|
preserve_code: true,
|
||
|
|
semantic_threshold: 0.3,
|
||
|
|
enable_parallel: true,
|
||
|
|
use_simd: true,
|
||
|
|
custom_stopwords: nil,
|
||
|
|
preserve_patterns: [],
|
||
|
|
target_reduction: nil,
|
||
|
|
enable_semantic_clustering: false
|
||
|
|
|
||
|
|
defimpl Jason.Encoder do
|
||
|
|
@doc false
|
||
|
|
def encode(value, opts) do
|
||
|
|
value
|
||
|
|
|> Map.from_struct()
|
||
|
|
|> Enum.reject(fn {_k, v} -> v == nil end)
|
||
|
|
|> Enum.into(%{})
|
||
|
|
|> Jason.Encoder.encode(opts)
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|