This commit is contained in:
20
docs/snippets/ruby/utils/token_reduction_example.md
Normal file
20
docs/snippets/ruby/utils/token_reduction_example.md
Normal file
@@ -0,0 +1,20 @@
|
||||
```ruby title="Ruby"
|
||||
require 'kreuzberg'
|
||||
|
||||
config = Kreuzberg::ExtractionConfig.new(
|
||||
token_reduction: Kreuzberg::TokenReductionConfig.new(
|
||||
mode: 'moderate',
|
||||
preserve_markdown: true
|
||||
)
|
||||
)
|
||||
|
||||
result = Kreuzberg.extract_file_sync('verbose_document.pdf', config: config)
|
||||
|
||||
# Check reduction statistics in metadata
|
||||
original_tokens = result.metadata['original_token_count']
|
||||
reduced_tokens = result.metadata['token_count']
|
||||
reduction_ratio = result.metadata['token_reduction_ratio']
|
||||
|
||||
puts "Reduced from #{original_tokens} to #{reduced_tokens} tokens"
|
||||
puts "Reduction: #{reduction_ratio * 100}%"
|
||||
```
|
||||
Reference in New Issue
Block a user