This commit is contained in:
36
docs/snippets/rust/advanced/chunking_rag.md
Normal file
36
docs/snippets/rust/advanced/chunking_rag.md
Normal file
@@ -0,0 +1,36 @@
|
||||
```rust title="Rust"
|
||||
use kreuzberg::{extract_file, ExtractionConfig, ChunkingConfig, EmbeddingConfig};
|
||||
|
||||
let config = ExtractionConfig {
|
||||
chunking: Some(ChunkingConfig {
|
||||
max_characters: 500,
|
||||
overlap: 50,
|
||||
embedding: Some(EmbeddingConfig {
|
||||
model: "balanced".to_string(),
|
||||
normalize: true,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let result = extract_file("research_paper.pdf", None, &config).await?;
|
||||
|
||||
if let Some(chunks) = result.chunks {
|
||||
for chunk in chunks {
|
||||
println!("Chunk {}/{}",
|
||||
chunk.metadata.chunk_index + 1,
|
||||
chunk.metadata.total_chunks
|
||||
);
|
||||
println!("Position: {}-{}",
|
||||
chunk.metadata.byte_start,
|
||||
chunk.metadata.byte_end
|
||||
);
|
||||
println!("Content: {}...", &chunk.content[..100.min(chunk.content.len())]);
|
||||
if let Some(embedding) = chunk.embedding {
|
||||
println!("Embedding: {} dimensions", embedding.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
Reference in New Issue
Block a user