This commit is contained in:
85
docs/snippets/csharp/advanced/chunking_rag.md
Normal file
85
docs/snippets/csharp/advanced/chunking_rag.md
Normal file
@@ -0,0 +1,85 @@
|
||||
```csharp title="C#"
|
||||
using Kreuzberg;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
class RagPipelineExample
|
||||
{
|
||||
static async Task Main()
|
||||
{
|
||||
var config = new ExtractionConfig
|
||||
{
|
||||
Chunking = new ChunkingConfig
|
||||
{
|
||||
MaxChars = 500,
|
||||
MaxOverlap = 50,
|
||||
Embedding = new EmbeddingConfig
|
||||
{
|
||||
Model = EmbeddingModelType.Preset("all-mpnet-base-v2"),
|
||||
Normalize = true,
|
||||
BatchSize = 16
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
var result = await KreuzbergLib.ExtractFileAsync(
|
||||
"research_paper.pdf",
|
||||
config
|
||||
).ConfigureAwait(false);
|
||||
|
||||
var vectorStore = await BuildVectorStoreAsync(result.Chunks)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var query = "machine learning optimization";
|
||||
var relevantChunks = await SearchAsync(vectorStore, query)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
Console.WriteLine($"Found {relevantChunks.Count} relevant chunks");
|
||||
foreach (var chunk in relevantChunks.Take(3))
|
||||
{
|
||||
Console.WriteLine($"Content: {chunk.Content[..80]}...");
|
||||
Console.WriteLine($"Similarity: {chunk.Similarity:F3}\n");
|
||||
}
|
||||
}
|
||||
catch (KreuzbergException ex)
|
||||
{
|
||||
Console.WriteLine($"Error: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
static async Task<List<VectorEntry>> BuildVectorStoreAsync(
|
||||
IEnumerable<Chunk> chunks)
|
||||
{
|
||||
return await Task.Run(() =>
|
||||
{
|
||||
return chunks.Select(c => new VectorEntry
|
||||
{
|
||||
Content = c.Content,
|
||||
Embedding = c.Embedding?.ToArray() ?? Array.Empty<float>(),
|
||||
Similarity = 0f
|
||||
}).ToList();
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
static async Task<List<VectorEntry>> SearchAsync(
|
||||
List<VectorEntry> store,
|
||||
string query)
|
||||
{
|
||||
return await Task.Run(() =>
|
||||
{
|
||||
return store
|
||||
.OrderByDescending(e => e.Similarity)
|
||||
.ToList();
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
class VectorEntry
|
||||
{
|
||||
public string Content { get; set; } = string.Empty;
|
||||
public float[] Embedding { get; set; } = Array.Empty<float>();
|
||||
public float Similarity { get; set; }
|
||||
}
|
||||
}
|
||||
```
|
||||
Reference in New Issue
Block a user