This commit is contained in:
Henrik Jess Nielsen
89
.task/config/platforms.yml
Normal file
89
.task/config/platforms.yml
Normal file
@@ -0,0 +1,89 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
includes:
|
||||
vars: ./vars.yml
|
||||
|
||||
vars:
|
||||
EXE_EXT:
|
||||
sh: |
|
||||
if [[ "$OSTYPE" == "msys" ]] || [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then
|
||||
echo ".exe"
|
||||
else
|
||||
echo ""
|
||||
fi
|
||||
|
||||
LIB_EXT:
|
||||
sh: |
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
echo "dylib"
|
||||
elif [[ "$OSTYPE" == "msys" ]] || [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then
|
||||
echo "dll"
|
||||
else
|
||||
echo "so"
|
||||
fi
|
||||
|
||||
LIB_PREFIX:
|
||||
sh: |
|
||||
if [[ "$OSTYPE" == "msys" ]] || [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then
|
||||
echo ""
|
||||
else
|
||||
echo "lib"
|
||||
fi
|
||||
|
||||
RUST_TARGET:
|
||||
sh: |
|
||||
ARCH=$(uname -m)
|
||||
OS_TYPE="$OSTYPE"
|
||||
case "$ARCH" in
|
||||
x86_64|x64)
|
||||
ARCH_STR="x86_64"
|
||||
;;
|
||||
aarch64|arm64)
|
||||
ARCH_STR="aarch64"
|
||||
;;
|
||||
armv7l|armv7)
|
||||
ARCH_STR="armv7"
|
||||
;;
|
||||
*)
|
||||
ARCH_STR="$ARCH"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "$OS_TYPE" == "darwin"* ]]; then
|
||||
echo "${ARCH_STR}-apple-darwin"
|
||||
elif [[ "$OS_TYPE" == "linux-gnu"* ]] || [[ "$OS_TYPE" == "linux"* ]]; then
|
||||
echo "${ARCH_STR}-unknown-linux-gnu"
|
||||
elif [[ "$OS_TYPE" == "msys" ]] || [[ "$OS_TYPE" == "cygwin" ]] || [[ "$OS_TYPE" == "win32" ]]; then
|
||||
echo "${ARCH_STR}-pc-windows-msvc"
|
||||
else
|
||||
echo "${ARCH_STR}-unknown-unknown"
|
||||
fi
|
||||
|
||||
IS_WINDOWS: "{{.IS_WINDOWS}}"
|
||||
IS_MACOS: "{{.IS_MACOS}}"
|
||||
IS_LINUX: "{{.IS_LINUX}}"
|
||||
|
||||
RUBY_FULL_PATH:
|
||||
sh: |
|
||||
if command -v ruby >/dev/null 2>&1; then
|
||||
command -v ruby
|
||||
elif [[ "$OSTYPE" == "darwin"* ]] && [[ -f "/opt/homebrew/opt/ruby/bin/ruby" ]]; then
|
||||
echo "/opt/homebrew/opt/ruby/bin/ruby"
|
||||
else
|
||||
echo "ruby"
|
||||
fi
|
||||
|
||||
CARGO_BIN:
|
||||
sh: command -v cargo 2>/dev/null || echo "cargo"
|
||||
|
||||
RUSTC_BIN:
|
||||
sh: command -v rustc 2>/dev/null || echo "rustc"
|
||||
|
||||
SHELL_EXT:
|
||||
sh: |
|
||||
if [[ "$OSTYPE" == "msys" ]] || [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then
|
||||
echo ".ps1"
|
||||
else
|
||||
echo ".sh"
|
||||
fi
|
||||
118
.task/config/vars.yml
Normal file
118
.task/config/vars.yml
Normal file
@@ -0,0 +1,118 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
vars:
|
||||
# Version extraction from Cargo.toml (workspace.package.version)
|
||||
VERSION:
|
||||
sh: grep -m 1 'version = ' Cargo.toml | sed 's/version = "\(.*\)"/\1/'
|
||||
|
||||
# Build profile (dev/release/ci) - default to release
|
||||
BUILD_PROFILE: '{{.BUILD_PROFILE | default "release"}}'
|
||||
|
||||
# Kreuzberg-specific versions
|
||||
ORT_VERSION: "1.24.1"
|
||||
|
||||
# Toolchain versions
|
||||
GOLANGCI_LINT_VERSION: "latest"
|
||||
|
||||
# Logging
|
||||
RUST_LOG: "info"
|
||||
|
||||
# Root project directories (absolute paths)
|
||||
ROOT: "{{.ROOT_DIR}}"
|
||||
CRATES_DIR: "{{.ROOT_DIR}}/crates"
|
||||
PACKAGES_DIR: "{{.ROOT_DIR}}/packages"
|
||||
SCRIPTS_DIR: "{{.ROOT_DIR}}/scripts"
|
||||
TOOLS_DIR: "{{.ROOT_DIR}}/tools"
|
||||
TARGET_DIR: "{{.ROOT_DIR}}/target"
|
||||
|
||||
# OS Detection
|
||||
OS:
|
||||
sh: |
|
||||
case "$(uname -s 2>/dev/null || echo 'unknown')" in
|
||||
Darwin*)
|
||||
echo "darwin"
|
||||
;;
|
||||
Linux*)
|
||||
echo "linux"
|
||||
;;
|
||||
MINGW*|MSYS*|CYGWIN*)
|
||||
echo "windows"
|
||||
;;
|
||||
*)
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
echo "darwin"
|
||||
elif [[ "$OSTYPE" == "linux-gnu"* ]] || [[ "$OSTYPE" == "linux"* ]]; then
|
||||
echo "linux"
|
||||
elif [[ "$OSTYPE" == "msys" ]] || [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then
|
||||
echo "windows"
|
||||
else
|
||||
echo "unknown"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
# OS Boolean helpers
|
||||
IS_WINDOWS:
|
||||
sh: |
|
||||
if [[ "$OSTYPE" == "msys" ]] || [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then
|
||||
echo "true"
|
||||
else
|
||||
echo "false"
|
||||
fi
|
||||
|
||||
IS_MACOS:
|
||||
sh: |
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
echo "true"
|
||||
else
|
||||
echo "false"
|
||||
fi
|
||||
|
||||
IS_LINUX:
|
||||
sh: |
|
||||
if [[ "$OSTYPE" == "linux-gnu"* ]] || [[ "$OSTYPE" == "linux"* ]]; then
|
||||
echo "true"
|
||||
else
|
||||
echo "false"
|
||||
fi
|
||||
|
||||
# Architecture detection
|
||||
ARCH:
|
||||
sh: |
|
||||
ARCH=$(uname -m)
|
||||
case "$ARCH" in
|
||||
x86_64|x64)
|
||||
echo "x86_64"
|
||||
;;
|
||||
aarch64|arm64)
|
||||
echo "arm64"
|
||||
;;
|
||||
armv7l|armv7)
|
||||
echo "armv7"
|
||||
;;
|
||||
armv6l|armv6)
|
||||
echo "armv6"
|
||||
;;
|
||||
i686|i386)
|
||||
echo "i386"
|
||||
;;
|
||||
*)
|
||||
echo "$ARCH"
|
||||
;;
|
||||
esac
|
||||
|
||||
# Number of CPUs available
|
||||
NUM_CPUS:
|
||||
sh: |
|
||||
if command -v nproc >/dev/null 2>&1; then
|
||||
nproc
|
||||
elif [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
sysctl -n hw.ncpu
|
||||
elif [[ "$OSTYPE" == "msys" ]] || [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "win32" ]]; then
|
||||
echo "${NUMBER_OF_PROCESSORS:-4}"
|
||||
else
|
||||
echo "4"
|
||||
fi
|
||||
|
||||
MAKE_JOBS: "{{.NUM_CPUS}}"
|
||||
20
.task/languages/csharp.yml
Normal file
20
.task/languages/csharp.yml
Normal file
@@ -0,0 +1,20 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
tasks:
|
||||
update:
|
||||
desc: "Update C# dependencies within major versions (dotnet outdated -u --version-lock Minor)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
cd packages/csharp && dotnet outdated -u --version-lock Minor
|
||||
ignore_error: false
|
||||
|
||||
upgrade:
|
||||
desc: "Upgrade C# dependencies to latest including breaking changes (dotnet outdated -u)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
command -v dotnet >/dev/null 2>&1 || { echo "Dotnet not found, skipping C# upgrade"; exit 0; }
|
||||
cd packages/csharp/Kreuzberg && dotnet outdated -u
|
||||
ignore_error: false
|
||||
20
.task/languages/dart.yml
Normal file
20
.task/languages/dart.yml
Normal file
@@ -0,0 +1,20 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
tasks:
|
||||
update:
|
||||
desc: "Update Dart dependencies within major versions (dart pub upgrade)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
cd packages/dart && dart pub upgrade
|
||||
ignore_error: false
|
||||
|
||||
upgrade:
|
||||
desc: "Upgrade Dart dependencies to latest including breaking changes (dart pub upgrade --major-versions)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
command -v dart >/dev/null 2>&1 || { echo "Dart not found, skipping Dart upgrade"; exit 0; }
|
||||
cd packages/dart && dart pub upgrade --major-versions
|
||||
ignore_error: false
|
||||
20
.task/languages/elixir.yml
Normal file
20
.task/languages/elixir.yml
Normal file
@@ -0,0 +1,20 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
tasks:
|
||||
update:
|
||||
desc: "Update Elixir dependencies within major versions (mix deps.update --all)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
cd packages/elixir && mix deps.update --all
|
||||
ignore_error: false
|
||||
|
||||
upgrade:
|
||||
desc: "Upgrade Elixir dependencies to latest including breaking changes (mix hex.outdated --all + mix deps.update)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
command -v mix >/dev/null 2>&1 || { echo "Elixir not found, skipping Elixir upgrade"; exit 0; }
|
||||
cd packages/elixir && mix hex.outdated --all || true && mix deps.update --all
|
||||
ignore_error: false
|
||||
20
.task/languages/go.yml
Normal file
20
.task/languages/go.yml
Normal file
@@ -0,0 +1,20 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
tasks:
|
||||
update:
|
||||
desc: "Update Go dependencies within major versions (go get -u=patch + go mod tidy)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
cd packages/go/v5 && go get -u=patch ./... && go mod tidy
|
||||
ignore_error: false
|
||||
|
||||
upgrade:
|
||||
desc: "Upgrade Go dependencies to latest including breaking changes (go get -u + go mod tidy)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
command -v go >/dev/null 2>&1 || { echo "Go not found, skipping Go upgrade"; exit 0; }
|
||||
cd packages/go/v5 && go get -u ./... && go mod tidy
|
||||
ignore_error: false
|
||||
19
.task/languages/java.yml
Normal file
19
.task/languages/java.yml
Normal file
@@ -0,0 +1,19 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
tasks:
|
||||
update:
|
||||
desc: "Update Java dependencies within major versions (mvn versions:use-next-versions)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
cd packages/java && command -v mvn >/dev/null 2>&1 && mvn versions:use-next-versions -DgenerateBackupPoms=false || echo "Maven not found, skipping Java upgrade"
|
||||
ignore_error: false
|
||||
|
||||
upgrade:
|
||||
desc: "Upgrade Java dependencies to latest including breaking changes (mvn versions:use-latest-releases)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
cd packages/java && command -v mvn >/dev/null 2>&1 && mvn versions:use-latest-releases -DgenerateBackupPoms=false || echo "Maven not found, skipping Java upgrade"
|
||||
ignore_error: false
|
||||
20
.task/languages/kotlin_android.yml
Normal file
20
.task/languages/kotlin_android.yml
Normal file
@@ -0,0 +1,20 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
tasks:
|
||||
update:
|
||||
desc: "Update Kotlin/Android dependencies within major versions (gradlew dependencyUpdates)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
cd packages/kotlin-android && ./gradlew dependencyUpdates
|
||||
ignore_error: false
|
||||
|
||||
upgrade:
|
||||
desc: "Upgrade Kotlin/Android dependencies to latest including breaking changes (gradlew useLatestVersions)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
test -f packages/kotlin-android/gradlew || { echo "Kotlin-Android gradlew not found, skipping upgrade"; exit 0; }
|
||||
cd packages/kotlin-android && ./gradlew useLatestVersions
|
||||
ignore_error: false
|
||||
19
.task/languages/node.yml
Normal file
19
.task/languages/node.yml
Normal file
@@ -0,0 +1,19 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
tasks:
|
||||
update:
|
||||
desc: "Update Node.js dependencies within major versions (pnpm up)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
pnpm up
|
||||
ignore_error: false
|
||||
|
||||
upgrade:
|
||||
desc: "Upgrade Node.js dependencies to latest including breaking changes (pnpm up --latest)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
pnpm up --latest
|
||||
ignore_error: false
|
||||
20
.task/languages/php.yml
Normal file
20
.task/languages/php.yml
Normal file
@@ -0,0 +1,20 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
tasks:
|
||||
update:
|
||||
desc: "Update PHP dependencies within major versions (composer update)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
cd packages/php && composer update
|
||||
ignore_error: false
|
||||
|
||||
upgrade:
|
||||
desc: "Upgrade PHP dependencies to latest including breaking changes (composer update --with-all-dependencies)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
command -v composer >/dev/null 2>&1 || { echo "Composer not found, skipping PHP upgrade"; exit 0; }
|
||||
cd packages/php && composer update --with-all-dependencies
|
||||
ignore_error: false
|
||||
22
.task/languages/python.yml
Normal file
22
.task/languages/python.yml
Normal file
@@ -0,0 +1,22 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
vars:
|
||||
PYTHON_PKG: "packages/python"
|
||||
|
||||
tasks:
|
||||
update:
|
||||
desc: "Update Python dependencies within major versions (uv sync --upgrade-package)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
cd {{.PYTHON_PKG}} && uv sync --no-install-project --no-install-workspace --all-extras --upgrade-package "*" --no-prerelease
|
||||
ignore_error: false
|
||||
|
||||
upgrade:
|
||||
desc: "Upgrade Python dependencies to latest including breaking changes (uv sync --upgrade)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
cd {{.PYTHON_PKG}} && uv sync --no-install-project --no-install-workspace --all-extras --upgrade
|
||||
ignore_error: false
|
||||
20
.task/languages/r.yml
Normal file
20
.task/languages/r.yml
Normal file
@@ -0,0 +1,20 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
tasks:
|
||||
update:
|
||||
desc: "Update R dependencies within major versions (devtools::update_packages(check.built = TRUE))"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
cd packages/r && Rscript -e "devtools::update_packages(check.built = TRUE)"
|
||||
ignore_error: false
|
||||
|
||||
upgrade:
|
||||
desc: "Upgrade R dependencies to latest including breaking changes (update.packages())"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
command -v Rscript >/dev/null 2>&1 || { echo "R not found, skipping R upgrade"; exit 0; }
|
||||
cd packages/r && Rscript -e 'options(repos = c(CRAN = "https://cloud.r-project.org")); update.packages(ask = FALSE)'
|
||||
ignore_error: false
|
||||
20
.task/languages/ruby.yml
Normal file
20
.task/languages/ruby.yml
Normal file
@@ -0,0 +1,20 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
tasks:
|
||||
update:
|
||||
desc: "Update Ruby dependencies within major versions (bundle update --conservative)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
cd packages/ruby && bundle update --conservative
|
||||
ignore_error: false
|
||||
|
||||
upgrade:
|
||||
desc: "Upgrade Ruby dependencies to latest including breaking changes (bundle update)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
command -v bundle >/dev/null 2>&1 || { echo "Bundle not found, skipping Ruby upgrade"; exit 0; }
|
||||
cd packages/ruby && bundle update
|
||||
ignore_error: false
|
||||
247
.task/languages/rust.yml
Normal file
247
.task/languages/rust.yml
Normal file
@@ -0,0 +1,247 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
includes:
|
||||
platforms: ../config/platforms.yml
|
||||
|
||||
vars:
|
||||
RUST_LOG: '{{.RUST_LOG | default "info"}}'
|
||||
BUILD_PROFILE: '{{.BUILD_PROFILE | default "release"}}'
|
||||
RUST_BACKTRACE: '{{.RUST_BACKTRACE | default "1"}}'
|
||||
CARGO_TERM_COLOR: "always"
|
||||
|
||||
tasks:
|
||||
install:
|
||||
desc: "Install Rust toolchain and components"
|
||||
cmds:
|
||||
- rustup update stable
|
||||
- rustup component add rustfmt clippy
|
||||
# Windows: install lld-link to avoid PATH conflicts with Git Bash's link.exe
|
||||
- cmd: rustup component add llvm-tools
|
||||
platforms: [windows]
|
||||
- cargo install cargo-llvm-cov --locked
|
||||
- cargo install cargo-edit --locked
|
||||
- cargo --version
|
||||
- rustc --version
|
||||
|
||||
build:
|
||||
desc: "Build all Rust crates with {{.BUILD_PROFILE}} profile"
|
||||
cmds:
|
||||
- task: build:{{.BUILD_PROFILE | default "release"}}
|
||||
|
||||
build:dev:
|
||||
desc: "Build all Rust crates in debug mode"
|
||||
cmds:
|
||||
- cmd: cargo build --workspace --all-features --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm
|
||||
platforms: [linux, darwin]
|
||||
# Note: exclude benchmark-harness on Windows as jemalloc doesn't build with MSVC
|
||||
- cmd: cargo build --workspace --all-features --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm --exclude benchmark-harness
|
||||
platforms: [windows]
|
||||
|
||||
build:release:
|
||||
desc: "Build all Rust crates in release mode"
|
||||
cmds:
|
||||
- cmd: cargo build --release --workspace --all-features --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm
|
||||
platforms: [linux, darwin]
|
||||
- cmd: cargo build --release --workspace --all-features --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm --exclude benchmark-harness
|
||||
platforms: [windows]
|
||||
|
||||
build:ci:
|
||||
desc: "Build for CI with debug info enabled"
|
||||
env:
|
||||
RUSTFLAGS: "-C debuginfo=2"
|
||||
cmds:
|
||||
- cmd: cargo build --release --workspace --all-features --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm
|
||||
platforms: [linux, darwin]
|
||||
- cmd: cargo build --release --workspace --all-features --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm --exclude benchmark-harness
|
||||
platforms: [windows]
|
||||
|
||||
build:profiling:
|
||||
desc: "Build all Rust crates with profiling features (for flamegraph generation)"
|
||||
env:
|
||||
ENABLE_PROFILING: "true"
|
||||
RUSTFLAGS: "-g"
|
||||
cmds:
|
||||
- cargo build --workspace --release --features full,profiling,api,mcp,otel --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm
|
||||
- cargo build --manifest-path tools/benchmark-harness/Cargo.toml --release --features profiling
|
||||
|
||||
cli:build:
|
||||
desc: "Build CLI binary"
|
||||
cmds:
|
||||
- cargo build --release --package kreuzberg-cli {{if .TARGET}}--target {{.TARGET}}{{end}}
|
||||
|
||||
cli:build:dev:
|
||||
desc: "Build CLI binary in debug mode"
|
||||
cmds:
|
||||
- cargo build --package kreuzberg-cli
|
||||
|
||||
cli:install:
|
||||
desc: "Install CLI binary"
|
||||
cmds:
|
||||
- cargo install --path crates/kreuzberg-cli
|
||||
|
||||
ffi:build:
|
||||
desc: "Build FFI library for language bindings"
|
||||
cmds:
|
||||
- cargo build --release --package kreuzberg-ffi {{.CARGO_ARGS | default ""}}
|
||||
|
||||
ffi:build:dev:
|
||||
desc: "Build FFI library in debug mode"
|
||||
cmds:
|
||||
- cargo build --package kreuzberg-ffi {{.CARGO_ARGS | default ""}}
|
||||
|
||||
ffi:build:ci:
|
||||
desc: "Build FFI library in CI mode (with debug info)"
|
||||
env:
|
||||
RUSTFLAGS: "-C debuginfo=2"
|
||||
cmds:
|
||||
- cargo build --release --package kreuzberg-ffi {{.CARGO_ARGS | default ""}}
|
||||
|
||||
wasm:build:all:
|
||||
desc: "Build all WASM targets (web/bundler/nodejs/deno) plus the TypeScript wrapper"
|
||||
dir: crates/kreuzberg-wasm
|
||||
cmds:
|
||||
- pnpm run build:all
|
||||
|
||||
test:
|
||||
desc: "Run all Rust tests"
|
||||
cmds:
|
||||
- cmd: bash scripts/ci/rust/run-unit-tests.sh
|
||||
platforms: [linux, darwin]
|
||||
|
||||
test:ci:
|
||||
desc: "Run Rust tests in CI mode with tessdata setup"
|
||||
env:
|
||||
RUST_BACKTRACE: "{{.RUST_BACKTRACE}}"
|
||||
CARGO_TERM_COLOR: "{{.CARGO_TERM_COLOR}}"
|
||||
cmds:
|
||||
- cmd: bash scripts/ci/rust/run-unit-tests.sh
|
||||
platforms: [linux, darwin]
|
||||
|
||||
test:quick:
|
||||
desc: "Run fast Rust tests (unit tests only)"
|
||||
cmds:
|
||||
- cargo test --lib --workspace --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm
|
||||
|
||||
lint:
|
||||
desc: "Lint Rust code WITH auto-fix (cargo fmt + cargo clippy --fix)"
|
||||
cmds:
|
||||
- cargo fmt --all
|
||||
- cargo clippy --workspace --fix --allow-dirty --allow-staged --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm
|
||||
|
||||
lint:check:
|
||||
desc: "Lint Rust code WITHOUT auto-fix (check-only)"
|
||||
cmds:
|
||||
- cargo fmt --all -- --check
|
||||
- cargo clippy --workspace --exclude kreuzberg-php --exclude kreuzberg-node --exclude kreuzberg-wasm -- -D warnings
|
||||
|
||||
format:
|
||||
desc: "Format Rust code (with modifications)"
|
||||
cmds:
|
||||
- cargo fmt --all
|
||||
|
||||
format:check:
|
||||
desc: "Check Rust formatting without modifications"
|
||||
cmds:
|
||||
- cargo fmt --all -- --check
|
||||
|
||||
licenses:
|
||||
desc: "Check Rust dependency licenses with cargo-deny"
|
||||
cmds:
|
||||
- cargo deny check licenses
|
||||
|
||||
doc:
|
||||
desc: "Generate Rust documentation and open in browser"
|
||||
cmds:
|
||||
- cargo doc --no-deps --open
|
||||
|
||||
doc:build:
|
||||
desc: "Build Rust documentation (without opening)"
|
||||
cmds:
|
||||
- cargo doc --no-deps
|
||||
|
||||
clean:
|
||||
desc: "Clean Rust build artifacts"
|
||||
cmds:
|
||||
- cargo clean
|
||||
|
||||
cache:cleanup:
|
||||
desc: "Clean up large build artifacts to reduce cache size (for CI)"
|
||||
cmds:
|
||||
- echo "Cleaning up large build artifacts to reduce cache size..."
|
||||
- cmd: find target -type f -name "*.rlib" -size +10M -exec rm -f {} \; 2>/dev/null || true
|
||||
ignore_error: true
|
||||
- cmd: find target -type f -name "*.so" -size +10M -exec rm -f {} \; 2>/dev/null || true
|
||||
ignore_error: true
|
||||
- cmd: find target -type f -name "*.dylib" -size +10M -exec rm -f {} \; 2>/dev/null || true
|
||||
ignore_error: true
|
||||
- cmd: find target -type f -name "*.dll" -size +10M -exec rm -f {} \; 2>/dev/null || true
|
||||
ignore_error: true
|
||||
- cmd: rm -rf target/*/incremental 2>/dev/null || true
|
||||
ignore_error: true
|
||||
- echo "Cleanup completed successfully"
|
||||
- cmd: du -sh target 2>/dev/null || echo "No target directory found"
|
||||
ignore_error: true
|
||||
|
||||
update:
|
||||
desc: "Update Rust dependencies (compatible only)"
|
||||
cmds:
|
||||
- cargo update
|
||||
- cmd: cargo update --manifest-path packages/ruby/ext/kreuzberg_rb/native/Cargo.toml
|
||||
ignore_error: true
|
||||
|
||||
upgrade:
|
||||
desc: "Upgrade Rust dependencies aggressively (cargo upgrade --incompatible)"
|
||||
cmds:
|
||||
- cargo upgrade --incompatible
|
||||
- cargo update
|
||||
- cmd: cargo upgrade --incompatible --manifest-path packages/ruby/ext/kreuzberg_rb/native/Cargo.toml
|
||||
ignore_error: true
|
||||
- cmd: cargo update --manifest-path packages/ruby/ext/kreuzberg_rb/native/Cargo.toml
|
||||
ignore_error: true
|
||||
|
||||
e2e:format:
|
||||
desc: "Format generated Rust E2E sources"
|
||||
cmds:
|
||||
- cargo fmt --manifest-path e2e/rust/Cargo.toml
|
||||
|
||||
e2e:lint:
|
||||
desc: "Lint Rust E2E crate with clippy"
|
||||
cmds:
|
||||
- cargo clippy --manifest-path e2e/rust/Cargo.toml -- -D warnings
|
||||
|
||||
e2e:test:
|
||||
desc: "Run Rust E2E tests"
|
||||
env:
|
||||
RUST_TEST_THREADS: "1"
|
||||
cmds:
|
||||
- cargo test --manifest-path e2e/rust/Cargo.toml --release
|
||||
|
||||
check:android:
|
||||
desc: "cargo check kreuzberg-dart + kreuzberg-ffi for Android ABIs (requires ANDROID_NDK_HOME and cargo-ndk)"
|
||||
preconditions:
|
||||
- sh: command -v cargo-ndk
|
||||
msg: "cargo-ndk not installed. Run: cargo install cargo-ndk --locked"
|
||||
- sh: 'test -n "${ANDROID_NDK_HOME:-}"'
|
||||
msg: "ANDROID_NDK_HOME is not set. Install Android NDK and export ANDROID_NDK_HOME=<path-to-ndk>"
|
||||
cmds:
|
||||
- cargo ndk --target arm64-v8a --platform 21 -- check -p kreuzberg-dart
|
||||
- cargo ndk --target x86_64 --platform 21 -- check -p kreuzberg-dart
|
||||
- cargo ndk --target arm64-v8a --platform 21 -- check -p kreuzberg-ffi
|
||||
- cargo ndk --target x86_64 --platform 21 -- check -p kreuzberg-ffi
|
||||
|
||||
check:ios:
|
||||
desc: "cargo check kreuzberg-dart + kreuzberg-swift for iOS targets (macOS only)"
|
||||
platforms: [darwin]
|
||||
cmds:
|
||||
- rustup target add aarch64-apple-ios aarch64-apple-ios-sim
|
||||
- cargo check -p kreuzberg-dart --target aarch64-apple-ios
|
||||
- cargo check -p kreuzberg-dart --target aarch64-apple-ios-sim
|
||||
- cargo check -p kreuzberg-swift --target aarch64-apple-ios
|
||||
- cargo check -p kreuzberg-swift --target aarch64-apple-ios-sim
|
||||
|
||||
check:mobile:
|
||||
desc: "cargo check Android + iOS mobile targets"
|
||||
cmds:
|
||||
- task: check:android
|
||||
- task: check:ios
|
||||
20
.task/languages/swift.yml
Normal file
20
.task/languages/swift.yml
Normal file
@@ -0,0 +1,20 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
tasks:
|
||||
update:
|
||||
desc: "Update Swift dependencies (swift package update)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
cd packages/swift && swift package update
|
||||
ignore_error: false
|
||||
|
||||
upgrade:
|
||||
desc: "Upgrade Swift dependencies (swift package update)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
command -v swift >/dev/null 2>&1 || { echo "Swift not found, skipping Swift upgrade"; exit 0; }
|
||||
cd packages/swift && swift package update
|
||||
ignore_error: false
|
||||
19
.task/languages/zig.yml
Normal file
19
.task/languages/zig.yml
Normal file
@@ -0,0 +1,19 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
tasks:
|
||||
update:
|
||||
desc: "Update Zig dependencies (zig fetch --update-hash)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
cd packages/zig && zig fetch --update-hash
|
||||
ignore_error: false
|
||||
|
||||
upgrade:
|
||||
desc: "Upgrade Zig dependencies (zig package manager)"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
echo "Zig package updates are manual; see packages/zig/build.zig.zon"
|
||||
ignore_error: false
|
||||
153
.task/tools/demo.yml
Normal file
153
.task/tools/demo.yml
Normal file
@@ -0,0 +1,153 @@
|
||||
version: "3"
|
||||
|
||||
# ============================================================================
|
||||
# Demo development environment
|
||||
#
|
||||
# Generates a patched demo-dev.html (CDN URLs → localhost), then starts two
|
||||
# servers that reproduce the cross-origin scenario the CDN creates in prod:
|
||||
#
|
||||
# Docs server → http://localhost:8001 (serves docs/)
|
||||
# Asset server → http://localhost:9000 (serves crates/kreuzberg-wasm/)
|
||||
#
|
||||
# First-time setup:
|
||||
# task demo:dev:setup # install wasm-pack, wasm-bindgen-cli, WASI SDK
|
||||
#
|
||||
# Usage:
|
||||
# task demo:dev # full build + patch + both servers
|
||||
# SKIP_WASM_BUILD=1 task demo:dev # skip Rust build, TS + patch only
|
||||
#
|
||||
# Open: http://localhost:8001/demo-dev.html
|
||||
# ============================================================================
|
||||
|
||||
tasks:
|
||||
dev:
|
||||
desc: "Build, patch demo for localhost, and start both dev servers"
|
||||
cmds:
|
||||
- task: dev:build
|
||||
- task: dev:patch
|
||||
- task: dev:ready
|
||||
- task: dev:serve
|
||||
|
||||
dev:build:
|
||||
desc: "Build WASM binary for web target (unless SKIP_WASM_BUILD=1)"
|
||||
dir: crates/kreuzberg-wasm
|
||||
cmds:
|
||||
- '[ "${SKIP_WASM_BUILD:-0}" = "1" ] || { pnpm run build:wasm:web && mkdir -p dist/pkg && cp pkg/web/kreuzberg_wasm_bg.wasm dist/pkg/kreuzberg_wasm_bg.wasm; }'
|
||||
|
||||
dev:patch:
|
||||
desc: "Generate docs/demo-dev.html with CDN URLs replaced by localhost:9000"
|
||||
cmds:
|
||||
- node scripts/task/patch-demo-dev.mjs
|
||||
|
||||
dev:ready:
|
||||
desc: "Print the ready banner"
|
||||
cmds:
|
||||
- |
|
||||
printf '\n'
|
||||
printf ' ┌─────────────────────────────────────────────────────┐\n'
|
||||
printf ' │ Demo dev environment ready │\n'
|
||||
printf ' │ │\n'
|
||||
printf ' │ Open → http://localhost:8001/demo-dev.html │\n'
|
||||
printf ' │ │\n'
|
||||
printf ' │ Assets served from http://localhost:9000 │\n'
|
||||
printf ' │ (same cross-origin setup as the live CDN) │\n'
|
||||
printf ' │ │\n'
|
||||
printf ' │ Ctrl+C to stop both servers │\n'
|
||||
printf ' └─────────────────────────────────────────────────────┘\n'
|
||||
printf '\n'
|
||||
|
||||
dev:serve:
|
||||
desc: "Start docs server (port 8001) and WASM asset server (port 9000) in parallel"
|
||||
deps:
|
||||
- task: dev:serve:docs
|
||||
- task: dev:serve:assets
|
||||
|
||||
dev:serve:docs:
|
||||
desc: "Serve docs on http://localhost:8001"
|
||||
cmds:
|
||||
- uv run --no-sync --group doc zensical serve -a localhost:8001
|
||||
|
||||
dev:serve:assets:
|
||||
desc: "Serve WASM assets on http://localhost:9000 (simulates CDN origin)"
|
||||
dir: crates/kreuzberg-wasm
|
||||
cmds:
|
||||
- npx --yes serve . --cors -l tcp://0.0.0.0:9000
|
||||
|
||||
dev:setup:
|
||||
desc: "Install demo:dev prerequisites (wasm-pack, wasm-bindgen-cli, WASI SDK 25)"
|
||||
cmds:
|
||||
- task: dev:setup:wasm-pack
|
||||
- task: dev:setup:wasm-bindgen-cli
|
||||
- task: dev:setup:wasi-sdk
|
||||
|
||||
dev:setup:wasm-pack:
|
||||
desc: "Install wasm-pack 0.13.1 (matches CI pinned version)"
|
||||
vars:
|
||||
WASM_PACK_VERSION: "0.13.1"
|
||||
cmds:
|
||||
- |
|
||||
if command -v wasm-pack &>/dev/null && [ "$(wasm-pack --version 2>/dev/null | awk '{print $2}')" = "{{.WASM_PACK_VERSION}}" ]; then
|
||||
echo "wasm-pack {{.WASM_PACK_VERSION}} already installed"
|
||||
exit 0
|
||||
fi
|
||||
cargo install wasm-pack --version {{.WASM_PACK_VERSION}} --locked
|
||||
|
||||
dev:setup:wasm-bindgen-cli:
|
||||
desc: "Install wasm-bindgen-cli at the version pinned in Cargo.lock"
|
||||
vars:
|
||||
WASM_BINDGEN_VERSION:
|
||||
sh: grep -A 3 '^name = "wasm-bindgen"$' Cargo.lock | grep '^version' | head -1 | awk '{print $3}' | tr -d '"'
|
||||
cmds:
|
||||
- |
|
||||
if command -v wasm-bindgen &>/dev/null && [ "$(wasm-bindgen --version 2>/dev/null | awk '{print $2}')" = "{{.WASM_BINDGEN_VERSION}}" ]; then
|
||||
echo "wasm-bindgen-cli {{.WASM_BINDGEN_VERSION}} already installed"
|
||||
exit 0
|
||||
fi
|
||||
cargo install wasm-bindgen-cli --version {{.WASM_BINDGEN_VERSION}} --locked
|
||||
|
||||
dev:setup:wasi-sdk:
|
||||
desc: "Install WASI SDK 25 to ~/wasi-sdk-25 (skips if already present)"
|
||||
vars:
|
||||
WASI_SDK_VERSION: "25"
|
||||
cmds:
|
||||
- |
|
||||
WASI_SDK_DIR="${WASI_SDK_PATH:-$HOME/wasi-sdk-{{.WASI_SDK_VERSION}}}"
|
||||
if [ -d "$WASI_SDK_DIR/share/wasi-sysroot" ]; then
|
||||
echo "WASI SDK {{.WASI_SDK_VERSION}} already installed at $WASI_SDK_DIR"
|
||||
exit 0
|
||||
fi
|
||||
OS=$(uname -s | tr '[:upper:]' '[:lower:]')
|
||||
ARCH=$(uname -m)
|
||||
case "$OS-$ARCH" in
|
||||
linux-x86_64) PKG="wasi-sdk-{{.WASI_SDK_VERSION}}.0-x86_64-linux.tar.gz" ;;
|
||||
linux-aarch64) PKG="wasi-sdk-{{.WASI_SDK_VERSION}}.0-arm64-linux.tar.gz" ;;
|
||||
darwin-x86_64) PKG="wasi-sdk-{{.WASI_SDK_VERSION}}.0-x86_64-macos.tar.gz" ;;
|
||||
darwin-arm64) PKG="wasi-sdk-{{.WASI_SDK_VERSION}}.0-arm64-macos.tar.gz" ;;
|
||||
*)
|
||||
echo "Unsupported platform: $OS-$ARCH"
|
||||
echo "Download manually: https://github.com/WebAssembly/wasi-sdk/releases/tag/wasi-sdk-{{.WASI_SDK_VERSION}}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
BASE_URL="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-{{.WASI_SDK_VERSION}}"
|
||||
TMPFILE=$(mktemp /tmp/wasi-sdk-XXXXXX.tar.gz)
|
||||
echo "Downloading WASI SDK {{.WASI_SDK_VERSION}} ..."
|
||||
curl -fL "${BASE_URL}/${PKG}" -o "$TMPFILE"
|
||||
curl -fL "${BASE_URL}/${PKG}.sha256" -o "${TMPFILE}.sha256"
|
||||
echo "Verifying SHA256 ..."
|
||||
EXPECTED_HASH=$(awk '{print $1}' "${TMPFILE}.sha256")
|
||||
if command -v sha256sum &>/dev/null; then
|
||||
echo "${EXPECTED_HASH} ${TMPFILE}" | sha256sum -c -
|
||||
elif command -v shasum &>/dev/null; then
|
||||
echo "${EXPECTED_HASH} ${TMPFILE}" | shasum -a 256 -c -
|
||||
else
|
||||
echo "Warning: no SHA256 tool found, skipping checksum verification"
|
||||
fi
|
||||
echo "Extracting to $WASI_SDK_DIR ..."
|
||||
mkdir -p "$WASI_SDK_DIR"
|
||||
tar -xzf "$TMPFILE" -C "$WASI_SDK_DIR" --strip-components=1
|
||||
rm -f "$TMPFILE" "${TMPFILE}.sha256"
|
||||
echo ""
|
||||
echo "WASI SDK {{.WASI_SDK_VERSION}} installed at $WASI_SDK_DIR"
|
||||
echo "Add to your shell profile:"
|
||||
echo " export WASI_SDK_PATH=$WASI_SDK_DIR"
|
||||
79
.task/tools/docs.yml
Normal file
79
.task/tools/docs.yml
Normal file
@@ -0,0 +1,79 @@
|
||||
version: "3"
|
||||
|
||||
includes:
|
||||
_cfg:
|
||||
taskfile: ../config/vars.yml
|
||||
internal: true
|
||||
|
||||
vars:
|
||||
ALEF_SNIPPETS_TAG: v0.16.69
|
||||
|
||||
tasks:
|
||||
build:
|
||||
desc: Build documentation
|
||||
cmds:
|
||||
- uv run --no-sync --group doc zensical build --clean
|
||||
|
||||
build:strict:
|
||||
desc: Build documentation in strict mode (CI-equivalent)
|
||||
cmds:
|
||||
- uv run --no-sync --group doc zensical build --strict --clean
|
||||
|
||||
serve:
|
||||
desc: Serve documentation locally with live reload
|
||||
cmds:
|
||||
- uv run --no-sync --group doc zensical serve
|
||||
|
||||
lint:links:
|
||||
desc: Check links with lychee
|
||||
cmds:
|
||||
# docs/overrides/main.html contains Jinja template variables (e.g.
|
||||
# `{{ page.edit_url }}`) that lychee interprets as broken local file
|
||||
# links — skip that path; the rendered site is link-checked separately.
|
||||
- lychee --offline --root-dir "$PWD/docs" --exclude-path "docs/overrides" docs/ README.md
|
||||
|
||||
lint:prose:
|
||||
desc: Lint prose with textlint (via prek hook)
|
||||
cmds:
|
||||
- prek run textlint --all-files
|
||||
|
||||
snippets:install:
|
||||
desc: Install alef CLI (which now includes the snippets subcommand) if missing
|
||||
cmds:
|
||||
- cmd: |
|
||||
if ! command -v alef >/dev/null 2>&1; then
|
||||
cargo install --locked --git https://github.com/kreuzberg-dev/alef --tag {{.ALEF_SNIPPETS_TAG}} alef-cli
|
||||
fi
|
||||
ignore_error: false
|
||||
|
||||
snippets:validate:
|
||||
desc: Validate all docs/snippets via `alef snippets` (syntax level)
|
||||
deps: ["snippets:install"]
|
||||
cmds:
|
||||
- alef snippets validate --snippets docs/snippets --level syntax --timeout 120 -j 4
|
||||
|
||||
snippets:validate:lang:
|
||||
desc: "Validate snippets for one language (set LANG=dart|kotlin|swift|...)"
|
||||
deps: ["snippets:install"]
|
||||
cmds:
|
||||
- alef snippets validate --snippets docs/snippets --level syntax --languages {{.LANG}} --timeout 120 -j 4
|
||||
requires:
|
||||
vars: [LANG]
|
||||
|
||||
snippets:list:
|
||||
desc: List all snippets discovered under docs/snippets
|
||||
deps: ["snippets:install"]
|
||||
cmds:
|
||||
- alef snippets list --snippets docs/snippets
|
||||
|
||||
snippets:audit:
|
||||
desc: Audit snippet metadata and coverage
|
||||
deps: ["snippets:install"]
|
||||
cmds:
|
||||
- alef snippets audit --snippets docs/snippets
|
||||
|
||||
snippets:gaps:
|
||||
desc: Report missing snippet variants across languages
|
||||
deps: ["snippets:install"]
|
||||
cmds:
|
||||
- alef snippets gaps --snippets docs/snippets
|
||||
30
.task/tools/fixtures.yml
Normal file
30
.task/tools/fixtures.yml
Normal file
@@ -0,0 +1,30 @@
|
||||
version: "3"
|
||||
|
||||
# ============================================================================
|
||||
# Test fixture generation
|
||||
#
|
||||
# Generates DOCX / ODT / XLSX / PPTX / PDF documents with track-changes,
|
||||
# revisions, comments, incremental updates, paired diff inputs, and
|
||||
# security edge cases — each paired with a *.gt.json ground-truth sidecar.
|
||||
#
|
||||
# Source: tools/generate_test_fixtures/
|
||||
# Output: test_documents/generated/<format>/
|
||||
#
|
||||
# Usage:
|
||||
# task fixtures:generate # produce every fixture
|
||||
# task fixtures:generate -- docx # produce a single category
|
||||
# task fixtures:test # run the toolkit's smoke test
|
||||
# ============================================================================
|
||||
|
||||
tasks:
|
||||
generate:
|
||||
desc: "Generate all (or selected) test fixtures + ground-truth sidecars"
|
||||
dir: tools/generate_test_fixtures
|
||||
cmds:
|
||||
- uv run python -m generate_test_fixtures {{.CLI_ARGS | default "all"}}
|
||||
|
||||
test:
|
||||
desc: "Run the fixture generator's smoke test suite"
|
||||
dir: tools/generate_test_fixtures
|
||||
cmds:
|
||||
- uv run pytest -q
|
||||
55
.task/tools/general.yml
Normal file
55
.task/tools/general.yml
Normal file
@@ -0,0 +1,55 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
includes:
|
||||
platforms: ../config/platforms.yml
|
||||
|
||||
vars:
|
||||
SCRIPTS_DIR: "{{.TASKFILE_DIR}}/../../scripts"
|
||||
|
||||
tasks:
|
||||
pre-commit:install:
|
||||
desc: "Install prek pre-commit hooks for commit and commit-msg"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: prek install
|
||||
ignore_error: false
|
||||
- cmd: prek install --hook-type commit-msg
|
||||
ignore_error: false
|
||||
|
||||
pre-commit:run:
|
||||
desc: "Run prek pre-commit hooks on all files"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: prek run --all-files
|
||||
ignore_error: false
|
||||
|
||||
pre-commit:uninstall:
|
||||
desc: "Uninstall prek hooks"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: prek uninstall
|
||||
ignore_error: true
|
||||
- cmd: prek uninstall --hook-type commit-msg
|
||||
ignore_error: true
|
||||
|
||||
validate:config:
|
||||
desc: "Validate YAML task configuration files"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
for file in {{.TASKFILE_DIR}}/**/*.yml; do
|
||||
echo "Validating $file..."
|
||||
if ! command -v yamllint &> /dev/null; then
|
||||
echo "yamllint not found, skipping validation"
|
||||
break
|
||||
fi
|
||||
yamllint "$file" || exit 1
|
||||
done
|
||||
ignore_error: false
|
||||
|
||||
validate:all:
|
||||
desc: "Validate all project configurations"
|
||||
silent: false
|
||||
cmds:
|
||||
- task: validate:config
|
||||
127
.task/tools/test-apps.yml
Normal file
127
.task/tools/test-apps.yml
Normal file
@@ -0,0 +1,127 @@
|
||||
version: "3"
|
||||
|
||||
tasks:
|
||||
smoke:
|
||||
desc: Run all test_app smoke tests
|
||||
cmds:
|
||||
- task: smoke:python
|
||||
- task: smoke:node
|
||||
- task: smoke:ruby
|
||||
- task: smoke:go
|
||||
- task: smoke:java
|
||||
- task: smoke:elixir
|
||||
- task: smoke:csharp
|
||||
- task: smoke:php
|
||||
- task: smoke:r
|
||||
- task: smoke:rust
|
||||
- task: smoke:wasm
|
||||
|
||||
smoke:python:
|
||||
desc: Run Python test_app smoke test
|
||||
dir: test_apps/python
|
||||
cmds:
|
||||
- uv sync
|
||||
- uv run --no-sync pytest smoke_test.py -v
|
||||
|
||||
smoke:node:
|
||||
desc: Run Node.js test_app smoke test
|
||||
dir: test_apps/node
|
||||
cmds:
|
||||
- pnpm install
|
||||
- pnpm run test:smoke
|
||||
|
||||
smoke:ruby:
|
||||
desc: Run Ruby test_app smoke test
|
||||
dir: test_apps/ruby
|
||||
cmds:
|
||||
- bundle install
|
||||
- bundle exec rspec smoke_test.rb
|
||||
|
||||
smoke:go:
|
||||
desc: Run Go test_app smoke test
|
||||
dir: test_apps/go
|
||||
cmds:
|
||||
- go test -v -run TestBasicFixtures
|
||||
- go test -v -run TestParseValidation
|
||||
|
||||
smoke:java:
|
||||
desc: Run Java test_app smoke test
|
||||
dir: test_apps/java
|
||||
cmds:
|
||||
- mvn test -Dtest=SmokeTest
|
||||
|
||||
smoke:elixir:
|
||||
desc: Run Elixir test_app smoke test
|
||||
dir: test_apps/elixir
|
||||
cmds:
|
||||
- mix deps.get
|
||||
- mix test
|
||||
|
||||
smoke:csharp:
|
||||
desc: Run C# test_app smoke test
|
||||
dir: test_apps/csharp
|
||||
cmds:
|
||||
- dotnet restore
|
||||
- dotnet test
|
||||
|
||||
smoke:php:
|
||||
desc: Run PHP test_app smoke test
|
||||
dir: test_apps/php
|
||||
cmds:
|
||||
- composer install
|
||||
- vendor/bin/phpunit smoke_test.php
|
||||
|
||||
smoke:r:
|
||||
desc: Run R test_app smoke test
|
||||
dir: test_apps/r
|
||||
cmds:
|
||||
- Rscript run_tests.R
|
||||
|
||||
smoke:rust:
|
||||
desc: Run Rust test_app smoke test
|
||||
dir: test_apps/rust
|
||||
cmds:
|
||||
- cargo run
|
||||
|
||||
smoke:wasm:
|
||||
desc: Run WASM test_app smoke test
|
||||
dir: test_apps/wasm
|
||||
cmds:
|
||||
- pnpm install
|
||||
- pnpm run test:smoke
|
||||
|
||||
comprehensive:
|
||||
desc: Run all test_app comprehensive tests
|
||||
cmds:
|
||||
- task: comprehensive:python
|
||||
- task: comprehensive:node
|
||||
- task: comprehensive:ruby
|
||||
- task: comprehensive:go
|
||||
|
||||
comprehensive:python:
|
||||
desc: Run Python test_app comprehensive tests
|
||||
dir: test_apps/python
|
||||
cmds:
|
||||
- uv sync
|
||||
- uv run --no-sync pytest comprehensive_test.py -v
|
||||
|
||||
comprehensive:node:
|
||||
desc: Run Node.js test_app comprehensive tests
|
||||
dir: test_apps/node
|
||||
cmds:
|
||||
- pnpm install
|
||||
- pnpm run test:comprehensive
|
||||
|
||||
comprehensive:ruby:
|
||||
desc: Run Ruby test_app comprehensive tests
|
||||
dir: test_apps/ruby
|
||||
cmds:
|
||||
- bundle install
|
||||
- bundle exec rspec comprehensive_test.rb
|
||||
|
||||
comprehensive:go:
|
||||
desc: Run Go test_app comprehensive tests
|
||||
dir: test_apps/go
|
||||
cmds:
|
||||
- go test -v -run TestProcessFixtures
|
||||
- go test -v -run TestChunkingFixtures
|
||||
57
.task/tools/version-sync.yml
Normal file
57
.task/tools/version-sync.yml
Normal file
@@ -0,0 +1,57 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
includes:
|
||||
platforms: ../config/platforms.yml
|
||||
|
||||
vars:
|
||||
ALEF: "alef"
|
||||
|
||||
tasks:
|
||||
sync:
|
||||
desc: "Synchronize version across all package manifests and regenerate everything"
|
||||
cmds:
|
||||
- "{{.ALEF}} sync-versions"
|
||||
- "{{.ALEF}} readme"
|
||||
- "{{.ALEF}} docs"
|
||||
- "{{.ALEF}} generate --clean"
|
||||
- "{{.ALEF}} stubs"
|
||||
- "{{.ALEF}} e2e generate"
|
||||
|
||||
check:
|
||||
desc: "Check if versions are synchronized (dry-run)"
|
||||
cmds:
|
||||
- cmd: grep -m 1 'version = ' Cargo.toml | sed 's/version = "\(.*\)"/\1/'
|
||||
silent: false
|
||||
|
||||
bump:major:
|
||||
desc: "Bump major version (X.0.0) and sync"
|
||||
cmds:
|
||||
- "{{.ALEF}} sync-versions --bump major"
|
||||
- task: sync
|
||||
|
||||
bump:minor:
|
||||
desc: "Bump minor version (0.X.0) and sync"
|
||||
cmds:
|
||||
- "{{.ALEF}} sync-versions --bump minor"
|
||||
- task: sync
|
||||
|
||||
bump:patch:
|
||||
desc: "Bump patch version (0.0.X) and sync"
|
||||
cmds:
|
||||
- "{{.ALEF}} sync-versions --bump patch"
|
||||
- task: sync
|
||||
|
||||
set:
|
||||
desc: "Set explicit version and sync (usage: task version:set -- 4.10.0-rc.1)"
|
||||
requires:
|
||||
vars: [CLI_ARGS]
|
||||
cmds:
|
||||
- "{{.ALEF}} sync-versions --set {{.CLI_ARGS}}"
|
||||
- task: sync
|
||||
|
||||
show:
|
||||
desc: "Show current version from Cargo.toml"
|
||||
cmds:
|
||||
- cmd: grep -m 1 'version = ' Cargo.toml | sed 's/version = "\(.*\)"/\1/'
|
||||
silent: false
|
||||
109
.task/workflows/benchmark.yml
Normal file
109
.task/workflows/benchmark.yml
Normal file
@@ -0,0 +1,109 @@
|
||||
version: "3"
|
||||
|
||||
vars:
|
||||
FIXTURES_DIR: "tools/benchmark-harness/fixtures"
|
||||
HARNESS_PATH: "./target/release/benchmark-harness"
|
||||
BENCHMARK_RESULTS_DIR: "benchmark-results"
|
||||
FLAMEGRAPH_DIR: "flamegraphs"
|
||||
|
||||
tasks:
|
||||
run:
|
||||
desc: "Run benchmark harness with profiling support"
|
||||
requires:
|
||||
vars:
|
||||
- FRAMEWORK
|
||||
- MODE
|
||||
vars:
|
||||
ITERATIONS: '{{ .ITERATIONS | default "1" }}'
|
||||
TIMEOUT: '{{ .TIMEOUT | default "900" }}'
|
||||
MAX_CONCURRENT: '{{ if eq .MODE "single-file" }}1{{ else }}4{{ end }}'
|
||||
env:
|
||||
RUST_BACKTRACE: short
|
||||
cmds:
|
||||
- mkdir -p "{{.BENCHMARK_RESULTS_DIR}}/{{.FRAMEWORK}}-{{.MODE}}"
|
||||
- |
|
||||
{{.HARNESS_PATH}} \
|
||||
run \
|
||||
--fixtures "{{.FIXTURES_DIR}}" \
|
||||
--frameworks "{{.FRAMEWORK}}" \
|
||||
--output "{{.BENCHMARK_RESULTS_DIR}}/{{.FRAMEWORK}}-{{.MODE}}" \
|
||||
--iterations "{{.ITERATIONS}}" \
|
||||
--timeout "{{.TIMEOUT}}" \
|
||||
--mode "{{.MODE}}" \
|
||||
--max-concurrent "{{.MAX_CONCURRENT}}"
|
||||
|
||||
profile:
|
||||
desc: "Run pipeline-benchmark with flamegraph profiling. Builds with --profile profiling so Rust symbols are resolved."
|
||||
vars:
|
||||
PIPELINE: '{{ .PIPELINE | default "baseline" }}'
|
||||
DOC_FILTER: '{{ .DOC_FILTER | default "pdf" }}'
|
||||
SHA:
|
||||
sh: git rev-parse --short HEAD
|
||||
env:
|
||||
RUST_BACKTRACE: short
|
||||
cmds:
|
||||
# Build with the `profiling` profile (inherits release, retains debug
|
||||
# info). A plain --release build strips Rust symbols, leaving the
|
||||
# flamegraph full of __mh_execute_header / raw addresses and unable
|
||||
# to surface kreuzberg::* hotspots. See docs/perf/profiling.md.
|
||||
- cargo build --profile profiling -p kreuzberg-cli --features all
|
||||
- cargo build --profile profiling -p benchmark-harness --features profiling
|
||||
- mkdir -p "{{.FLAMEGRAPH_DIR}}/{{.SHA}}"
|
||||
- |
|
||||
target/profiling/benchmark-harness pipeline-benchmark \
|
||||
--fixtures "{{.FIXTURES_DIR}}" \
|
||||
--paths "{{.PIPELINE}}" \
|
||||
--doc "{{.DOC_FILTER}}" \
|
||||
--profile-dir "{{.FLAMEGRAPH_DIR}}/{{.SHA}}"
|
||||
- 'echo "Flamegraph SVGs in {{.FLAMEGRAPH_DIR}}/{{.SHA}}/. Run: python3 tools/perf/extract_top_symbols.py {{.FLAMEGRAPH_DIR}}/{{.SHA}}/{{.PIPELINE}}.svg"'
|
||||
|
||||
compare:
|
||||
desc: "Framework comparison with quality guardrails (baseline vs layout)"
|
||||
cmds:
|
||||
- cargo run -p benchmark-harness -- compare --fixtures "{{.FIXTURES_DIR}}" --guardrails
|
||||
|
||||
pipeline:quick:
|
||||
desc: "Pipeline benchmark — native paths only (P1+P2)"
|
||||
cmds:
|
||||
- cargo run -p benchmark-harness -- pipeline-benchmark --fixtures "{{.FIXTURES_DIR}}" --paths baseline,layout
|
||||
|
||||
pipeline:all:
|
||||
desc: "Pipeline benchmark — all 6 extraction paths"
|
||||
cmds:
|
||||
- cargo run -p benchmark-harness -- pipeline-benchmark --fixtures "{{.FIXTURES_DIR}}"
|
||||
|
||||
survey:
|
||||
desc: "Corpus-wide extraction stats for all PDFs"
|
||||
cmds:
|
||||
- cargo run -p benchmark-harness -- survey --fixtures "{{.FIXTURES_DIR}}" --types pdf
|
||||
|
||||
models:
|
||||
desc: "Layout model A/B comparison (fast vs accurate)"
|
||||
cmds:
|
||||
- cargo run -p benchmark-harness -- model-benchmark --fixtures "{{.FIXTURES_DIR}}"
|
||||
|
||||
generate-gt:
|
||||
desc: "Generate markdown ground truth from PDFs using Gemini"
|
||||
cmds:
|
||||
- uv run --no-sync tools/benchmark-harness/scripts/generate_markdown_gt.py
|
||||
|
||||
download:omnidocbench:
|
||||
desc: "Download OmniDocBench dataset from HuggingFace (~1.3 GB)"
|
||||
cmds:
|
||||
- bash tools/benchmark-harness/scripts/download_omnidocbench.sh
|
||||
status:
|
||||
- test -f tools/benchmark-harness/datasets/omnidocbench/OmniDocBench.json
|
||||
|
||||
import:omnidocbench:
|
||||
desc: "Import OmniDocBench into benchmark fixtures (run download:omnidocbench first)"
|
||||
deps: ["download:omnidocbench"]
|
||||
cmds:
|
||||
- python3 tools/benchmark-harness/scripts/import_omnidocbench.py tools/benchmark-harness/datasets/omnidocbench .
|
||||
status:
|
||||
- test -f tools/benchmark-harness/fixtures/pdf/omnidoc_*.json
|
||||
|
||||
clean-results:
|
||||
desc: "Clean up benchmark results and profiles"
|
||||
cmds:
|
||||
- rm -rf "{{.BENCHMARK_RESULTS_DIR}}"
|
||||
- rm -rf "{{.FLAMEGRAPH_DIR}}"
|
||||
Reference in New Issue
Block a user