This commit is contained in:
Henrik Jess Nielsen
153
.task/tools/demo.yml
Normal file
153
.task/tools/demo.yml
Normal file
@@ -0,0 +1,153 @@
|
||||
version: "3"
|
||||
|
||||
# ============================================================================
|
||||
# Demo development environment
|
||||
#
|
||||
# Generates a patched demo-dev.html (CDN URLs → localhost), then starts two
|
||||
# servers that reproduce the cross-origin scenario the CDN creates in prod:
|
||||
#
|
||||
# Docs server → http://localhost:8001 (serves docs/)
|
||||
# Asset server → http://localhost:9000 (serves crates/kreuzberg-wasm/)
|
||||
#
|
||||
# First-time setup:
|
||||
# task demo:dev:setup # install wasm-pack, wasm-bindgen-cli, WASI SDK
|
||||
#
|
||||
# Usage:
|
||||
# task demo:dev # full build + patch + both servers
|
||||
# SKIP_WASM_BUILD=1 task demo:dev # skip Rust build, TS + patch only
|
||||
#
|
||||
# Open: http://localhost:8001/demo-dev.html
|
||||
# ============================================================================
|
||||
|
||||
tasks:
|
||||
dev:
|
||||
desc: "Build, patch demo for localhost, and start both dev servers"
|
||||
cmds:
|
||||
- task: dev:build
|
||||
- task: dev:patch
|
||||
- task: dev:ready
|
||||
- task: dev:serve
|
||||
|
||||
dev:build:
|
||||
desc: "Build WASM binary for web target (unless SKIP_WASM_BUILD=1)"
|
||||
dir: crates/kreuzberg-wasm
|
||||
cmds:
|
||||
- '[ "${SKIP_WASM_BUILD:-0}" = "1" ] || { pnpm run build:wasm:web && mkdir -p dist/pkg && cp pkg/web/kreuzberg_wasm_bg.wasm dist/pkg/kreuzberg_wasm_bg.wasm; }'
|
||||
|
||||
dev:patch:
|
||||
desc: "Generate docs/demo-dev.html with CDN URLs replaced by localhost:9000"
|
||||
cmds:
|
||||
- node scripts/task/patch-demo-dev.mjs
|
||||
|
||||
dev:ready:
|
||||
desc: "Print the ready banner"
|
||||
cmds:
|
||||
- |
|
||||
printf '\n'
|
||||
printf ' ┌─────────────────────────────────────────────────────┐\n'
|
||||
printf ' │ Demo dev environment ready │\n'
|
||||
printf ' │ │\n'
|
||||
printf ' │ Open → http://localhost:8001/demo-dev.html │\n'
|
||||
printf ' │ │\n'
|
||||
printf ' │ Assets served from http://localhost:9000 │\n'
|
||||
printf ' │ (same cross-origin setup as the live CDN) │\n'
|
||||
printf ' │ │\n'
|
||||
printf ' │ Ctrl+C to stop both servers │\n'
|
||||
printf ' └─────────────────────────────────────────────────────┘\n'
|
||||
printf '\n'
|
||||
|
||||
dev:serve:
|
||||
desc: "Start docs server (port 8001) and WASM asset server (port 9000) in parallel"
|
||||
deps:
|
||||
- task: dev:serve:docs
|
||||
- task: dev:serve:assets
|
||||
|
||||
dev:serve:docs:
|
||||
desc: "Serve docs on http://localhost:8001"
|
||||
cmds:
|
||||
- uv run --no-sync --group doc zensical serve -a localhost:8001
|
||||
|
||||
dev:serve:assets:
|
||||
desc: "Serve WASM assets on http://localhost:9000 (simulates CDN origin)"
|
||||
dir: crates/kreuzberg-wasm
|
||||
cmds:
|
||||
- npx --yes serve . --cors -l tcp://0.0.0.0:9000
|
||||
|
||||
dev:setup:
|
||||
desc: "Install demo:dev prerequisites (wasm-pack, wasm-bindgen-cli, WASI SDK 25)"
|
||||
cmds:
|
||||
- task: dev:setup:wasm-pack
|
||||
- task: dev:setup:wasm-bindgen-cli
|
||||
- task: dev:setup:wasi-sdk
|
||||
|
||||
dev:setup:wasm-pack:
|
||||
desc: "Install wasm-pack 0.13.1 (matches CI pinned version)"
|
||||
vars:
|
||||
WASM_PACK_VERSION: "0.13.1"
|
||||
cmds:
|
||||
- |
|
||||
if command -v wasm-pack &>/dev/null && [ "$(wasm-pack --version 2>/dev/null | awk '{print $2}')" = "{{.WASM_PACK_VERSION}}" ]; then
|
||||
echo "wasm-pack {{.WASM_PACK_VERSION}} already installed"
|
||||
exit 0
|
||||
fi
|
||||
cargo install wasm-pack --version {{.WASM_PACK_VERSION}} --locked
|
||||
|
||||
dev:setup:wasm-bindgen-cli:
|
||||
desc: "Install wasm-bindgen-cli at the version pinned in Cargo.lock"
|
||||
vars:
|
||||
WASM_BINDGEN_VERSION:
|
||||
sh: grep -A 3 '^name = "wasm-bindgen"$' Cargo.lock | grep '^version' | head -1 | awk '{print $3}' | tr -d '"'
|
||||
cmds:
|
||||
- |
|
||||
if command -v wasm-bindgen &>/dev/null && [ "$(wasm-bindgen --version 2>/dev/null | awk '{print $2}')" = "{{.WASM_BINDGEN_VERSION}}" ]; then
|
||||
echo "wasm-bindgen-cli {{.WASM_BINDGEN_VERSION}} already installed"
|
||||
exit 0
|
||||
fi
|
||||
cargo install wasm-bindgen-cli --version {{.WASM_BINDGEN_VERSION}} --locked
|
||||
|
||||
dev:setup:wasi-sdk:
|
||||
desc: "Install WASI SDK 25 to ~/wasi-sdk-25 (skips if already present)"
|
||||
vars:
|
||||
WASI_SDK_VERSION: "25"
|
||||
cmds:
|
||||
- |
|
||||
WASI_SDK_DIR="${WASI_SDK_PATH:-$HOME/wasi-sdk-{{.WASI_SDK_VERSION}}}"
|
||||
if [ -d "$WASI_SDK_DIR/share/wasi-sysroot" ]; then
|
||||
echo "WASI SDK {{.WASI_SDK_VERSION}} already installed at $WASI_SDK_DIR"
|
||||
exit 0
|
||||
fi
|
||||
OS=$(uname -s | tr '[:upper:]' '[:lower:]')
|
||||
ARCH=$(uname -m)
|
||||
case "$OS-$ARCH" in
|
||||
linux-x86_64) PKG="wasi-sdk-{{.WASI_SDK_VERSION}}.0-x86_64-linux.tar.gz" ;;
|
||||
linux-aarch64) PKG="wasi-sdk-{{.WASI_SDK_VERSION}}.0-arm64-linux.tar.gz" ;;
|
||||
darwin-x86_64) PKG="wasi-sdk-{{.WASI_SDK_VERSION}}.0-x86_64-macos.tar.gz" ;;
|
||||
darwin-arm64) PKG="wasi-sdk-{{.WASI_SDK_VERSION}}.0-arm64-macos.tar.gz" ;;
|
||||
*)
|
||||
echo "Unsupported platform: $OS-$ARCH"
|
||||
echo "Download manually: https://github.com/WebAssembly/wasi-sdk/releases/tag/wasi-sdk-{{.WASI_SDK_VERSION}}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
BASE_URL="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-{{.WASI_SDK_VERSION}}"
|
||||
TMPFILE=$(mktemp /tmp/wasi-sdk-XXXXXX.tar.gz)
|
||||
echo "Downloading WASI SDK {{.WASI_SDK_VERSION}} ..."
|
||||
curl -fL "${BASE_URL}/${PKG}" -o "$TMPFILE"
|
||||
curl -fL "${BASE_URL}/${PKG}.sha256" -o "${TMPFILE}.sha256"
|
||||
echo "Verifying SHA256 ..."
|
||||
EXPECTED_HASH=$(awk '{print $1}' "${TMPFILE}.sha256")
|
||||
if command -v sha256sum &>/dev/null; then
|
||||
echo "${EXPECTED_HASH} ${TMPFILE}" | sha256sum -c -
|
||||
elif command -v shasum &>/dev/null; then
|
||||
echo "${EXPECTED_HASH} ${TMPFILE}" | shasum -a 256 -c -
|
||||
else
|
||||
echo "Warning: no SHA256 tool found, skipping checksum verification"
|
||||
fi
|
||||
echo "Extracting to $WASI_SDK_DIR ..."
|
||||
mkdir -p "$WASI_SDK_DIR"
|
||||
tar -xzf "$TMPFILE" -C "$WASI_SDK_DIR" --strip-components=1
|
||||
rm -f "$TMPFILE" "${TMPFILE}.sha256"
|
||||
echo ""
|
||||
echo "WASI SDK {{.WASI_SDK_VERSION}} installed at $WASI_SDK_DIR"
|
||||
echo "Add to your shell profile:"
|
||||
echo " export WASI_SDK_PATH=$WASI_SDK_DIR"
|
||||
79
.task/tools/docs.yml
Normal file
79
.task/tools/docs.yml
Normal file
@@ -0,0 +1,79 @@
|
||||
version: "3"
|
||||
|
||||
includes:
|
||||
_cfg:
|
||||
taskfile: ../config/vars.yml
|
||||
internal: true
|
||||
|
||||
vars:
|
||||
ALEF_SNIPPETS_TAG: v0.16.69
|
||||
|
||||
tasks:
|
||||
build:
|
||||
desc: Build documentation
|
||||
cmds:
|
||||
- uv run --no-sync --group doc zensical build --clean
|
||||
|
||||
build:strict:
|
||||
desc: Build documentation in strict mode (CI-equivalent)
|
||||
cmds:
|
||||
- uv run --no-sync --group doc zensical build --strict --clean
|
||||
|
||||
serve:
|
||||
desc: Serve documentation locally with live reload
|
||||
cmds:
|
||||
- uv run --no-sync --group doc zensical serve
|
||||
|
||||
lint:links:
|
||||
desc: Check links with lychee
|
||||
cmds:
|
||||
# docs/overrides/main.html contains Jinja template variables (e.g.
|
||||
# `{{ page.edit_url }}`) that lychee interprets as broken local file
|
||||
# links — skip that path; the rendered site is link-checked separately.
|
||||
- lychee --offline --root-dir "$PWD/docs" --exclude-path "docs/overrides" docs/ README.md
|
||||
|
||||
lint:prose:
|
||||
desc: Lint prose with textlint (via prek hook)
|
||||
cmds:
|
||||
- prek run textlint --all-files
|
||||
|
||||
snippets:install:
|
||||
desc: Install alef CLI (which now includes the snippets subcommand) if missing
|
||||
cmds:
|
||||
- cmd: |
|
||||
if ! command -v alef >/dev/null 2>&1; then
|
||||
cargo install --locked --git https://github.com/kreuzberg-dev/alef --tag {{.ALEF_SNIPPETS_TAG}} alef-cli
|
||||
fi
|
||||
ignore_error: false
|
||||
|
||||
snippets:validate:
|
||||
desc: Validate all docs/snippets via `alef snippets` (syntax level)
|
||||
deps: ["snippets:install"]
|
||||
cmds:
|
||||
- alef snippets validate --snippets docs/snippets --level syntax --timeout 120 -j 4
|
||||
|
||||
snippets:validate:lang:
|
||||
desc: "Validate snippets for one language (set LANG=dart|kotlin|swift|...)"
|
||||
deps: ["snippets:install"]
|
||||
cmds:
|
||||
- alef snippets validate --snippets docs/snippets --level syntax --languages {{.LANG}} --timeout 120 -j 4
|
||||
requires:
|
||||
vars: [LANG]
|
||||
|
||||
snippets:list:
|
||||
desc: List all snippets discovered under docs/snippets
|
||||
deps: ["snippets:install"]
|
||||
cmds:
|
||||
- alef snippets list --snippets docs/snippets
|
||||
|
||||
snippets:audit:
|
||||
desc: Audit snippet metadata and coverage
|
||||
deps: ["snippets:install"]
|
||||
cmds:
|
||||
- alef snippets audit --snippets docs/snippets
|
||||
|
||||
snippets:gaps:
|
||||
desc: Report missing snippet variants across languages
|
||||
deps: ["snippets:install"]
|
||||
cmds:
|
||||
- alef snippets gaps --snippets docs/snippets
|
||||
30
.task/tools/fixtures.yml
Normal file
30
.task/tools/fixtures.yml
Normal file
@@ -0,0 +1,30 @@
|
||||
version: "3"
|
||||
|
||||
# ============================================================================
|
||||
# Test fixture generation
|
||||
#
|
||||
# Generates DOCX / ODT / XLSX / PPTX / PDF documents with track-changes,
|
||||
# revisions, comments, incremental updates, paired diff inputs, and
|
||||
# security edge cases — each paired with a *.gt.json ground-truth sidecar.
|
||||
#
|
||||
# Source: tools/generate_test_fixtures/
|
||||
# Output: test_documents/generated/<format>/
|
||||
#
|
||||
# Usage:
|
||||
# task fixtures:generate # produce every fixture
|
||||
# task fixtures:generate -- docx # produce a single category
|
||||
# task fixtures:test # run the toolkit's smoke test
|
||||
# ============================================================================
|
||||
|
||||
tasks:
|
||||
generate:
|
||||
desc: "Generate all (or selected) test fixtures + ground-truth sidecars"
|
||||
dir: tools/generate_test_fixtures
|
||||
cmds:
|
||||
- uv run python -m generate_test_fixtures {{.CLI_ARGS | default "all"}}
|
||||
|
||||
test:
|
||||
desc: "Run the fixture generator's smoke test suite"
|
||||
dir: tools/generate_test_fixtures
|
||||
cmds:
|
||||
- uv run pytest -q
|
||||
55
.task/tools/general.yml
Normal file
55
.task/tools/general.yml
Normal file
@@ -0,0 +1,55 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
includes:
|
||||
platforms: ../config/platforms.yml
|
||||
|
||||
vars:
|
||||
SCRIPTS_DIR: "{{.TASKFILE_DIR}}/../../scripts"
|
||||
|
||||
tasks:
|
||||
pre-commit:install:
|
||||
desc: "Install prek pre-commit hooks for commit and commit-msg"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: prek install
|
||||
ignore_error: false
|
||||
- cmd: prek install --hook-type commit-msg
|
||||
ignore_error: false
|
||||
|
||||
pre-commit:run:
|
||||
desc: "Run prek pre-commit hooks on all files"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: prek run --all-files
|
||||
ignore_error: false
|
||||
|
||||
pre-commit:uninstall:
|
||||
desc: "Uninstall prek hooks"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: prek uninstall
|
||||
ignore_error: true
|
||||
- cmd: prek uninstall --hook-type commit-msg
|
||||
ignore_error: true
|
||||
|
||||
validate:config:
|
||||
desc: "Validate YAML task configuration files"
|
||||
silent: false
|
||||
cmds:
|
||||
- cmd: |
|
||||
for file in {{.TASKFILE_DIR}}/**/*.yml; do
|
||||
echo "Validating $file..."
|
||||
if ! command -v yamllint &> /dev/null; then
|
||||
echo "yamllint not found, skipping validation"
|
||||
break
|
||||
fi
|
||||
yamllint "$file" || exit 1
|
||||
done
|
||||
ignore_error: false
|
||||
|
||||
validate:all:
|
||||
desc: "Validate all project configurations"
|
||||
silent: false
|
||||
cmds:
|
||||
- task: validate:config
|
||||
127
.task/tools/test-apps.yml
Normal file
127
.task/tools/test-apps.yml
Normal file
@@ -0,0 +1,127 @@
|
||||
version: "3"
|
||||
|
||||
tasks:
|
||||
smoke:
|
||||
desc: Run all test_app smoke tests
|
||||
cmds:
|
||||
- task: smoke:python
|
||||
- task: smoke:node
|
||||
- task: smoke:ruby
|
||||
- task: smoke:go
|
||||
- task: smoke:java
|
||||
- task: smoke:elixir
|
||||
- task: smoke:csharp
|
||||
- task: smoke:php
|
||||
- task: smoke:r
|
||||
- task: smoke:rust
|
||||
- task: smoke:wasm
|
||||
|
||||
smoke:python:
|
||||
desc: Run Python test_app smoke test
|
||||
dir: test_apps/python
|
||||
cmds:
|
||||
- uv sync
|
||||
- uv run --no-sync pytest smoke_test.py -v
|
||||
|
||||
smoke:node:
|
||||
desc: Run Node.js test_app smoke test
|
||||
dir: test_apps/node
|
||||
cmds:
|
||||
- pnpm install
|
||||
- pnpm run test:smoke
|
||||
|
||||
smoke:ruby:
|
||||
desc: Run Ruby test_app smoke test
|
||||
dir: test_apps/ruby
|
||||
cmds:
|
||||
- bundle install
|
||||
- bundle exec rspec smoke_test.rb
|
||||
|
||||
smoke:go:
|
||||
desc: Run Go test_app smoke test
|
||||
dir: test_apps/go
|
||||
cmds:
|
||||
- go test -v -run TestBasicFixtures
|
||||
- go test -v -run TestParseValidation
|
||||
|
||||
smoke:java:
|
||||
desc: Run Java test_app smoke test
|
||||
dir: test_apps/java
|
||||
cmds:
|
||||
- mvn test -Dtest=SmokeTest
|
||||
|
||||
smoke:elixir:
|
||||
desc: Run Elixir test_app smoke test
|
||||
dir: test_apps/elixir
|
||||
cmds:
|
||||
- mix deps.get
|
||||
- mix test
|
||||
|
||||
smoke:csharp:
|
||||
desc: Run C# test_app smoke test
|
||||
dir: test_apps/csharp
|
||||
cmds:
|
||||
- dotnet restore
|
||||
- dotnet test
|
||||
|
||||
smoke:php:
|
||||
desc: Run PHP test_app smoke test
|
||||
dir: test_apps/php
|
||||
cmds:
|
||||
- composer install
|
||||
- vendor/bin/phpunit smoke_test.php
|
||||
|
||||
smoke:r:
|
||||
desc: Run R test_app smoke test
|
||||
dir: test_apps/r
|
||||
cmds:
|
||||
- Rscript run_tests.R
|
||||
|
||||
smoke:rust:
|
||||
desc: Run Rust test_app smoke test
|
||||
dir: test_apps/rust
|
||||
cmds:
|
||||
- cargo run
|
||||
|
||||
smoke:wasm:
|
||||
desc: Run WASM test_app smoke test
|
||||
dir: test_apps/wasm
|
||||
cmds:
|
||||
- pnpm install
|
||||
- pnpm run test:smoke
|
||||
|
||||
comprehensive:
|
||||
desc: Run all test_app comprehensive tests
|
||||
cmds:
|
||||
- task: comprehensive:python
|
||||
- task: comprehensive:node
|
||||
- task: comprehensive:ruby
|
||||
- task: comprehensive:go
|
||||
|
||||
comprehensive:python:
|
||||
desc: Run Python test_app comprehensive tests
|
||||
dir: test_apps/python
|
||||
cmds:
|
||||
- uv sync
|
||||
- uv run --no-sync pytest comprehensive_test.py -v
|
||||
|
||||
comprehensive:node:
|
||||
desc: Run Node.js test_app comprehensive tests
|
||||
dir: test_apps/node
|
||||
cmds:
|
||||
- pnpm install
|
||||
- pnpm run test:comprehensive
|
||||
|
||||
comprehensive:ruby:
|
||||
desc: Run Ruby test_app comprehensive tests
|
||||
dir: test_apps/ruby
|
||||
cmds:
|
||||
- bundle install
|
||||
- bundle exec rspec comprehensive_test.rb
|
||||
|
||||
comprehensive:go:
|
||||
desc: Run Go test_app comprehensive tests
|
||||
dir: test_apps/go
|
||||
cmds:
|
||||
- go test -v -run TestProcessFixtures
|
||||
- go test -v -run TestChunkingFixtures
|
||||
57
.task/tools/version-sync.yml
Normal file
57
.task/tools/version-sync.yml
Normal file
@@ -0,0 +1,57 @@
|
||||
version: "3"
|
||||
internal: true
|
||||
|
||||
includes:
|
||||
platforms: ../config/platforms.yml
|
||||
|
||||
vars:
|
||||
ALEF: "alef"
|
||||
|
||||
tasks:
|
||||
sync:
|
||||
desc: "Synchronize version across all package manifests and regenerate everything"
|
||||
cmds:
|
||||
- "{{.ALEF}} sync-versions"
|
||||
- "{{.ALEF}} readme"
|
||||
- "{{.ALEF}} docs"
|
||||
- "{{.ALEF}} generate --clean"
|
||||
- "{{.ALEF}} stubs"
|
||||
- "{{.ALEF}} e2e generate"
|
||||
|
||||
check:
|
||||
desc: "Check if versions are synchronized (dry-run)"
|
||||
cmds:
|
||||
- cmd: grep -m 1 'version = ' Cargo.toml | sed 's/version = "\(.*\)"/\1/'
|
||||
silent: false
|
||||
|
||||
bump:major:
|
||||
desc: "Bump major version (X.0.0) and sync"
|
||||
cmds:
|
||||
- "{{.ALEF}} sync-versions --bump major"
|
||||
- task: sync
|
||||
|
||||
bump:minor:
|
||||
desc: "Bump minor version (0.X.0) and sync"
|
||||
cmds:
|
||||
- "{{.ALEF}} sync-versions --bump minor"
|
||||
- task: sync
|
||||
|
||||
bump:patch:
|
||||
desc: "Bump patch version (0.0.X) and sync"
|
||||
cmds:
|
||||
- "{{.ALEF}} sync-versions --bump patch"
|
||||
- task: sync
|
||||
|
||||
set:
|
||||
desc: "Set explicit version and sync (usage: task version:set -- 4.10.0-rc.1)"
|
||||
requires:
|
||||
vars: [CLI_ARGS]
|
||||
cmds:
|
||||
- "{{.ALEF}} sync-versions --set {{.CLI_ARGS}}"
|
||||
- task: sync
|
||||
|
||||
show:
|
||||
desc: "Show current version from Cargo.toml"
|
||||
cmds:
|
||||
- cmd: grep -m 1 'version = ' Cargo.toml | sed 's/version = "\(.*\)"/\1/'
|
||||
silent: false
|
||||
Reference in New Issue
Block a user