Nomad changes

2026-06-01 23:40:55 +02:00
parent 72b1a0a6ed
commit b4c07d3693
5723 changed files with 1130655 additions and 0 deletions
--- a/scripts/ci/actions/setup-onnx-runtime/linux.sh
+++ b/scripts/ci/actions/setup-onnx-runtime/linux.sh
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ort_version="${1:?ort-version required}"
+dest_dir="${2:-crates/kreuzberg-node}"
+arch_id="${3:-}"
+strategy="${4:-system}"
+
+extract_dir="$RUNNER_TEMP/onnxruntime"
+
+if [ -z "$arch_id" ]; then
+  case "$(uname -m)" in
+  x86_64 | amd64) arch_id="x64" ;;
+  arm64 | aarch64) arch_id="arm64" ;;
+  *)
+    echo "Unsupported Linux architecture: $(uname -m)" >&2
+    exit 1
+    ;;
+  esac
+fi
+
+case "$arch_id" in
+x64)
+  ort_dir_name="onnxruntime-linux-x64-${ort_version}"
+  archive="onnxruntime-linux-x64-${ort_version}.tgz"
+  ;;
+arm64)
+  ort_dir_name="onnxruntime-linux-aarch64-${ort_version}"
+  archive="onnxruntime-linux-aarch64-${ort_version}.tgz"
+  ;;
+*)
+  echo "Unsupported Linux arch-id: $arch_id" >&2
+  exit 1
+  ;;
+esac
+
+if [ ! -d "$extract_dir/$ort_dir_name" ]; then
+  echo "Cache miss: Downloading ONNX Runtime ${ort_version}"
+  curl -fsSL --retry 5 --retry-delay 5 --retry-all-errors -o "$RUNNER_TEMP/$archive" "https://github.com/microsoft/onnxruntime/releases/download/v${ort_version}/$archive"
+  mkdir -p "$extract_dir"
+  tar -xzf "$RUNNER_TEMP/$archive" -C "$extract_dir"
+else
+  echo "Cache hit: Using cached ONNX Runtime ${ort_version}"
+fi
+
+ort_root="$extract_dir/$ort_dir_name"
+
+if [ ! -d "$ort_root/lib" ]; then
+  echo "ERROR: ONNX Runtime lib directory missing at $ort_root/lib" >&2
+  echo "Available directories:" >&2
+  ls -la "$extract_dir" >&2 || true
+  exit 1
+fi
+
+if ! ls "$ort_root/lib"/*.so* 1>/dev/null 2>&1; then
+  echo "ERROR: No ONNX Runtime libraries found in $ort_root/lib" >&2
+  echo "Directory contents:" >&2
+  ls -la "$ort_root/lib" >&2 || true
+  exit 1
+fi
+
+dest="$GITHUB_WORKSPACE/$dest_dir"
+mkdir -p "$dest"
+cp -f "$ort_root/lib/"*.so* "$dest/"
+
+if [ -n "${RUSTFLAGS:-}" ]; then
+  rustflags="$RUSTFLAGS -L $ort_root/lib"
+else
+  rustflags="-L $ort_root/lib"
+fi
+
+if [ "$strategy" = "bundled" ]; then
+  echo "Using bundled ORT strategy — letting ort-sys download-binaries handle static linking"
+  {
+    echo "LD_LIBRARY_PATH=$ort_root/lib:$dest:${LD_LIBRARY_PATH:-}"
+    echo "LIBRARY_PATH=$ort_root/lib:$dest:${LIBRARY_PATH:-}"
+  } >>"$GITHUB_ENV"
+else
+  {
+    ort_lib=$(find "$ort_root/lib" -name "libonnxruntime*.so*" -print -quit)
+    echo "ORT_LIB_LOCATION=$ort_root/lib"
+    echo "ORT_PREFER_DYNAMIC_LINK=1"
+    echo "ORT_SKIP_DOWNLOAD=1"
+    echo "ORT_STRATEGY=system"
+    echo "ORT_DYLIB_PATH=$ort_root/lib/${ort_lib##*/}"
+    echo "LD_LIBRARY_PATH=$ort_root/lib:$dest:${LD_LIBRARY_PATH:-}"
+    echo "LIBRARY_PATH=$ort_root/lib:$dest:${LIBRARY_PATH:-}"
+    echo "RUSTFLAGS=$rustflags"
+  } >>"$GITHUB_ENV"
+fi
--- a/scripts/ci/actions/setup-onnx-runtime/macos.sh
+++ b/scripts/ci/actions/setup-onnx-runtime/macos.sh
@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ort_version="${1:?ort-version required}"
+dest_dir="${2:-crates/kreuzberg-node}"
+arch_id="${3:-}"
+strategy="${4:-system}"
+
+extract_dir="$RUNNER_TEMP/onnxruntime"
+
+if [ -z "$arch_id" ]; then
+  arch="$(uname -m)"
+  if [ "$arch" = "arm64" ]; then
+    arch_id="arm64"
+  else
+    arch_id="x64"
+  fi
+fi
+
+case "$arch_id" in
+arm64) ort_arch="arm64" ;;
+x64) ort_arch="x86_64" ;;
+*)
+  echo "Unsupported macOS arch-id: $arch_id" >&2
+  exit 1
+  ;;
+esac
+echo "Using macOS ONNX Runtime arch: $ort_arch"
+
+if [ ! -d "$extract_dir/onnxruntime-osx-${ort_arch}-${ort_version}" ]; then
+  echo "Cache miss: Downloading ONNX Runtime ${ort_version} for macOS ${ort_arch}"
+  archive="onnxruntime-osx-${ort_arch}-${ort_version}.tgz"
+  curl -fsSL --retry 5 --retry-delay 5 --retry-all-errors -o "$RUNNER_TEMP/$archive" "https://github.com/microsoft/onnxruntime/releases/download/v${ort_version}/$archive"
+  mkdir -p "$extract_dir"
+  tar -xzf "$RUNNER_TEMP/$archive" -C "$extract_dir"
+else
+  echo "Cache hit: Using cached ONNX Runtime ${ort_version}"
+fi
+
+ort_root="$extract_dir/onnxruntime-osx-${ort_arch}-${ort_version}"
+
+if [ ! -d "$ort_root/lib" ]; then
+  echo "ERROR: ONNX Runtime lib directory missing at $ort_root/lib" >&2
+  echo "Available directories:" >&2
+  ls -la "$extract_dir" >&2 || true
+  exit 1
+fi
+
+if ! ls "$ort_root/lib"/libonnxruntime*.dylib 1>/dev/null 2>&1; then
+  echo "ERROR: No ONNX Runtime libraries found in $ort_root/lib" >&2
+  echo "Directory contents:" >&2
+  ls -la "$ort_root/lib" >&2 || true
+  exit 1
+fi
+
+dest="$GITHUB_WORKSPACE/$dest_dir"
+mkdir -p "$dest"
+cp -f "$ort_root/lib/"libonnxruntime*.dylib "$dest/"
+
+if [ -n "${RUSTFLAGS:-}" ]; then
+  rustflags="$RUSTFLAGS -L $ort_root/lib"
+else
+  rustflags="-L $ort_root/lib"
+fi
+
+if [ "$strategy" = "bundled" ]; then
+  echo "Using bundled ORT strategy — letting ort-sys download-binaries handle static linking"
+  {
+    echo "DYLD_LIBRARY_PATH=$ort_root/lib:$dest:${DYLD_LIBRARY_PATH:-}"
+    echo "DYLD_FALLBACK_LIBRARY_PATH=$ort_root/lib:$dest:${DYLD_FALLBACK_LIBRARY_PATH:-}"
+    echo "LIBRARY_PATH=$ort_root/lib:$dest:${LIBRARY_PATH:-}"
+  } >>"$GITHUB_ENV"
+else
+  {
+    ort_lib=$(find "$ort_root/lib" -name "libonnxruntime*.dylib" -print -quit)
+    echo "ORT_LIB_LOCATION=$ort_root/lib"
+    echo "ORT_PREFER_DYNAMIC_LINK=1"
+    echo "ORT_SKIP_DOWNLOAD=1"
+    echo "ORT_STRATEGY=system"
+    echo "ORT_DYLIB_PATH=$ort_root/lib/${ort_lib##*/}"
+    echo "DYLD_LIBRARY_PATH=$ort_root/lib:$dest:${DYLD_LIBRARY_PATH:-}"
+    echo "DYLD_FALLBACK_LIBRARY_PATH=$ort_root/lib:$dest:${DYLD_FALLBACK_LIBRARY_PATH:-}"
+    echo "LIBRARY_PATH=$ort_root/lib:$dest:${LIBRARY_PATH:-}"
+    echo "RUSTFLAGS=$rustflags"
+  } >>"$GITHUB_ENV"
+fi
--- a/scripts/ci/actions/setup-onnx-runtime/windows.ps1
+++ b/scripts/ci/actions/setup-onnx-runtime/windows.ps1
@@ -0,0 +1,100 @@
+$OrtVersion = $args[0]
+if ([string]::IsNullOrWhiteSpace($OrtVersion)) { throw "Usage: windows.ps1 <ortVersion> [destDir] [archId] [strategy]" }
+
+$DestDir = if ($args.Count -ge 2 -and -not [string]::IsNullOrWhiteSpace($args[1])) { $args[1] } else { "crates/kreuzberg-node" }
+$ArchId = if ($args.Count -ge 3) { $args[2] } else { "" }
+$Strategy = if ($args.Count -ge 4 -and -not [string]::IsNullOrWhiteSpace($args[3])) { $args[3] } else { "system" }
+
+$ExtractRoot = Join-Path $env:TEMP "onnxruntime"
+if ([string]::IsNullOrWhiteSpace($ArchId)) {
+  $ArchId = $env:RUNNER_ARCH
+}
+$ArchId = $ArchId.ToLowerInvariant()
+if ($ArchId -eq "arm64") { $ArchId = "arm64" } else { $ArchId = "x64" }
+
+$OrtRoot = Join-Path $ExtractRoot "onnxruntime-win-$ArchId-$OrtVersion"
+$OrtBin = Join-Path $OrtRoot 'bin'
+$OrtLib = Join-Path $OrtRoot 'lib'
+
+if (-Not (Test-Path $OrtRoot)) {
+  Write-Host "Cache miss: Downloading ONNX Runtime $OrtVersion"
+  $Archive = "onnxruntime-win-$ArchId-$OrtVersion.zip"
+  $DownloadPath = Join-Path $env:TEMP $Archive
+  Invoke-WebRequest -Uri "https://github.com/microsoft/onnxruntime/releases/download/v$OrtVersion/$Archive" -OutFile $DownloadPath -UseBasicParsing -MaximumRetryCount 5 -RetryIntervalSec 5
+  New-Item -ItemType Directory -Path $ExtractRoot -Force | Out-Null
+  Expand-Archive -Path $DownloadPath -DestinationPath $ExtractRoot -Force
+} else {
+  Write-Host "Cache hit: Using cached ONNX Runtime $OrtVersion"
+}
+
+if (!(Test-Path $OrtLib)) {
+  Write-Error "ERROR: ONNX Runtime lib directory missing at $OrtLib"
+  Get-ChildItem -Path $ExtractRoot -Recurse | Write-Host
+  exit 1
+}
+
+$LibFiles = @(Get-ChildItem -Path $OrtLib -Filter "*.lib" -ErrorAction SilentlyContinue)
+if ($LibFiles.Count -eq 0) {
+  Write-Error "ERROR: No ONNX Runtime library files found in $OrtLib"
+  Get-ChildItem -Path $OrtLib | Write-Host
+  exit 1
+}
+
+$DllDirs = @()
+foreach ($Candidate in @($OrtLib, $OrtBin)) {
+  if (Test-Path $Candidate) {
+    $CandidateDlls = @(Get-ChildItem -Path $Candidate -Filter "*.dll" -File -ErrorAction SilentlyContinue)
+    if ($CandidateDlls.Count -gt 0) {
+      $DllDirs += $Candidate
+    }
+  }
+}
+if ($DllDirs.Count -eq 0) {
+  $OrtDll = Get-ChildItem -Path $OrtRoot -Recurse -Filter "onnxruntime.dll" -File -ErrorAction SilentlyContinue | Select-Object -First 1
+  if ($OrtDll) { $DllDirs += $OrtDll.DirectoryName }
+}
+if ($DllDirs.Count -eq 0) {
+  $AnyDll = Get-ChildItem -Path $OrtRoot -Recurse -Filter "*.dll" -File -ErrorAction SilentlyContinue | Select-Object -First 1
+  if ($AnyDll) { $DllDirs += $AnyDll.DirectoryName }
+}
+$DllDirs = $DllDirs | Select-Object -Unique
+if ($DllDirs.Count -eq 0) {
+  Write-Error "ERROR: No ONNX Runtime runtime DLLs found under $OrtRoot"
+  Get-ChildItem -Path $OrtRoot -Recurse | Write-Host
+  exit 1
+}
+
+$Dest = Join-Path $env:GITHUB_WORKSPACE $DestDir
+New-Item -ItemType Directory -Path $Dest -Force | Out-Null
+Copy-Item -Path (Join-Path $OrtLib '*') -Destination $Dest -Force
+foreach ($Dir in $DllDirs) {
+  Copy-Item -Path (Join-Path $Dir '*.dll') -Destination $Dest -Force
+}
+
+$RustFlags = if ($env:RUSTFLAGS) { "$env:RUSTFLAGS -L $OrtLib" } else { "-L $OrtLib" }
+
+if ($Strategy -eq "bundled") {
+  # ort-sys has no prebuilt static binaries for x86_64-pc-windows-gnu (MSYS2/MinGW).
+  # Use the pre-downloaded Microsoft ORT with dynamic linking for Windows GNU targets.
+  Write-Host "Using bundled ORT strategy (Windows) - dynamic linking against pre-downloaded ORT (no static binaries for windows-gnu)"
+  @(
+    "ORT_LIB_LOCATION=$OrtLib"
+    "ORT_PREFER_DYNAMIC_LINK=1"
+    "RUSTFLAGS=$RustFlags"
+    "LIB=$OrtLib;$env:LIB"
+    "LIBRARY_PATH=$OrtLib;$env:LIBRARY_PATH"
+    "PATH=$Dest;$env:PATH"
+  ) | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+} else {
+  @(
+    "ORT_LIB_LOCATION=$OrtLib"
+    "ORT_PREFER_DYNAMIC_LINK=1"
+    "ORT_SKIP_DOWNLOAD=1"
+    "ORT_STRATEGY=system"
+    "ORT_DYLIB_PATH=$Dest\onnxruntime.dll"
+    "RUSTFLAGS=$RustFlags"
+    "LIB=$OrtLib;$env:LIB"
+    "LIBRARY_PATH=$OrtLib;$env:LIBRARY_PATH"
+    "PATH=$Dest;$env:PATH"
+  ) | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+}