Nomad changes

2026-06-01 23:40:55 +02:00
parent 72b1a0a6ed
commit b4c07d3693
5723 changed files with 1130655 additions and 0 deletions
--- a/.github/workflows/publish-docker.yaml
+++ b/.github/workflows/publish-docker.yaml
@@ -0,0 +1,262 @@
+name: Publish Docker Images
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Release tag to build (e.g., v4.3.6)"
+        required: true
+        type: string
+      dry_run:
+        description: "Prepare artifacts without publishing"
+        required: false
+        type: boolean
+        default: false
+      ref:
+        description: "Git ref (branch, tag, or commit) to build; defaults to the tag"
+        required: false
+        type: string
+      force_republish:
+        description: "Force re-publish even if artifacts already exist"
+        required: false
+        type: boolean
+        default: false
+  release:
+    types: [published]
+  repository_dispatch:
+    types: [publish-docker]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ (github.event_name == 'workflow_dispatch' && (github.event.inputs.ref || github.event.inputs.tag)) || github.ref || github.run_id }}
+  cancel-in-progress: false
+
+env:
+  CARGO_TERM_COLOR: always
+  ORT_VERSION: "1.24.2"
+  MACOSX_DEPLOYMENT_TARGET: "14.0"
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+
+permissions:
+  contents: read
+
+jobs:
+  prepare:
+    name: Prepare metadata
+    if: ${{ github.event_name != 'release' || !github.event.release.prerelease }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+      tag: ${{ steps.meta.outputs.tag }}
+      version: ${{ steps.meta.outputs.version }}
+      ref: ${{ steps.meta.outputs.ref }}
+      dry_run: ${{ steps.meta.outputs.dry_run }}
+      force_republish: ${{ steps.meta.outputs.force_republish }}
+      checkout_ref: ${{ steps.meta.outputs.checkout_ref }}
+      target_sha: ${{ steps.meta.outputs.target_sha }}
+      is_tag: ${{ steps.meta.outputs.is_tag }}
+      release_docker: ${{ steps.meta.outputs.release_docker }}
+    steps:
+      - name: Checkout code (default)
+        uses: actions/checkout@v6
+
+      - name: Resolve release metadata
+        id: meta
+        uses: kreuzberg-dev/actions/prepare-release-metadata@v1
+        with:
+          tag: ${{ inputs.tag }}
+          ref: ${{ inputs.ref }}
+          targets: docker
+          dry-run: ${{ inputs.dry_run }}
+          force-republish: ${{ inputs.force_republish }}
+
+      - name: Re-checkout at target ref
+        if: ${{ steps.meta.outputs.checkout_ref != '' }}
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ steps.meta.outputs.checkout_ref }}
+          fetch-depth: 0
+          submodules: recursive
+
+      - name: Show metadata
+        env:
+          META_TAG: ${{ steps.meta.outputs.tag }}
+          META_VERSION: ${{ steps.meta.outputs.version }}
+          META_REF: ${{ steps.meta.outputs.ref }}
+          META_DRY_RUN: ${{ steps.meta.outputs.dry_run }}
+          META_FORCE_REPUBLISH: ${{ steps.meta.outputs.force_republish }}
+          META_CHECKOUT_REF: ${{ steps.meta.outputs.checkout_ref }}
+          META_TARGET_SHA: ${{ steps.meta.outputs.target_sha }}
+          META_IS_TAG: ${{ steps.meta.outputs.is_tag }}
+          META_RELEASE_DOCKER: ${{ steps.meta.outputs.release_docker }}
+        run: |
+          {
+            echo "## Release Metadata"
+            echo "- **Tag**: \`$META_TAG\`"
+            echo "- **Version**: \`$META_VERSION\`"
+            echo "- **Ref**: \`$META_REF\`"
+            echo "- **Dry Run**: \`$META_DRY_RUN\`"
+            echo "- **Force Republish**: \`$META_FORCE_REPUBLISH\`"
+            echo "- **Checkout Ref**: \`$META_CHECKOUT_REF\`"
+            echo "- **Target SHA**: \`$META_TARGET_SHA\`"
+            echo "- **Is Tag**: \`$META_IS_TAG\`"
+            echo "- **Release Docker**: \`$META_RELEASE_DOCKER\`"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+  check-docker:
+    name: Check if Docker image tag exists
+    needs: prepare
+    if: ${{ needs.prepare.outputs.release_docker == 'true' }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: read
+    outputs:
+      core_exists: ${{ steps.core.outputs.exists }}
+      full_exists: ${{ steps.full.outputs.exists }}
+      cli_exists: ${{ steps.cli.outputs.exists }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ needs.prepare.outputs.tag }}
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check core image tag
+        id: core
+        env:
+          DOCKER_TAG: ghcr.io/kreuzberg-dev/kreuzberg:${{ needs.prepare.outputs.version }}-core
+          SUMMARY_LABEL: core
+        run: scripts/publish/check-docker-tag.sh
+
+      - name: Check full image tag
+        id: full
+        env:
+          DOCKER_TAG: ghcr.io/kreuzberg-dev/kreuzberg:${{ needs.prepare.outputs.version }}
+          SUMMARY_LABEL: full
+        run: scripts/publish/check-docker-tag.sh
+
+      - name: Check CLI image tag
+        id: cli
+        env:
+          DOCKER_TAG: ghcr.io/kreuzberg-dev/kreuzberg-cli:${{ needs.prepare.outputs.version }}
+          SUMMARY_LABEL: cli
+        run: scripts/publish/check-docker-tag.sh
+
+  publish-docker:
+    name: Publish Docker image (${{ matrix.variant }})
+    needs:
+      - prepare
+      - check-docker
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - variant: core
+            dockerfile: docker/Dockerfile.core
+            image: ghcr.io/kreuzberg-dev/kreuzberg
+            tag_suffix: "-core"
+            extra_tag: "core"
+          - variant: full
+            dockerfile: docker/Dockerfile.full
+            image: ghcr.io/kreuzberg-dev/kreuzberg
+            tag_suffix: ""
+            extra_tag: "latest"
+          - variant: cli
+            dockerfile: docker/Dockerfile.cli
+            image: ghcr.io/kreuzberg-dev/kreuzberg-cli
+            tag_suffix: ""
+            extra_tag: "latest"
+    if: ${{ needs.prepare.outputs.release_docker == 'true' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ needs.prepare.outputs.checkout_ref }}
+          fetch-depth: 0
+          submodules: recursive
+
+      - name: Free up disk space
+        uses: kreuzberg-dev/actions/free-disk-space-linux@v1
+
+      - name: Ensure target commit
+        if: ${{ needs.prepare.outputs.target_sha != '' }}
+        run: git checkout --progress --force ${{ needs.prepare.outputs.target_sha }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Skip because tag already exists
+        if: ${{ needs.prepare.outputs.force_republish != 'true' && ((matrix.variant == 'core' && needs.check-docker.outputs.core_exists == 'true') || (matrix.variant == 'full' && needs.check-docker.outputs.full_exists == 'true') || (matrix.variant == 'cli' && needs.check-docker.outputs.cli_exists == 'true')) }}
+        run: echo "Docker tag already exists for variant ${{ matrix.variant }}; skipping publish." >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Build AMD64 test image
+        if: ${{ needs.prepare.outputs.force_republish == 'true' || (matrix.variant == 'core' && needs.check-docker.outputs.core_exists != 'true') || (matrix.variant == 'full' && needs.check-docker.outputs.full_exists != 'true') || (matrix.variant == 'cli' && needs.check-docker.outputs.cli_exists != 'true') }}
+        run: docker build -f ${{ matrix.dockerfile }} --build-arg ONNXRUNTIME_VERSION=${{ env.ORT_VERSION }} -t kreuzberg-publish:${{ matrix.variant }}-test .
+
+      - name: Run Docker tests
+        if: ${{ needs.prepare.outputs.force_republish == 'true' || (matrix.variant == 'core' && needs.check-docker.outputs.core_exists != 'true') || (matrix.variant == 'full' && needs.check-docker.outputs.full_exists != 'true') || (matrix.variant == 'cli' && needs.check-docker.outputs.cli_exists != 'true') }}
+        run: python3 scripts/ci/docker/test_docker.py --image kreuzberg-publish:${{ matrix.variant }}-test --variant ${{ matrix.variant }} --verbose
+
+      - name: Log in to GitHub Container Registry
+        if: ${{ needs.prepare.outputs.dry_run != 'true' && (needs.prepare.outputs.force_republish == 'true' || (matrix.variant == 'core' && needs.check-docker.outputs.core_exists != 'true') || (matrix.variant == 'full' && needs.check-docker.outputs.full_exists != 'true') || (matrix.variant == 'cli' && needs.check-docker.outputs.cli_exists != 'true')) }}
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        if: ${{ needs.prepare.outputs.dry_run != 'true' && (needs.prepare.outputs.force_republish == 'true' || (matrix.variant == 'core' && needs.check-docker.outputs.core_exists != 'true') || (matrix.variant == 'full' && needs.check-docker.outputs.full_exists != 'true') || (matrix.variant == 'cli' && needs.check-docker.outputs.cli_exists != 'true')) }}
+        id: docker_meta
+        uses: docker/metadata-action@v6
+        with:
+          images: ${{ matrix.image }}
+          tags: |
+            type=raw,value=${{ needs.prepare.outputs.version }}${{ matrix.tag_suffix }}
+            type=raw,value=${{ matrix.extra_tag }}
+
+      - name: Build and push image
+        if: ${{ needs.prepare.outputs.dry_run != 'true' && (needs.prepare.outputs.force_republish == 'true' || (matrix.variant == 'core' && needs.check-docker.outputs.core_exists != 'true') || (matrix.variant == 'full' && needs.check-docker.outputs.full_exists != 'true') || (matrix.variant == 'cli' && needs.check-docker.outputs.cli_exists != 'true')) }}
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: ${{ matrix.dockerfile }}
+          push: true
+          build-args: |
+            ONNXRUNTIME_VERSION=${{ env.ORT_VERSION }}
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: |
+            ${{ steps.docker_meta.outputs.labels }}
+            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+            org.opencontainers.image.description=Kreuzberg document intelligence - ${{ matrix.variant }} variant
+            org.opencontainers.image.licenses=MIT
+          platforms: linux/amd64,linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max,scope=publish-docker-${{ matrix.variant }}
+
+      - name: Docker dry-run summary
+        if: ${{ needs.prepare.outputs.dry_run == 'true' }}
+        env:
+          IMAGE: ${{ matrix.image }}
+          VERSION: ${{ needs.prepare.outputs.version }}
+          TAG_SUFFIX: ${{ matrix.tag_suffix }}
+        run: scripts/publish/docker/dry-run-summary.sh
+
+      - name: Clean up local Docker images
+        if: ${{ always() }}
+        run: docker rmi kreuzberg-publish:${{ matrix.variant }}-test || true