feat: containerize for mmd.i80.dk deployment via Gitea/Nomad

- Add Dockerfile (python:3.12-slim, HF_HOME=/app/data/hf-cache) - Add mmd.nomad (multi-task: web=dashboard, worker=scheduler) - Add .gitea/workflows/deploy.yml (build->Harbor, deploy->Nomad) - Add scheduler.py (stdlib scheduler replaces cron in container) - Add requirements.txt - dashboard.py: LOG_DIR + PORT/HOST from env vars - saxo_auth.py: TOKEN_FILE from SAXO_TOKEN_FILE env var - .gitignore: proper ignores for container project Volume moneymaker-data (/app/data) holds: - logs/ (shared between web+worker) - .saxo_token.json (pre-copy once after first deploy) - hf-cache/ (HuggingFace FinBERT cache) Gitea secrets required: DATABASE_URL, ANTHROPIC_API_KEY, SAXO_APP_KEY, SAXO_APP_SECRET_1, HARBOR_ROBOT_TOKEN
2026-05-26 22:30:38 +02:00
parent 05eed51e7d
commit 6f1ee72e10
9 changed files with 417 additions and 5 deletions
--- a/.gitea/workflows/deploy.yml
+++ b/.gitea/workflows/deploy.yml
@@ -0,0 +1,113 @@
+name: Build and Deploy MoneyMaker
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  build-and-deploy:
+    runs-on: debian-host
+
+    env:
+      PATH: /usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/sbin:/bin:/snap/bin
+      DOCKER_HOST: unix:///var/run/docker.sock
+      BUILDX_CONFIG: /tmp/buildx
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: System info
+      run: |
+        uname -a
+        whoami
+
+    - name: Set up Docker Context for Buildx
+      id: buildx-context
+      run: |
+        export DOCKER_HOST=tcp://docker:2376/
+        export DOCKER_TLS_VERIFY=0
+        docker context rm builders || true
+        docker context create builders
+
+    - name: Verify Docker
+      run: docker --version
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      env:
+        PATH: /usr/bin:/usr/local/bin:/bin:/sbin:/usr/sbin
+
+    - name: Log in to Harbor Registry
+      run: |
+        echo "${{ secrets.HARBOR_ROBOT_TOKEN }}" | docker login registry.i80.dk -u "robot\$gitserver" --password-stdin
+      env:
+        PATH: /usr/bin:/usr/local/bin:/bin:/sbin:/usr/sbin
+
+    - name: Build and push Docker image
+      uses: docker/build-push-action@v5
+      env:
+        PATH: /usr/bin:/usr/local/bin:/bin:/sbin:/usr/sbin
+      with:
+        context: .
+        file: ./Dockerfile
+        push: true
+        tags: |
+          registry.i80.dk/gitea/mmd:latest
+          registry.i80.dk/gitea/mmd:${{ github.sha }}
+        build-args: |
+          BUILD_VERSION=${{ github.ref_name }}-${{ github.sha }}
+          GIT_COMMIT=${{ github.sha }}
+          BUILD_TIME=${{ github.event.head_commit.timestamp }}
+
+    - name: Substitute secrets into Nomad job
+      run: |
+        sed \
+          -e "s|__DATABASE_URL__|${{ secrets.DATABASE_URL }}|g" \
+          -e "s|__ANTHROPIC_API_KEY__|${{ secrets.ANTHROPIC_API_KEY }}|g" \
+          -e "s|__SAXO_APP_KEY__|${{ secrets.SAXO_APP_KEY }}|g" \
+          -e "s|__SAXO_APP_SECRET_1__|${{ secrets.SAXO_APP_SECRET_1 }}|g" \
+          mmd.nomad > mmd_deploy.nomad
+      env:
+        PATH: /usr/bin:/usr/local/bin:/bin:/sbin:/usr/sbin
+
+    - name: Validate Nomad job
+      run: |
+        echo "Validating Nomad job specification..."
+        nomad job validate mmd_deploy.nomad
+      env:
+        NOMAD_ADDR: "https://nomad.i80.dk:4646"
+
+    - name: Deploy to Nomad
+      run: |
+        echo "Deploying to Nomad cluster..."
+        nomad job run mmd_deploy.nomad
+      env:
+        NOMAD_ADDR: "https://nomad.i80.dk:4646"
+
+    - name: Wait for deployment
+      run: |
+        echo "Checking deployment status..."
+        sleep 15
+        nomad job status moneymaker
+        echo "=== Allocations ==="
+        nomad job allocs moneymaker
+      env:
+        NOMAD_ADDR: "https://nomad.i80.dk:4646"
+
+    - name: Health check
+      run: |
+        echo "Waiting for Traefik routing..."
+        sleep 30
+        curl -f https://mmd.i80.dk/health || echo "Not yet available via Traefik — check Nomad UI"
+      env:
+        PATH: /usr/bin:/usr/local/bin:/bin:/sbin:/usr/sbin
+
+    - name: Deployment summary
+      run: |
+        echo "Deployment complete!"
+        echo "  Dashboard : https://mmd.i80.dk"
+        echo "  Health    : https://mmd.i80.dk/health"
+        echo "  Nomad UI  : https://nomad.i80.dk:4646"
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,14 @@
+.env
+.venv/
+__pycache__/
+*.pyc
+*.pyo
+*.log
+logs/*.log
+logs/dashboard.log
 .saxo_token.json
+ground_news.db
+mmd_deploy.nomad
+*.egg-info/
+dist/
+.pytest_cache/
--- a/34
+++ b/34
@@ -0,0 +1,34 @@
+FROM python:3.12-slim
+
+ARG BUILD_VERSION=unknown
+ARG BUILD_TIME=unknown
+ARG GIT_COMMIT=unknown
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    BUILD_VERSION=${BUILD_VERSION} \
+    BUILD_TIME=${BUILD_TIME} \
+    GIT_COMMIT=${GIT_COMMIT} \
+    HF_HOME=/app/data/hf-cache \
+    TRANSFORMERS_VERBOSITY=error \
+    HF_HUB_DISABLE_PROGRESS_BARS=1
+
+WORKDIR /app
+
+RUN pip install --upgrade --no-cache-dir pip
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+RUN mkdir -p /app/data/logs /app/data/hf-cache
+
+ENV PORT=5001 \
+    HOST=0.0.0.0 \
+    LOG_DIR=/app/data/logs \
+    SAXO_TOKEN_FILE=/app/data/.saxo_token.json
+
+EXPOSE 5001
+
+CMD ["python", "dashboard.py"]
--- a/dashboard.py
+++ b/dashboard.py
@@ -14,6 +14,7 @@ Auto-refreshes every 60 seconds. Shows:
 """
 import argparse
 import json
+import os
 import time
 from datetime import datetime, timezone
 from pathlib import Path
@@ -25,7 +26,7 @@ from db import get_conn, DB_TYPE
 from report import _c25_day_return, _unrealised_pnl, _realised_pnl, _total_fees, _signal_accuracy

 CAPITAL  = 10_000
-LOG_DIR  = Path(__file__).parent / "logs"
+LOG_DIR  = Path(os.getenv("LOG_DIR", str(Path(__file__).parent / "logs")))
 REFRESH  = 60   # seconds

 app = Flask(__name__)
@@ -316,10 +317,10 @@ def health():

 def main():
    parser = argparse.ArgumentParser(description="MoneyMaker Dashboard")
-    parser.add_argument("--port", type=int, default=5001)
-    parser.add_argument("--host", default="0.0.0.0")
+    parser.add_argument("--port", type=int, default=int(os.getenv("PORT", 5001)))
+    parser.add_argument("--host", default=os.getenv("HOST", "0.0.0.0"))
    args = parser.parse_args()
-    print(f"\n  MoneyMaker Dashboard → http://localhost:{args.port}\n")
+    print(f"\n  MoneyMaker Dashboard -> http://localhost:{args.port}\n")
    app.run(host=args.host, port=args.port, debug=False)


--- a/logs/.gitkeep
+++ b/logs/.gitkeep
--- a/mmd.nomad
+++ b/mmd.nomad
@@ -0,0 +1,155 @@
+job "moneymaker" {
+  region      = "global"
+  datacenters = ["dc1"]
+  type        = "service"
+
+  update {
+    stagger           = "30s"
+    max_parallel      = 1
+    canary            = 1
+    min_healthy_time  = "10s"
+    healthy_deadline  = "5m"
+    auto_revert       = true
+    auto_promote      = true
+    progress_deadline = "10m"
+  }
+
+  group "app" {
+    count = 1
+
+    network {
+      port "http" {}
+    }
+
+    reschedule {
+      attempts       = 5
+      interval       = "10m"
+      delay          = "30s"
+      delay_function = "exponential"
+      max_delay      = "120s"
+      unlimited      = false
+    }
+
+    constraint {
+      attribute = "${node.unique.name}"
+      value     = "autobox"
+    }
+
+    volume "moneymaker-data" {
+      type      = "host"
+      source    = "moneymaker-data"
+      read_only = false
+    }
+
+    service {
+      provider = "consul"
+      name     = "moneymaker"
+      port     = "http"
+
+      tags = [
+        "traefik.enable=true",
+        "traefik.http.routers.moneymaker.rule=Host(`mmd.i80.dk`)",
+        "traefik.http.routers.moneymaker.tls=true",
+      ]
+
+      canary_tags = [
+        "traefik.enable=false",
+      ]
+
+      check {
+        name     = "http_health_check"
+        type     = "http"
+        path     = "/health"
+        interval = "10s"
+        timeout  = "5s"
+
+        check_restart {
+          limit = 3
+          grace = "30s"
+        }
+      }
+    }
+
+    # -- Dashboard (Flask web app) ----------------------------------------
+    task "web" {
+      driver = "docker"
+
+      volume_mount {
+        volume      = "moneymaker-data"
+        destination = "/app/data"
+        read_only   = false
+      }
+
+      config {
+        image      = "registry.i80.dk/gitea/mmd:latest"
+        ports      = ["http"]
+        force_pull = true
+      }
+
+      restart {
+        attempts = 10
+        interval = "10m"
+        delay    = "10s"
+        mode     = "fail"
+      }
+
+      env {
+        APP_ENV           = "production"
+        PORT              = "${NOMAD_PORT_http}"
+        HOST              = "0.0.0.0"
+        LOG_DIR           = "/app/data/logs"
+        SAXO_TOKEN_FILE   = "/app/data/.saxo_token.json"
+        HF_HOME           = "/app/data/hf-cache"
+        DATABASE_URL      = "__DATABASE_URL__"
+        ANTHROPIC_API_KEY = "__ANTHROPIC_API_KEY__"
+        SAXO_APP_KEY      = "__SAXO_APP_KEY__"
+        SAXO_APP_SECRET_1 = "__SAXO_APP_SECRET_1__"
+      }
+
+      resources {
+        cpu    = 300
+        memory = 512
+      }
+    }
+
+    # -- Worker (pipeline scheduler, runs FinBERT + Claude) ----------------
+    task "worker" {
+      driver = "docker"
+
+      volume_mount {
+        volume      = "moneymaker-data"
+        destination = "/app/data"
+        read_only   = false
+      }
+
+      config {
+        image      = "registry.i80.dk/gitea/mmd:latest"
+        command    = "python"
+        args       = ["scheduler.py"]
+        force_pull = true
+      }
+
+      restart {
+        attempts = 10
+        interval = "10m"
+        delay    = "10s"
+        mode     = "fail"
+      }
+
+      env {
+        LOG_DIR           = "/app/data/logs"
+        SAXO_TOKEN_FILE   = "/app/data/.saxo_token.json"
+        HF_HOME           = "/app/data/hf-cache"
+        DATABASE_URL      = "__DATABASE_URL__"
+        ANTHROPIC_API_KEY = "__ANTHROPIC_API_KEY__"
+        SAXO_APP_KEY      = "__SAXO_APP_KEY__"
+        SAXO_APP_SECRET_1 = "__SAXO_APP_SECRET_1__"
+      }
+
+      resources {
+        cpu    = 1500
+        memory = 3072
+      }
+    }
+  }
+}
--- a/requirements.txt
+++ b/requirements.txt
@@ -0,0 +1,10 @@
+Flask==3.1.3
+psycopg2-binary==2.9.12
+python-dotenv==1.2.2
+requests==2.34.2
+httpx==0.28.1
+anthropic==0.104.1
+yfinance==1.4.0
+transformers==5.9.0
+torch==2.12.0
+beautifulsoup4==4.14.3
--- a/saxo_auth.py
+++ b/saxo_auth.py
@@ -34,7 +34,7 @@ APP_SECRET   = os.getenv("SAXO_APP_SECRET_1", "")
 AUTH_URL     = os.getenv("SAXO_AUTH_URL", "https://sim.logonvalidation.net/authorize")
 TOKEN_URL    = os.getenv("SAXO_TOKEN_URL", "https://sim.logonvalidation.net/token")
 REDIRECT     = os.getenv("SAXO_REDIRECT", "http://localhost:8765/callback")
-TOKEN_FILE   = Path(__file__).parent / ".saxo_token.json"
+TOKEN_FILE   = Path(os.getenv("SAXO_TOKEN_FILE", str(Path(__file__).parent / ".saxo_token.json")))
 REDIRECT_PORT = 8765


--- a/scheduler.py
+++ b/scheduler.py
@@ -0,0 +1,86 @@
+"""
+scheduler.py — MoneyMaker worker daemon.
+
+Runs the pipeline at scheduled times (Mon-Fri UTC):
+  04:00  analyze-only  (NLP before Copenhagen market open 09:00 CET)
+  07:30  trade window 1
+  10:00  trade window 2
+  12:30  trade window 3
+  14:30  trade window 4
+  17:00  daily P&L report
+
+Outputs appended to LOG_DIR/runner_YYYY-MM-DD.log so the dashboard can
+show a live log tail. Uses only stdlib — no extra scheduler dependency.
+"""
+import os
+import sys
+import time
+import traceback
+from datetime import datetime, timezone
+from pathlib import Path
+
+LOG_DIR = Path(os.getenv("LOG_DIR", str(Path(__file__).parent / "data" / "logs")))
+LOG_DIR.mkdir(parents=True, exist_ok=True)
+
+# hour, minute, weekdays (0=Mon…4=Fri), analyze_only, is_report
+SCHEDULE = [
+    (4,  0,  frozenset(range(5)), True,  False),
+    (7,  30, frozenset(range(5)), False, False),
+    (10, 0,  frozenset(range(5)), False, False),
+    (12, 30, frozenset(range(5)), False, False),
+    (14, 30, frozenset(range(5)), False, False),
+    (17, 0,  frozenset(range(5)), False, True),
+]
+
+
+def _log_path() -> Path:
+    today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    return LOG_DIR / f"runner_{today}.log"
+
+
+def _run_job(analyze_only: bool, is_report: bool) -> None:
+    log = _log_path()
+    with open(log, "a", buffering=1) as fh:
+        old_out, old_err = sys.stdout, sys.stderr
+        sys.stdout = sys.stderr = fh
+        try:
+            if is_report:
+                from report import print_report
+                print_report()
+            else:
+                from runner import run_pipeline
+                run_pipeline(analyze_only=analyze_only)
+        except Exception as exc:
+            print(f"[scheduler] ERROR: {exc}")
+            traceback.print_exc()
+        finally:
+            sys.stdout = old_out
+            sys.stderr = old_err
+
+
+def main() -> None:
+    last_run: dict = {}
+    print(f"[scheduler] started — LOG_DIR={LOG_DIR}", flush=True)
+
+    while True:
+        now = datetime.now(timezone.utc)
+        dow = now.weekday()
+
+        for hour, minute, days, analyze_only, is_report in SCHEDULE:
+            if dow not in days:
+                continue
+            key = (hour, minute, now.date())
+            if key in last_run:
+                continue
+            if now.hour == hour and now.minute == minute:
+                last_run[key] = True
+                label = "report" if is_report else ("analyze-only" if analyze_only else "full")
+                print(f"[scheduler] {now.strftime('%H:%M UTC')} — {label}", flush=True)
+                _run_job(analyze_only, is_report)
+                print(f"[scheduler] {label} done", flush=True)
+
+        time.sleep(30)
+
+
+if __name__ == "__main__":
+    main()