From 6f1ee72e102febd62ff8406a8b083c9f1a6a6915 Mon Sep 17 00:00:00 2001 From: Henrik Jess Nielsen Date: Tue, 26 May 2026 22:30:38 +0200 Subject: [PATCH] feat: containerize for mmd.i80.dk deployment via Gitea/Nomad - Add Dockerfile (python:3.12-slim, HF_HOME=/app/data/hf-cache) - Add mmd.nomad (multi-task: web=dashboard, worker=scheduler) - Add .gitea/workflows/deploy.yml (build->Harbor, deploy->Nomad) - Add scheduler.py (stdlib scheduler replaces cron in container) - Add requirements.txt - dashboard.py: LOG_DIR + PORT/HOST from env vars - saxo_auth.py: TOKEN_FILE from SAXO_TOKEN_FILE env var - .gitignore: proper ignores for container project Volume moneymaker-data (/app/data) holds: - logs/ (shared between web+worker) - .saxo_token.json (pre-copy once after first deploy) - hf-cache/ (HuggingFace FinBERT cache) Gitea secrets required: DATABASE_URL, ANTHROPIC_API_KEY, SAXO_APP_KEY, SAXO_APP_SECRET_1, HARBOR_ROBOT_TOKEN --- .gitea/workflows/deploy.yml | 113 ++++++++++++++++++++++++++ .gitignore | 13 +++ Dockerfile | 34 ++++++++ dashboard.py | 9 ++- logs/.gitkeep | 0 mmd.nomad | 155 ++++++++++++++++++++++++++++++++++++ requirements.txt | 10 +++ saxo_auth.py | 2 +- scheduler.py | 86 ++++++++++++++++++++ 9 files changed, 417 insertions(+), 5 deletions(-) create mode 100644 .gitea/workflows/deploy.yml create mode 100644 Dockerfile create mode 100644 logs/.gitkeep create mode 100644 mmd.nomad create mode 100644 requirements.txt create mode 100644 scheduler.py diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml new file mode 100644 index 0000000..b7aa518 --- /dev/null +++ b/.gitea/workflows/deploy.yml @@ -0,0 +1,113 @@ +name: Build and Deploy MoneyMaker + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + build-and-deploy: + runs-on: debian-host + + env: + PATH: /usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/sbin:/bin:/snap/bin + DOCKER_HOST: unix:///var/run/docker.sock + BUILDX_CONFIG: /tmp/buildx + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: System info + run: | + uname -a + whoami + + - name: Set up Docker Context for Buildx + id: buildx-context + run: | + export DOCKER_HOST=tcp://docker:2376/ + export DOCKER_TLS_VERIFY=0 + docker context rm builders || true + docker context create builders + + - name: Verify Docker + run: docker --version + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + env: + PATH: /usr/bin:/usr/local/bin:/bin:/sbin:/usr/sbin + + - name: Log in to Harbor Registry + run: | + echo "${{ secrets.HARBOR_ROBOT_TOKEN }}" | docker login registry.i80.dk -u "robot\$gitserver" --password-stdin + env: + PATH: /usr/bin:/usr/local/bin:/bin:/sbin:/usr/sbin + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + env: + PATH: /usr/bin:/usr/local/bin:/bin:/sbin:/usr/sbin + with: + context: . + file: ./Dockerfile + push: true + tags: | + registry.i80.dk/gitea/mmd:latest + registry.i80.dk/gitea/mmd:${{ github.sha }} + build-args: | + BUILD_VERSION=${{ github.ref_name }}-${{ github.sha }} + GIT_COMMIT=${{ github.sha }} + BUILD_TIME=${{ github.event.head_commit.timestamp }} + + - name: Substitute secrets into Nomad job + run: | + sed \ + -e "s|__DATABASE_URL__|${{ secrets.DATABASE_URL }}|g" \ + -e "s|__ANTHROPIC_API_KEY__|${{ secrets.ANTHROPIC_API_KEY }}|g" \ + -e "s|__SAXO_APP_KEY__|${{ secrets.SAXO_APP_KEY }}|g" \ + -e "s|__SAXO_APP_SECRET_1__|${{ secrets.SAXO_APP_SECRET_1 }}|g" \ + mmd.nomad > mmd_deploy.nomad + env: + PATH: /usr/bin:/usr/local/bin:/bin:/sbin:/usr/sbin + + - name: Validate Nomad job + run: | + echo "Validating Nomad job specification..." + nomad job validate mmd_deploy.nomad + env: + NOMAD_ADDR: "https://nomad.i80.dk:4646" + + - name: Deploy to Nomad + run: | + echo "Deploying to Nomad cluster..." + nomad job run mmd_deploy.nomad + env: + NOMAD_ADDR: "https://nomad.i80.dk:4646" + + - name: Wait for deployment + run: | + echo "Checking deployment status..." + sleep 15 + nomad job status moneymaker + echo "=== Allocations ===" + nomad job allocs moneymaker + env: + NOMAD_ADDR: "https://nomad.i80.dk:4646" + + - name: Health check + run: | + echo "Waiting for Traefik routing..." + sleep 30 + curl -f https://mmd.i80.dk/health || echo "Not yet available via Traefik — check Nomad UI" + env: + PATH: /usr/bin:/usr/local/bin:/bin:/sbin:/usr/sbin + + - name: Deployment summary + run: | + echo "Deployment complete!" + echo " Dashboard : https://mmd.i80.dk" + echo " Health : https://mmd.i80.dk/health" + echo " Nomad UI : https://nomad.i80.dk:4646" diff --git a/.gitignore b/.gitignore index a3eff6c..b3f17f2 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,14 @@ +.env +.venv/ +__pycache__/ +*.pyc +*.pyo +*.log +logs/*.log +logs/dashboard.log .saxo_token.json +ground_news.db +mmd_deploy.nomad +*.egg-info/ +dist/ +.pytest_cache/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1032dcf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +FROM python:3.12-slim + +ARG BUILD_VERSION=unknown +ARG BUILD_TIME=unknown +ARG GIT_COMMIT=unknown + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + BUILD_VERSION=${BUILD_VERSION} \ + BUILD_TIME=${BUILD_TIME} \ + GIT_COMMIT=${GIT_COMMIT} \ + HF_HOME=/app/data/hf-cache \ + TRANSFORMERS_VERBOSITY=error \ + HF_HUB_DISABLE_PROGRESS_BARS=1 + +WORKDIR /app + +RUN pip install --upgrade --no-cache-dir pip + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +RUN mkdir -p /app/data/logs /app/data/hf-cache + +ENV PORT=5001 \ + HOST=0.0.0.0 \ + LOG_DIR=/app/data/logs \ + SAXO_TOKEN_FILE=/app/data/.saxo_token.json + +EXPOSE 5001 + +CMD ["python", "dashboard.py"] diff --git a/dashboard.py b/dashboard.py index a4345a0..1a7771d 100644 --- a/dashboard.py +++ b/dashboard.py @@ -14,6 +14,7 @@ Auto-refreshes every 60 seconds. Shows: """ import argparse import json +import os import time from datetime import datetime, timezone from pathlib import Path @@ -25,7 +26,7 @@ from db import get_conn, DB_TYPE from report import _c25_day_return, _unrealised_pnl, _realised_pnl, _total_fees, _signal_accuracy CAPITAL = 10_000 -LOG_DIR = Path(__file__).parent / "logs" +LOG_DIR = Path(os.getenv("LOG_DIR", str(Path(__file__).parent / "logs"))) REFRESH = 60 # seconds app = Flask(__name__) @@ -316,10 +317,10 @@ def health(): def main(): parser = argparse.ArgumentParser(description="MoneyMaker Dashboard") - parser.add_argument("--port", type=int, default=5001) - parser.add_argument("--host", default="0.0.0.0") + parser.add_argument("--port", type=int, default=int(os.getenv("PORT", 5001))) + parser.add_argument("--host", default=os.getenv("HOST", "0.0.0.0")) args = parser.parse_args() - print(f"\n MoneyMaker Dashboard → http://localhost:{args.port}\n") + print(f"\n MoneyMaker Dashboard -> http://localhost:{args.port}\n") app.run(host=args.host, port=args.port, debug=False) diff --git a/logs/.gitkeep b/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/mmd.nomad b/mmd.nomad new file mode 100644 index 0000000..e74f3a8 --- /dev/null +++ b/mmd.nomad @@ -0,0 +1,155 @@ +job "moneymaker" { + region = "global" + datacenters = ["dc1"] + type = "service" + + update { + stagger = "30s" + max_parallel = 1 + canary = 1 + min_healthy_time = "10s" + healthy_deadline = "5m" + auto_revert = true + auto_promote = true + progress_deadline = "10m" + } + + group "app" { + count = 1 + + network { + port "http" {} + } + + reschedule { + attempts = 5 + interval = "10m" + delay = "30s" + delay_function = "exponential" + max_delay = "120s" + unlimited = false + } + + constraint { + attribute = "${node.unique.name}" + value = "autobox" + } + + volume "moneymaker-data" { + type = "host" + source = "moneymaker-data" + read_only = false + } + + service { + provider = "consul" + name = "moneymaker" + port = "http" + + tags = [ + "traefik.enable=true", + "traefik.http.routers.moneymaker.rule=Host(`mmd.i80.dk`)", + "traefik.http.routers.moneymaker.tls=true", + ] + + canary_tags = [ + "traefik.enable=false", + ] + + check { + name = "http_health_check" + type = "http" + path = "/health" + interval = "10s" + timeout = "5s" + + check_restart { + limit = 3 + grace = "30s" + } + } + } + + # -- Dashboard (Flask web app) ---------------------------------------- + task "web" { + driver = "docker" + + volume_mount { + volume = "moneymaker-data" + destination = "/app/data" + read_only = false + } + + config { + image = "registry.i80.dk/gitea/mmd:latest" + ports = ["http"] + force_pull = true + } + + restart { + attempts = 10 + interval = "10m" + delay = "10s" + mode = "fail" + } + + env { + APP_ENV = "production" + PORT = "${NOMAD_PORT_http}" + HOST = "0.0.0.0" + LOG_DIR = "/app/data/logs" + SAXO_TOKEN_FILE = "/app/data/.saxo_token.json" + HF_HOME = "/app/data/hf-cache" + DATABASE_URL = "__DATABASE_URL__" + ANTHROPIC_API_KEY = "__ANTHROPIC_API_KEY__" + SAXO_APP_KEY = "__SAXO_APP_KEY__" + SAXO_APP_SECRET_1 = "__SAXO_APP_SECRET_1__" + } + + resources { + cpu = 300 + memory = 512 + } + } + + # -- Worker (pipeline scheduler, runs FinBERT + Claude) ---------------- + task "worker" { + driver = "docker" + + volume_mount { + volume = "moneymaker-data" + destination = "/app/data" + read_only = false + } + + config { + image = "registry.i80.dk/gitea/mmd:latest" + command = "python" + args = ["scheduler.py"] + force_pull = true + } + + restart { + attempts = 10 + interval = "10m" + delay = "10s" + mode = "fail" + } + + env { + LOG_DIR = "/app/data/logs" + SAXO_TOKEN_FILE = "/app/data/.saxo_token.json" + HF_HOME = "/app/data/hf-cache" + DATABASE_URL = "__DATABASE_URL__" + ANTHROPIC_API_KEY = "__ANTHROPIC_API_KEY__" + SAXO_APP_KEY = "__SAXO_APP_KEY__" + SAXO_APP_SECRET_1 = "__SAXO_APP_SECRET_1__" + } + + resources { + cpu = 1500 + memory = 3072 + } + } + } +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1e87a79 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +Flask==3.1.3 +psycopg2-binary==2.9.12 +python-dotenv==1.2.2 +requests==2.34.2 +httpx==0.28.1 +anthropic==0.104.1 +yfinance==1.4.0 +transformers==5.9.0 +torch==2.12.0 +beautifulsoup4==4.14.3 diff --git a/saxo_auth.py b/saxo_auth.py index d3ba310..a3d468f 100644 --- a/saxo_auth.py +++ b/saxo_auth.py @@ -34,7 +34,7 @@ APP_SECRET = os.getenv("SAXO_APP_SECRET_1", "") AUTH_URL = os.getenv("SAXO_AUTH_URL", "https://sim.logonvalidation.net/authorize") TOKEN_URL = os.getenv("SAXO_TOKEN_URL", "https://sim.logonvalidation.net/token") REDIRECT = os.getenv("SAXO_REDIRECT", "http://localhost:8765/callback") -TOKEN_FILE = Path(__file__).parent / ".saxo_token.json" +TOKEN_FILE = Path(os.getenv("SAXO_TOKEN_FILE", str(Path(__file__).parent / ".saxo_token.json"))) REDIRECT_PORT = 8765 diff --git a/scheduler.py b/scheduler.py new file mode 100644 index 0000000..6e5744f --- /dev/null +++ b/scheduler.py @@ -0,0 +1,86 @@ +""" +scheduler.py — MoneyMaker worker daemon. + +Runs the pipeline at scheduled times (Mon-Fri UTC): + 04:00 analyze-only (NLP before Copenhagen market open 09:00 CET) + 07:30 trade window 1 + 10:00 trade window 2 + 12:30 trade window 3 + 14:30 trade window 4 + 17:00 daily P&L report + +Outputs appended to LOG_DIR/runner_YYYY-MM-DD.log so the dashboard can +show a live log tail. Uses only stdlib — no extra scheduler dependency. +""" +import os +import sys +import time +import traceback +from datetime import datetime, timezone +from pathlib import Path + +LOG_DIR = Path(os.getenv("LOG_DIR", str(Path(__file__).parent / "data" / "logs"))) +LOG_DIR.mkdir(parents=True, exist_ok=True) + +# hour, minute, weekdays (0=Mon…4=Fri), analyze_only, is_report +SCHEDULE = [ + (4, 0, frozenset(range(5)), True, False), + (7, 30, frozenset(range(5)), False, False), + (10, 0, frozenset(range(5)), False, False), + (12, 30, frozenset(range(5)), False, False), + (14, 30, frozenset(range(5)), False, False), + (17, 0, frozenset(range(5)), False, True), +] + + +def _log_path() -> Path: + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + return LOG_DIR / f"runner_{today}.log" + + +def _run_job(analyze_only: bool, is_report: bool) -> None: + log = _log_path() + with open(log, "a", buffering=1) as fh: + old_out, old_err = sys.stdout, sys.stderr + sys.stdout = sys.stderr = fh + try: + if is_report: + from report import print_report + print_report() + else: + from runner import run_pipeline + run_pipeline(analyze_only=analyze_only) + except Exception as exc: + print(f"[scheduler] ERROR: {exc}") + traceback.print_exc() + finally: + sys.stdout = old_out + sys.stderr = old_err + + +def main() -> None: + last_run: dict = {} + print(f"[scheduler] started — LOG_DIR={LOG_DIR}", flush=True) + + while True: + now = datetime.now(timezone.utc) + dow = now.weekday() + + for hour, minute, days, analyze_only, is_report in SCHEDULE: + if dow not in days: + continue + key = (hour, minute, now.date()) + if key in last_run: + continue + if now.hour == hour and now.minute == minute: + last_run[key] = True + label = "report" if is_report else ("analyze-only" if analyze_only else "full") + print(f"[scheduler] {now.strftime('%H:%M UTC')} — {label}", flush=True) + _run_job(analyze_only, is_report) + print(f"[scheduler] {label} done", flush=True) + + time.sleep(30) + + +if __name__ == "__main__": + main()