feat: containerize for mmd.i80.dk deployment via Gitea/Nomad
Some checks failed
Build and Deploy MoneyMaker / build-and-deploy (push) Failing after 15s
Some checks failed
Build and Deploy MoneyMaker / build-and-deploy (push) Failing after 15s
- Add Dockerfile (python:3.12-slim, HF_HOME=/app/data/hf-cache) - Add mmd.nomad (multi-task: web=dashboard, worker=scheduler) - Add .gitea/workflows/deploy.yml (build->Harbor, deploy->Nomad) - Add scheduler.py (stdlib scheduler replaces cron in container) - Add requirements.txt - dashboard.py: LOG_DIR + PORT/HOST from env vars - saxo_auth.py: TOKEN_FILE from SAXO_TOKEN_FILE env var - .gitignore: proper ignores for container project Volume moneymaker-data (/app/data) holds: - logs/ (shared between web+worker) - .saxo_token.json (pre-copy once after first deploy) - hf-cache/ (HuggingFace FinBERT cache) Gitea secrets required: DATABASE_URL, ANTHROPIC_API_KEY, SAXO_APP_KEY, SAXO_APP_SECRET_1, HARBOR_ROBOT_TOKEN
This commit is contained in:
113
.gitea/workflows/deploy.yml
Normal file
113
.gitea/workflows/deploy.yml
Normal file
@@ -0,0 +1,113 @@
|
|||||||
|
name: Build and Deploy MoneyMaker
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build-and-deploy:
|
||||||
|
runs-on: debian-host
|
||||||
|
|
||||||
|
env:
|
||||||
|
PATH: /usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/sbin:/bin:/snap/bin
|
||||||
|
DOCKER_HOST: unix:///var/run/docker.sock
|
||||||
|
BUILDX_CONFIG: /tmp/buildx
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: System info
|
||||||
|
run: |
|
||||||
|
uname -a
|
||||||
|
whoami
|
||||||
|
|
||||||
|
- name: Set up Docker Context for Buildx
|
||||||
|
id: buildx-context
|
||||||
|
run: |
|
||||||
|
export DOCKER_HOST=tcp://docker:2376/
|
||||||
|
export DOCKER_TLS_VERIFY=0
|
||||||
|
docker context rm builders || true
|
||||||
|
docker context create builders
|
||||||
|
|
||||||
|
- name: Verify Docker
|
||||||
|
run: docker --version
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v3
|
||||||
|
env:
|
||||||
|
PATH: /usr/bin:/usr/local/bin:/bin:/sbin:/usr/sbin
|
||||||
|
|
||||||
|
- name: Log in to Harbor Registry
|
||||||
|
run: |
|
||||||
|
echo "${{ secrets.HARBOR_ROBOT_TOKEN }}" | docker login registry.i80.dk -u "robot\$gitserver" --password-stdin
|
||||||
|
env:
|
||||||
|
PATH: /usr/bin:/usr/local/bin:/bin:/sbin:/usr/sbin
|
||||||
|
|
||||||
|
- name: Build and push Docker image
|
||||||
|
uses: docker/build-push-action@v5
|
||||||
|
env:
|
||||||
|
PATH: /usr/bin:/usr/local/bin:/bin:/sbin:/usr/sbin
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: ./Dockerfile
|
||||||
|
push: true
|
||||||
|
tags: |
|
||||||
|
registry.i80.dk/gitea/mmd:latest
|
||||||
|
registry.i80.dk/gitea/mmd:${{ github.sha }}
|
||||||
|
build-args: |
|
||||||
|
BUILD_VERSION=${{ github.ref_name }}-${{ github.sha }}
|
||||||
|
GIT_COMMIT=${{ github.sha }}
|
||||||
|
BUILD_TIME=${{ github.event.head_commit.timestamp }}
|
||||||
|
|
||||||
|
- name: Substitute secrets into Nomad job
|
||||||
|
run: |
|
||||||
|
sed \
|
||||||
|
-e "s|__DATABASE_URL__|${{ secrets.DATABASE_URL }}|g" \
|
||||||
|
-e "s|__ANTHROPIC_API_KEY__|${{ secrets.ANTHROPIC_API_KEY }}|g" \
|
||||||
|
-e "s|__SAXO_APP_KEY__|${{ secrets.SAXO_APP_KEY }}|g" \
|
||||||
|
-e "s|__SAXO_APP_SECRET_1__|${{ secrets.SAXO_APP_SECRET_1 }}|g" \
|
||||||
|
mmd.nomad > mmd_deploy.nomad
|
||||||
|
env:
|
||||||
|
PATH: /usr/bin:/usr/local/bin:/bin:/sbin:/usr/sbin
|
||||||
|
|
||||||
|
- name: Validate Nomad job
|
||||||
|
run: |
|
||||||
|
echo "Validating Nomad job specification..."
|
||||||
|
nomad job validate mmd_deploy.nomad
|
||||||
|
env:
|
||||||
|
NOMAD_ADDR: "https://nomad.i80.dk:4646"
|
||||||
|
|
||||||
|
- name: Deploy to Nomad
|
||||||
|
run: |
|
||||||
|
echo "Deploying to Nomad cluster..."
|
||||||
|
nomad job run mmd_deploy.nomad
|
||||||
|
env:
|
||||||
|
NOMAD_ADDR: "https://nomad.i80.dk:4646"
|
||||||
|
|
||||||
|
- name: Wait for deployment
|
||||||
|
run: |
|
||||||
|
echo "Checking deployment status..."
|
||||||
|
sleep 15
|
||||||
|
nomad job status moneymaker
|
||||||
|
echo "=== Allocations ==="
|
||||||
|
nomad job allocs moneymaker
|
||||||
|
env:
|
||||||
|
NOMAD_ADDR: "https://nomad.i80.dk:4646"
|
||||||
|
|
||||||
|
- name: Health check
|
||||||
|
run: |
|
||||||
|
echo "Waiting for Traefik routing..."
|
||||||
|
sleep 30
|
||||||
|
curl -f https://mmd.i80.dk/health || echo "Not yet available via Traefik — check Nomad UI"
|
||||||
|
env:
|
||||||
|
PATH: /usr/bin:/usr/local/bin:/bin:/sbin:/usr/sbin
|
||||||
|
|
||||||
|
- name: Deployment summary
|
||||||
|
run: |
|
||||||
|
echo "Deployment complete!"
|
||||||
|
echo " Dashboard : https://mmd.i80.dk"
|
||||||
|
echo " Health : https://mmd.i80.dk/health"
|
||||||
|
echo " Nomad UI : https://nomad.i80.dk:4646"
|
||||||
13
.gitignore
vendored
13
.gitignore
vendored
@@ -1 +1,14 @@
|
|||||||
|
.env
|
||||||
|
.venv/
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
*.pyo
|
||||||
|
*.log
|
||||||
|
logs/*.log
|
||||||
|
logs/dashboard.log
|
||||||
.saxo_token.json
|
.saxo_token.json
|
||||||
|
ground_news.db
|
||||||
|
mmd_deploy.nomad
|
||||||
|
*.egg-info/
|
||||||
|
dist/
|
||||||
|
.pytest_cache/
|
||||||
|
|||||||
34
Dockerfile
Normal file
34
Dockerfile
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
FROM python:3.12-slim
|
||||||
|
|
||||||
|
ARG BUILD_VERSION=unknown
|
||||||
|
ARG BUILD_TIME=unknown
|
||||||
|
ARG GIT_COMMIT=unknown
|
||||||
|
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||||
|
PYTHONUNBUFFERED=1 \
|
||||||
|
BUILD_VERSION=${BUILD_VERSION} \
|
||||||
|
BUILD_TIME=${BUILD_TIME} \
|
||||||
|
GIT_COMMIT=${GIT_COMMIT} \
|
||||||
|
HF_HOME=/app/data/hf-cache \
|
||||||
|
TRANSFORMERS_VERBOSITY=error \
|
||||||
|
HF_HUB_DISABLE_PROGRESS_BARS=1
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
RUN pip install --upgrade --no-cache-dir pip
|
||||||
|
|
||||||
|
COPY requirements.txt ./
|
||||||
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
RUN mkdir -p /app/data/logs /app/data/hf-cache
|
||||||
|
|
||||||
|
ENV PORT=5001 \
|
||||||
|
HOST=0.0.0.0 \
|
||||||
|
LOG_DIR=/app/data/logs \
|
||||||
|
SAXO_TOKEN_FILE=/app/data/.saxo_token.json
|
||||||
|
|
||||||
|
EXPOSE 5001
|
||||||
|
|
||||||
|
CMD ["python", "dashboard.py"]
|
||||||
@@ -14,6 +14,7 @@ Auto-refreshes every 60 seconds. Shows:
|
|||||||
"""
|
"""
|
||||||
import argparse
|
import argparse
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
import time
|
import time
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -25,7 +26,7 @@ from db import get_conn, DB_TYPE
|
|||||||
from report import _c25_day_return, _unrealised_pnl, _realised_pnl, _total_fees, _signal_accuracy
|
from report import _c25_day_return, _unrealised_pnl, _realised_pnl, _total_fees, _signal_accuracy
|
||||||
|
|
||||||
CAPITAL = 10_000
|
CAPITAL = 10_000
|
||||||
LOG_DIR = Path(__file__).parent / "logs"
|
LOG_DIR = Path(os.getenv("LOG_DIR", str(Path(__file__).parent / "logs")))
|
||||||
REFRESH = 60 # seconds
|
REFRESH = 60 # seconds
|
||||||
|
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
@@ -316,10 +317,10 @@ def health():
|
|||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = argparse.ArgumentParser(description="MoneyMaker Dashboard")
|
parser = argparse.ArgumentParser(description="MoneyMaker Dashboard")
|
||||||
parser.add_argument("--port", type=int, default=5001)
|
parser.add_argument("--port", type=int, default=int(os.getenv("PORT", 5001)))
|
||||||
parser.add_argument("--host", default="0.0.0.0")
|
parser.add_argument("--host", default=os.getenv("HOST", "0.0.0.0"))
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
print(f"\n MoneyMaker Dashboard → http://localhost:{args.port}\n")
|
print(f"\n MoneyMaker Dashboard -> http://localhost:{args.port}\n")
|
||||||
app.run(host=args.host, port=args.port, debug=False)
|
app.run(host=args.host, port=args.port, debug=False)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
0
logs/.gitkeep
Normal file
0
logs/.gitkeep
Normal file
155
mmd.nomad
Normal file
155
mmd.nomad
Normal file
@@ -0,0 +1,155 @@
|
|||||||
|
job "moneymaker" {
|
||||||
|
region = "global"
|
||||||
|
datacenters = ["dc1"]
|
||||||
|
type = "service"
|
||||||
|
|
||||||
|
update {
|
||||||
|
stagger = "30s"
|
||||||
|
max_parallel = 1
|
||||||
|
canary = 1
|
||||||
|
min_healthy_time = "10s"
|
||||||
|
healthy_deadline = "5m"
|
||||||
|
auto_revert = true
|
||||||
|
auto_promote = true
|
||||||
|
progress_deadline = "10m"
|
||||||
|
}
|
||||||
|
|
||||||
|
group "app" {
|
||||||
|
count = 1
|
||||||
|
|
||||||
|
network {
|
||||||
|
port "http" {}
|
||||||
|
}
|
||||||
|
|
||||||
|
reschedule {
|
||||||
|
attempts = 5
|
||||||
|
interval = "10m"
|
||||||
|
delay = "30s"
|
||||||
|
delay_function = "exponential"
|
||||||
|
max_delay = "120s"
|
||||||
|
unlimited = false
|
||||||
|
}
|
||||||
|
|
||||||
|
constraint {
|
||||||
|
attribute = "${node.unique.name}"
|
||||||
|
value = "autobox"
|
||||||
|
}
|
||||||
|
|
||||||
|
volume "moneymaker-data" {
|
||||||
|
type = "host"
|
||||||
|
source = "moneymaker-data"
|
||||||
|
read_only = false
|
||||||
|
}
|
||||||
|
|
||||||
|
service {
|
||||||
|
provider = "consul"
|
||||||
|
name = "moneymaker"
|
||||||
|
port = "http"
|
||||||
|
|
||||||
|
tags = [
|
||||||
|
"traefik.enable=true",
|
||||||
|
"traefik.http.routers.moneymaker.rule=Host(`mmd.i80.dk`)",
|
||||||
|
"traefik.http.routers.moneymaker.tls=true",
|
||||||
|
]
|
||||||
|
|
||||||
|
canary_tags = [
|
||||||
|
"traefik.enable=false",
|
||||||
|
]
|
||||||
|
|
||||||
|
check {
|
||||||
|
name = "http_health_check"
|
||||||
|
type = "http"
|
||||||
|
path = "/health"
|
||||||
|
interval = "10s"
|
||||||
|
timeout = "5s"
|
||||||
|
|
||||||
|
check_restart {
|
||||||
|
limit = 3
|
||||||
|
grace = "30s"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# -- Dashboard (Flask web app) ----------------------------------------
|
||||||
|
task "web" {
|
||||||
|
driver = "docker"
|
||||||
|
|
||||||
|
volume_mount {
|
||||||
|
volume = "moneymaker-data"
|
||||||
|
destination = "/app/data"
|
||||||
|
read_only = false
|
||||||
|
}
|
||||||
|
|
||||||
|
config {
|
||||||
|
image = "registry.i80.dk/gitea/mmd:latest"
|
||||||
|
ports = ["http"]
|
||||||
|
force_pull = true
|
||||||
|
}
|
||||||
|
|
||||||
|
restart {
|
||||||
|
attempts = 10
|
||||||
|
interval = "10m"
|
||||||
|
delay = "10s"
|
||||||
|
mode = "fail"
|
||||||
|
}
|
||||||
|
|
||||||
|
env {
|
||||||
|
APP_ENV = "production"
|
||||||
|
PORT = "${NOMAD_PORT_http}"
|
||||||
|
HOST = "0.0.0.0"
|
||||||
|
LOG_DIR = "/app/data/logs"
|
||||||
|
SAXO_TOKEN_FILE = "/app/data/.saxo_token.json"
|
||||||
|
HF_HOME = "/app/data/hf-cache"
|
||||||
|
DATABASE_URL = "__DATABASE_URL__"
|
||||||
|
ANTHROPIC_API_KEY = "__ANTHROPIC_API_KEY__"
|
||||||
|
SAXO_APP_KEY = "__SAXO_APP_KEY__"
|
||||||
|
SAXO_APP_SECRET_1 = "__SAXO_APP_SECRET_1__"
|
||||||
|
}
|
||||||
|
|
||||||
|
resources {
|
||||||
|
cpu = 300
|
||||||
|
memory = 512
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# -- Worker (pipeline scheduler, runs FinBERT + Claude) ----------------
|
||||||
|
task "worker" {
|
||||||
|
driver = "docker"
|
||||||
|
|
||||||
|
volume_mount {
|
||||||
|
volume = "moneymaker-data"
|
||||||
|
destination = "/app/data"
|
||||||
|
read_only = false
|
||||||
|
}
|
||||||
|
|
||||||
|
config {
|
||||||
|
image = "registry.i80.dk/gitea/mmd:latest"
|
||||||
|
command = "python"
|
||||||
|
args = ["scheduler.py"]
|
||||||
|
force_pull = true
|
||||||
|
}
|
||||||
|
|
||||||
|
restart {
|
||||||
|
attempts = 10
|
||||||
|
interval = "10m"
|
||||||
|
delay = "10s"
|
||||||
|
mode = "fail"
|
||||||
|
}
|
||||||
|
|
||||||
|
env {
|
||||||
|
LOG_DIR = "/app/data/logs"
|
||||||
|
SAXO_TOKEN_FILE = "/app/data/.saxo_token.json"
|
||||||
|
HF_HOME = "/app/data/hf-cache"
|
||||||
|
DATABASE_URL = "__DATABASE_URL__"
|
||||||
|
ANTHROPIC_API_KEY = "__ANTHROPIC_API_KEY__"
|
||||||
|
SAXO_APP_KEY = "__SAXO_APP_KEY__"
|
||||||
|
SAXO_APP_SECRET_1 = "__SAXO_APP_SECRET_1__"
|
||||||
|
}
|
||||||
|
|
||||||
|
resources {
|
||||||
|
cpu = 1500
|
||||||
|
memory = 3072
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
10
requirements.txt
Normal file
10
requirements.txt
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
Flask==3.1.3
|
||||||
|
psycopg2-binary==2.9.12
|
||||||
|
python-dotenv==1.2.2
|
||||||
|
requests==2.34.2
|
||||||
|
httpx==0.28.1
|
||||||
|
anthropic==0.104.1
|
||||||
|
yfinance==1.4.0
|
||||||
|
transformers==5.9.0
|
||||||
|
torch==2.12.0
|
||||||
|
beautifulsoup4==4.14.3
|
||||||
@@ -34,7 +34,7 @@ APP_SECRET = os.getenv("SAXO_APP_SECRET_1", "")
|
|||||||
AUTH_URL = os.getenv("SAXO_AUTH_URL", "https://sim.logonvalidation.net/authorize")
|
AUTH_URL = os.getenv("SAXO_AUTH_URL", "https://sim.logonvalidation.net/authorize")
|
||||||
TOKEN_URL = os.getenv("SAXO_TOKEN_URL", "https://sim.logonvalidation.net/token")
|
TOKEN_URL = os.getenv("SAXO_TOKEN_URL", "https://sim.logonvalidation.net/token")
|
||||||
REDIRECT = os.getenv("SAXO_REDIRECT", "http://localhost:8765/callback")
|
REDIRECT = os.getenv("SAXO_REDIRECT", "http://localhost:8765/callback")
|
||||||
TOKEN_FILE = Path(__file__).parent / ".saxo_token.json"
|
TOKEN_FILE = Path(os.getenv("SAXO_TOKEN_FILE", str(Path(__file__).parent / ".saxo_token.json")))
|
||||||
REDIRECT_PORT = 8765
|
REDIRECT_PORT = 8765
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
86
scheduler.py
Normal file
86
scheduler.py
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
"""
|
||||||
|
scheduler.py — MoneyMaker worker daemon.
|
||||||
|
|
||||||
|
Runs the pipeline at scheduled times (Mon-Fri UTC):
|
||||||
|
04:00 analyze-only (NLP before Copenhagen market open 09:00 CET)
|
||||||
|
07:30 trade window 1
|
||||||
|
10:00 trade window 2
|
||||||
|
12:30 trade window 3
|
||||||
|
14:30 trade window 4
|
||||||
|
17:00 daily P&L report
|
||||||
|
|
||||||
|
Outputs appended to LOG_DIR/runner_YYYY-MM-DD.log so the dashboard can
|
||||||
|
show a live log tail. Uses only stdlib — no extra scheduler dependency.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import traceback
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
LOG_DIR = Path(os.getenv("LOG_DIR", str(Path(__file__).parent / "data" / "logs")))
|
||||||
|
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# hour, minute, weekdays (0=Mon…4=Fri), analyze_only, is_report
|
||||||
|
SCHEDULE = [
|
||||||
|
(4, 0, frozenset(range(5)), True, False),
|
||||||
|
(7, 30, frozenset(range(5)), False, False),
|
||||||
|
(10, 0, frozenset(range(5)), False, False),
|
||||||
|
(12, 30, frozenset(range(5)), False, False),
|
||||||
|
(14, 30, frozenset(range(5)), False, False),
|
||||||
|
(17, 0, frozenset(range(5)), False, True),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _log_path() -> Path:
|
||||||
|
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||||
|
return LOG_DIR / f"runner_{today}.log"
|
||||||
|
|
||||||
|
|
||||||
|
def _run_job(analyze_only: bool, is_report: bool) -> None:
|
||||||
|
log = _log_path()
|
||||||
|
with open(log, "a", buffering=1) as fh:
|
||||||
|
old_out, old_err = sys.stdout, sys.stderr
|
||||||
|
sys.stdout = sys.stderr = fh
|
||||||
|
try:
|
||||||
|
if is_report:
|
||||||
|
from report import print_report
|
||||||
|
print_report()
|
||||||
|
else:
|
||||||
|
from runner import run_pipeline
|
||||||
|
run_pipeline(analyze_only=analyze_only)
|
||||||
|
except Exception as exc:
|
||||||
|
print(f"[scheduler] ERROR: {exc}")
|
||||||
|
traceback.print_exc()
|
||||||
|
finally:
|
||||||
|
sys.stdout = old_out
|
||||||
|
sys.stderr = old_err
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
last_run: dict = {}
|
||||||
|
print(f"[scheduler] started — LOG_DIR={LOG_DIR}", flush=True)
|
||||||
|
|
||||||
|
while True:
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
dow = now.weekday()
|
||||||
|
|
||||||
|
for hour, minute, days, analyze_only, is_report in SCHEDULE:
|
||||||
|
if dow not in days:
|
||||||
|
continue
|
||||||
|
key = (hour, minute, now.date())
|
||||||
|
if key in last_run:
|
||||||
|
continue
|
||||||
|
if now.hour == hour and now.minute == minute:
|
||||||
|
last_run[key] = True
|
||||||
|
label = "report" if is_report else ("analyze-only" if analyze_only else "full")
|
||||||
|
print(f"[scheduler] {now.strftime('%H:%M UTC')} — {label}", flush=True)
|
||||||
|
_run_job(analyze_only, is_report)
|
||||||
|
print(f"[scheduler] {label} done", flush=True)
|
||||||
|
|
||||||
|
time.sleep(30)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user