iObj: initial service — FastAPI + TinyDB memory object store
All checks were successful
Build and Deploy iObj / build-image (push) Successful in 1h17m41s
All checks were successful
Build and Deploy iObj / build-image (push) Successful in 1h17m41s
- Month-sharded TinyDB storage with search/filter (type, project, tag, date range, free-text) - FastAPI app with bearer-token auth, CRUD + search endpoints - 27 passing pytest tests (storage + API) - Dockerfile + Nomad job spec + Gitea Actions deploy workflow (mirrors devops-mcp pattern)
This commit is contained in:
78
.gitea/workflows/deploy.yml
Normal file
78
.gitea/workflows/deploy.yml
Normal file
@@ -0,0 +1,78 @@
|
|||||||
|
name: Build and Deploy iObj
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
env:
|
||||||
|
SERVICE_NAME: iobj
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build-image:
|
||||||
|
runs-on: debian-host
|
||||||
|
|
||||||
|
env:
|
||||||
|
PATH: /usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/sbin:/bin:/snap/bin
|
||||||
|
DOCKER_HOST: unix:///var/run/docker.sock
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: System info
|
||||||
|
run: |
|
||||||
|
uname -a
|
||||||
|
whoami
|
||||||
|
|
||||||
|
- name: Log in to Docker Registry
|
||||||
|
run: |
|
||||||
|
echo "${{ secrets.HARBOR_ROBOT_TOKEN }}" | docker login registry.i80.dk -u "robot\$gitserver" --password-stdin
|
||||||
|
|
||||||
|
- name: Build Docker image
|
||||||
|
run: |
|
||||||
|
SHA=$(git rev-parse --short HEAD)
|
||||||
|
docker build \
|
||||||
|
-t registry.i80.dk/gitea/iobj:latest \
|
||||||
|
-t registry.i80.dk/gitea/iobj:$SHA .
|
||||||
|
|
||||||
|
- name: Push Docker image
|
||||||
|
run: |
|
||||||
|
SHA=$(git rev-parse --short HEAD)
|
||||||
|
docker push registry.i80.dk/gitea/iobj:latest
|
||||||
|
docker push registry.i80.dk/gitea/iobj:$SHA
|
||||||
|
env:
|
||||||
|
PATH: /usr/bin:/usr/local/bin:/bin:/sbin:/usr/sbin
|
||||||
|
|
||||||
|
- name: Deploy to Nomad
|
||||||
|
run: |
|
||||||
|
SHA=$(git rev-parse --short HEAD)
|
||||||
|
nomad job validate -var="image_tag=$SHA" ${SERVICE_NAME}.nomad
|
||||||
|
# Rolling deploy: canary replaces the existing job, keeping old alloc alive
|
||||||
|
# until the new one passes its health check (zero-downtime)
|
||||||
|
nomad job run -var="image_tag=$SHA" ${SERVICE_NAME}.nomad
|
||||||
|
env:
|
||||||
|
NOMAD_ADDR: "https://nomad.i80.dk:4646"
|
||||||
|
|
||||||
|
- name: Wait for deployment
|
||||||
|
run: |
|
||||||
|
echo "Checking deployment status..."
|
||||||
|
nomad job status ${SERVICE_NAME}
|
||||||
|
|
||||||
|
echo "=== Allocation Details ==="
|
||||||
|
nomad job allocs ${SERVICE_NAME}
|
||||||
|
|
||||||
|
echo "=== Getting logs from allocations ==="
|
||||||
|
for alloc in $(nomad job allocs ${SERVICE_NAME} -short | tail -n +2 | awk '{print $1}'); do
|
||||||
|
echo "Logs for allocation $alloc:"
|
||||||
|
nomad alloc logs $alloc || echo "No logs available for $alloc"
|
||||||
|
done
|
||||||
|
env:
|
||||||
|
NOMAD_ADDR: "https://nomad.i80.dk:4646"
|
||||||
|
|
||||||
|
- name: Notify deployment status
|
||||||
|
run: |
|
||||||
|
echo "Deployment completed!"
|
||||||
|
echo "iObj service should be available at: https://${SERVICE_NAME}.i80.dk"
|
||||||
|
echo "Health check endpoint: https://${SERVICE_NAME}.i80.dk/health"
|
||||||
7
.gitignore
vendored
Normal file
7
.gitignore
vendored
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
.venv/
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
data/*.json
|
||||||
|
!data/.gitkeep
|
||||||
|
.pytest_cache/
|
||||||
|
*.egg-info/
|
||||||
39
Dockerfile
Normal file
39
Dockerfile
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
# Stage 1: base image with system deps (rarely changes)
|
||||||
|
FROM python:3.11-slim AS base
|
||||||
|
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||||
|
PYTHONUNBUFFERED=1
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
curl \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
RUN groupadd -r iobj && useradd -r -g iobj -m iobj
|
||||||
|
WORKDIR /app
|
||||||
|
RUN mkdir -p /app/data && chown -R iobj:iobj /app
|
||||||
|
|
||||||
|
# Copy requirements first for pip layer caching
|
||||||
|
COPY requirements.txt ./
|
||||||
|
RUN pip install --no-cache-dir --upgrade pip \
|
||||||
|
&& pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
|
# Copy application code last (rebuilds only when code changes)
|
||||||
|
COPY iobj/ ./iobj/
|
||||||
|
COPY README.md ./
|
||||||
|
|
||||||
|
RUN chown -R iobj:iobj /app
|
||||||
|
USER iobj
|
||||||
|
|
||||||
|
ENV IOBJ_DATA_DIR=/app/data
|
||||||
|
|
||||||
|
# Health check — dynamic port support for Nomad
|
||||||
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||||
|
CMD curl -f http://localhost:${PORT:-8000}/health || exit 1
|
||||||
|
|
||||||
|
EXPOSE 8000
|
||||||
|
|
||||||
|
CMD ["sh", "-c", "uvicorn iobj.app:app --host 0.0.0.0 --port ${PORT:-8000}"]
|
||||||
|
|
||||||
|
LABEL name="iObj" \
|
||||||
|
description="Personal JSON object / memory bank service" \
|
||||||
|
version="0.1.0"
|
||||||
0
data/.gitkeep
Normal file
0
data/.gitkeep
Normal file
148
iobj.nomad
Normal file
148
iobj.nomad
Normal file
@@ -0,0 +1,148 @@
|
|||||||
|
variable "service_name" {
|
||||||
|
description = "Service name for consistent naming"
|
||||||
|
type = string
|
||||||
|
default = "iobj"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "image_tag" {
|
||||||
|
description = "Docker image tag — override in CI with commit SHA: -var=\"image_tag=$SHA\""
|
||||||
|
type = string
|
||||||
|
default = "latest"
|
||||||
|
}
|
||||||
|
|
||||||
|
job "iobj" {
|
||||||
|
region = "global"
|
||||||
|
datacenters = ["dc1"]
|
||||||
|
type = "service"
|
||||||
|
|
||||||
|
meta {
|
||||||
|
uuid = uuidv4()
|
||||||
|
deployed_at = "[[ timeNowUTC ]]"
|
||||||
|
service_name = var.service_name
|
||||||
|
}
|
||||||
|
|
||||||
|
update {
|
||||||
|
stagger = "30s"
|
||||||
|
max_parallel = 1
|
||||||
|
auto_revert = true
|
||||||
|
progress_deadline = "25m"
|
||||||
|
}
|
||||||
|
|
||||||
|
group "iobj-group" {
|
||||||
|
count = 1
|
||||||
|
|
||||||
|
# Deploy specifically on the 'autobox.i80.dk' node (same host as DevOpsMCP)
|
||||||
|
constraint {
|
||||||
|
attribute = "${node.unique.name}"
|
||||||
|
value = "autobox.i80.dk"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Zero-downtime update: canary ensures new alloc is healthy before old one stops
|
||||||
|
update {
|
||||||
|
canary = 1
|
||||||
|
auto_promote = true
|
||||||
|
min_healthy_time = "15s"
|
||||||
|
healthy_deadline = "20m"
|
||||||
|
progress_deadline = "25m"
|
||||||
|
auto_revert = true
|
||||||
|
}
|
||||||
|
|
||||||
|
network {
|
||||||
|
port "http" {}
|
||||||
|
}
|
||||||
|
|
||||||
|
reschedule {
|
||||||
|
attempts = 5
|
||||||
|
interval = "10m"
|
||||||
|
delay = "30s"
|
||||||
|
delay_function = "exponential"
|
||||||
|
max_delay = "120s"
|
||||||
|
unlimited = false
|
||||||
|
}
|
||||||
|
|
||||||
|
volume "iobj-data" {
|
||||||
|
type = "host"
|
||||||
|
source = "iobj-data"
|
||||||
|
read_only = false
|
||||||
|
}
|
||||||
|
|
||||||
|
service {
|
||||||
|
provider = "consul"
|
||||||
|
name = var.service_name
|
||||||
|
port = "http"
|
||||||
|
|
||||||
|
tags = [
|
||||||
|
"traefik.enable=true",
|
||||||
|
"traefik.http.routers.${var.service_name}.rule=Host(`${var.service_name}.i80.dk`)",
|
||||||
|
"traefik.http.routers.${var.service_name}.tls=true",
|
||||||
|
]
|
||||||
|
|
||||||
|
check {
|
||||||
|
name = "http_health_check"
|
||||||
|
type = "http"
|
||||||
|
port = "http"
|
||||||
|
path = "/health"
|
||||||
|
interval = "10s"
|
||||||
|
timeout = "5s"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
task "iobj-task" {
|
||||||
|
driver = "docker"
|
||||||
|
|
||||||
|
config {
|
||||||
|
image = "registry.i80.dk/gitea/iobj:${var.image_tag}"
|
||||||
|
ports = ["http"]
|
||||||
|
force_pull = true
|
||||||
|
auth {
|
||||||
|
username = "robot$gitserver"
|
||||||
|
password = "${HARBOR_ROBOT_TOKEN}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
restart {
|
||||||
|
attempts = 10
|
||||||
|
interval = "10m"
|
||||||
|
delay = "15s"
|
||||||
|
mode = "fail"
|
||||||
|
}
|
||||||
|
|
||||||
|
volume_mount {
|
||||||
|
volume = "iobj-data"
|
||||||
|
destination = "/app/data"
|
||||||
|
read_only = false
|
||||||
|
}
|
||||||
|
|
||||||
|
env {
|
||||||
|
PYTHONUNBUFFERED = "1"
|
||||||
|
PORT = "${NOMAD_PORT_http}"
|
||||||
|
IOBJ_DATA_DIR = "/app/data"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Registry auth + API token, both loaded from Consul KV
|
||||||
|
template {
|
||||||
|
data = <<EOH
|
||||||
|
HARBOR_ROBOT_TOKEN="{{ key "harbor/robot/token" }}"
|
||||||
|
EOH
|
||||||
|
destination = "secrets/registry.env"
|
||||||
|
env = true
|
||||||
|
}
|
||||||
|
|
||||||
|
# NOTE: auth is disabled (open API) until "iobj/api/token" is set in Consul KV.
|
||||||
|
# Set it before going live: consul kv put iobj/api/token "<random-token>"
|
||||||
|
template {
|
||||||
|
data = <<EOH
|
||||||
|
IOBJ_API_TOKEN="{{ keyOrDefault "iobj/api/token" "" }}"
|
||||||
|
EOH
|
||||||
|
destination = "secrets/iobj.env"
|
||||||
|
env = true
|
||||||
|
}
|
||||||
|
|
||||||
|
resources {
|
||||||
|
cpu = 200 # MHz
|
||||||
|
memory = 128 # MB
|
||||||
|
memory_max = 256 # MB
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
3
iobj/__init__.py
Normal file
3
iobj/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
"""iObj — personal JSON object / memory bank service."""
|
||||||
|
|
||||||
|
__version__ = "0.1.0"
|
||||||
106
iobj/app.py
Normal file
106
iobj/app.py
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
"""FastAPI application exposing the iObj memory-object API."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from datetime import date
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from fastapi import Depends, FastAPI, HTTPException, Query, Security, status
|
||||||
|
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||||
|
|
||||||
|
from iobj.models import MemoryObject, ObjectCreate, ObjectType, ObjectUpdate, SearchQuery
|
||||||
|
from iobj.storage import ObjectStore
|
||||||
|
|
||||||
|
logging.basicConfig(
|
||||||
|
level=os.environ.get("IOBJ_LOG_LEVEL", "INFO"),
|
||||||
|
format='{"time":"%(asctime)s","level":"%(levelname)s","msg":%(message)r}',
|
||||||
|
)
|
||||||
|
logger = logging.getLogger("iobj")
|
||||||
|
|
||||||
|
DATA_DIR = os.environ.get("IOBJ_DATA_DIR", "data")
|
||||||
|
API_TOKEN = os.environ.get("IOBJ_API_TOKEN")
|
||||||
|
|
||||||
|
app = FastAPI(title="iObj", version="0.1.0", description="Personal JSON object / memory bank service")
|
||||||
|
store = ObjectStore(DATA_DIR)
|
||||||
|
|
||||||
|
_bearer = HTTPBearer(auto_error=False)
|
||||||
|
|
||||||
|
|
||||||
|
def require_auth(creds: Optional[HTTPAuthorizationCredentials] = Security(_bearer)) -> None:
|
||||||
|
"""Validate the bearer token against IOBJ_API_TOKEN.
|
||||||
|
|
||||||
|
If IOBJ_API_TOKEN is unset, auth is disabled (useful for local dev only —
|
||||||
|
always set it in any networked deployment).
|
||||||
|
"""
|
||||||
|
if not API_TOKEN:
|
||||||
|
return
|
||||||
|
if creds is None or creds.credentials != API_TOKEN:
|
||||||
|
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid or missing bearer token")
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/health")
|
||||||
|
def health() -> dict:
|
||||||
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/objects", response_model=MemoryObject, dependencies=[Depends(require_auth)])
|
||||||
|
def create_object(payload: ObjectCreate) -> MemoryObject:
|
||||||
|
obj = store.create(payload)
|
||||||
|
logger.info("created object id=%s type=%s", obj.id, obj.type)
|
||||||
|
return obj
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/objects", response_model=list[MemoryObject], dependencies=[Depends(require_auth)])
|
||||||
|
def search_objects(
|
||||||
|
type: Optional[ObjectType] = None,
|
||||||
|
project: Optional[str] = None,
|
||||||
|
tag: Optional[str] = None,
|
||||||
|
date_from: Optional[date] = None,
|
||||||
|
date_to: Optional[date] = None,
|
||||||
|
q: Optional[str] = None,
|
||||||
|
limit: int = Query(50, ge=1, le=1000),
|
||||||
|
offset: int = Query(0, ge=0),
|
||||||
|
) -> list[MemoryObject]:
|
||||||
|
query = SearchQuery(
|
||||||
|
type=type,
|
||||||
|
project=project,
|
||||||
|
tag=tag,
|
||||||
|
date_from=date_from,
|
||||||
|
date_to=date_to,
|
||||||
|
q=q,
|
||||||
|
limit=limit,
|
||||||
|
offset=offset,
|
||||||
|
)
|
||||||
|
return store.search(query)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/objects/date/{object_date}", response_model=list[MemoryObject], dependencies=[Depends(require_auth)])
|
||||||
|
def objects_for_date(object_date: date) -> list[MemoryObject]:
|
||||||
|
query = SearchQuery(date_from=object_date, date_to=object_date, limit=1000)
|
||||||
|
return store.search(query)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/objects/{object_id}", response_model=MemoryObject, dependencies=[Depends(require_auth)])
|
||||||
|
def get_object(object_id: str) -> MemoryObject:
|
||||||
|
obj = store.get(object_id)
|
||||||
|
if obj is None:
|
||||||
|
raise HTTPException(status_code=404, detail="Object not found")
|
||||||
|
return obj
|
||||||
|
|
||||||
|
|
||||||
|
@app.patch("/objects/{object_id}", response_model=MemoryObject, dependencies=[Depends(require_auth)])
|
||||||
|
def update_object(object_id: str, payload: ObjectUpdate) -> MemoryObject:
|
||||||
|
obj = store.update(object_id, payload)
|
||||||
|
if obj is None:
|
||||||
|
raise HTTPException(status_code=404, detail="Object not found")
|
||||||
|
logger.info("updated object id=%s", obj.id)
|
||||||
|
return obj
|
||||||
|
|
||||||
|
|
||||||
|
@app.delete("/objects/{object_id}", status_code=status.HTTP_204_NO_CONTENT, dependencies=[Depends(require_auth)])
|
||||||
|
def delete_object(object_id: str) -> None:
|
||||||
|
deleted = store.delete(object_id)
|
||||||
|
if not deleted:
|
||||||
|
raise HTTPException(status_code=404, detail="Object not found")
|
||||||
|
logger.info("deleted object id=%s", object_id)
|
||||||
85
iobj/models.py
Normal file
85
iobj/models.py
Normal file
@@ -0,0 +1,85 @@
|
|||||||
|
"""Data model for iObj memory objects."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from datetime import date, datetime
|
||||||
|
from enum import Enum
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
|
||||||
|
class ObjectType(str, Enum):
|
||||||
|
"""The kind of memory object being stored."""
|
||||||
|
|
||||||
|
GIT_COMMIT = "git_commit"
|
||||||
|
SESSION_LOG = "session_log"
|
||||||
|
RECAP = "recap"
|
||||||
|
MEETING_NOTE = "meeting_note"
|
||||||
|
ONEONONE = "oneonone"
|
||||||
|
STANDUP = "standup"
|
||||||
|
CUSTOM = "custom"
|
||||||
|
|
||||||
|
|
||||||
|
class ObjectCreate(BaseModel):
|
||||||
|
"""Payload for creating a new memory object."""
|
||||||
|
|
||||||
|
type: ObjectType
|
||||||
|
object_date: date = Field(
|
||||||
|
default_factory=lambda: datetime.now().date(),
|
||||||
|
description="The date this object is about (may differ from created_at).",
|
||||||
|
)
|
||||||
|
project: Optional[str] = None
|
||||||
|
tags: list[str] = Field(default_factory=list)
|
||||||
|
title: str
|
||||||
|
body: str = ""
|
||||||
|
payload: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
source_machine: Optional[str] = None
|
||||||
|
source_tool: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
class ObjectUpdate(BaseModel):
|
||||||
|
"""Payload for partially updating an existing memory object. All fields optional."""
|
||||||
|
|
||||||
|
type: Optional[ObjectType] = None
|
||||||
|
object_date: Optional[date] = None
|
||||||
|
project: Optional[str] = None
|
||||||
|
tags: Optional[list[str]] = None
|
||||||
|
title: Optional[str] = None
|
||||||
|
body: Optional[str] = None
|
||||||
|
payload: Optional[dict[str, Any]] = None
|
||||||
|
source_machine: Optional[str] = None
|
||||||
|
source_tool: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
class MemoryObject(BaseModel):
|
||||||
|
"""A fully stored memory object, as returned by the API."""
|
||||||
|
|
||||||
|
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
||||||
|
type: ObjectType
|
||||||
|
created_at: datetime = Field(default_factory=datetime.now)
|
||||||
|
object_date: date
|
||||||
|
project: Optional[str] = None
|
||||||
|
tags: list[str] = Field(default_factory=list)
|
||||||
|
title: str
|
||||||
|
body: str = ""
|
||||||
|
payload: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
source_machine: Optional[str] = None
|
||||||
|
source_tool: Optional[str] = None
|
||||||
|
|
||||||
|
def shard_key(self) -> str:
|
||||||
|
"""The YYYY-MM shard this object belongs to, based on object_date."""
|
||||||
|
return self.object_date.strftime("%Y-%m")
|
||||||
|
|
||||||
|
|
||||||
|
class SearchQuery(BaseModel):
|
||||||
|
"""Search/filter parameters for listing objects."""
|
||||||
|
|
||||||
|
type: Optional[ObjectType] = None
|
||||||
|
project: Optional[str] = None
|
||||||
|
tag: Optional[str] = None
|
||||||
|
date_from: Optional[date] = None
|
||||||
|
date_to: Optional[date] = None
|
||||||
|
q: Optional[str] = None
|
||||||
|
limit: int = 50
|
||||||
|
offset: int = 0
|
||||||
178
iobj/storage.py
Normal file
178
iobj/storage.py
Normal file
@@ -0,0 +1,178 @@
|
|||||||
|
"""Storage layer for iObj: TinyDB shards (one JSON file per YYYY-MM) with
|
||||||
|
simple in-process filtering/search across the shards relevant to a query.
|
||||||
|
|
||||||
|
Sharding by month keeps individual files small and bounded as the object
|
||||||
|
count grows over years, while queries only need to open the shards that can
|
||||||
|
possibly contain matches for a given date range.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
import threading
|
||||||
|
from datetime import date, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from tinydb import Query, TinyDB
|
||||||
|
from tinydb.storages import JSONStorage
|
||||||
|
from tinydb.middlewares import CachingMiddleware
|
||||||
|
|
||||||
|
from iobj.models import MemoryObject, ObjectCreate, ObjectType, ObjectUpdate, SearchQuery
|
||||||
|
|
||||||
|
_MONTH_RE = re.compile(r"^\d{4}-\d{2}$")
|
||||||
|
|
||||||
|
|
||||||
|
def _month_range(d_from: Optional[date], d_to: Optional[date]) -> Optional[list[str]]:
|
||||||
|
"""Return the list of YYYY-MM shard keys spanning [d_from, d_to], or None
|
||||||
|
if no bound was given (meaning: caller should scan all shards)."""
|
||||||
|
if d_from is None and d_to is None:
|
||||||
|
return None
|
||||||
|
start = d_from or date(2000, 1, 1)
|
||||||
|
end = d_to or date(2100, 1, 1)
|
||||||
|
months = []
|
||||||
|
y, m = start.year, start.month
|
||||||
|
while (y, m) <= (end.year, end.month):
|
||||||
|
months.append(f"{y:04d}-{m:02d}")
|
||||||
|
m += 1
|
||||||
|
if m > 12:
|
||||||
|
m = 1
|
||||||
|
y += 1
|
||||||
|
return months
|
||||||
|
|
||||||
|
|
||||||
|
class ObjectStore:
|
||||||
|
"""Thread-safe TinyDB-backed store for MemoryObjects, sharded by month."""
|
||||||
|
|
||||||
|
def __init__(self, data_dir: str | Path):
|
||||||
|
self.data_dir = Path(data_dir)
|
||||||
|
self.data_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
self._dbs: dict[str, TinyDB] = {}
|
||||||
|
|
||||||
|
def _shard_path(self, shard_key: str) -> Path:
|
||||||
|
return self.data_dir / f"{shard_key}.json"
|
||||||
|
|
||||||
|
def _db_for(self, shard_key: str) -> TinyDB:
|
||||||
|
with self._lock:
|
||||||
|
db = self._dbs.get(shard_key)
|
||||||
|
if db is None:
|
||||||
|
db = TinyDB(
|
||||||
|
self._shard_path(shard_key),
|
||||||
|
storage=CachingMiddleware(JSONStorage),
|
||||||
|
)
|
||||||
|
self._dbs[shard_key] = db
|
||||||
|
return db
|
||||||
|
|
||||||
|
def _existing_shard_keys(self) -> list[str]:
|
||||||
|
keys = []
|
||||||
|
for p in self.data_dir.glob("*.json"):
|
||||||
|
stem = p.stem
|
||||||
|
if _MONTH_RE.match(stem):
|
||||||
|
keys.append(stem)
|
||||||
|
return sorted(keys)
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
with self._lock:
|
||||||
|
for db in self._dbs.values():
|
||||||
|
db.close()
|
||||||
|
self._dbs.clear()
|
||||||
|
|
||||||
|
# --- CRUD -------------------------------------------------------------
|
||||||
|
|
||||||
|
def create(self, payload: ObjectCreate) -> MemoryObject:
|
||||||
|
obj = MemoryObject(**payload.model_dump())
|
||||||
|
db = self._db_for(obj.shard_key())
|
||||||
|
with self._lock:
|
||||||
|
db.insert(_to_doc(obj))
|
||||||
|
return obj
|
||||||
|
|
||||||
|
def get(self, object_id: str) -> Optional[MemoryObject]:
|
||||||
|
Obj = Query()
|
||||||
|
with self._lock:
|
||||||
|
for shard_key in self._existing_shard_keys():
|
||||||
|
db = self._db_for(shard_key)
|
||||||
|
doc = db.get(Obj.id == object_id)
|
||||||
|
if doc is not None:
|
||||||
|
return _from_doc(doc)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def update(self, object_id: str, payload: ObjectUpdate) -> Optional[MemoryObject]:
|
||||||
|
existing = self.get(object_id)
|
||||||
|
if existing is None:
|
||||||
|
return None
|
||||||
|
changes = payload.model_dump(exclude_unset=True)
|
||||||
|
merged = existing.model_dump()
|
||||||
|
merged.update(changes)
|
||||||
|
updated = MemoryObject(**merged)
|
||||||
|
|
||||||
|
old_shard = existing.shard_key()
|
||||||
|
new_shard = updated.shard_key()
|
||||||
|
Obj = Query()
|
||||||
|
with self._lock:
|
||||||
|
if old_shard == new_shard:
|
||||||
|
self._db_for(old_shard).update(_to_doc(updated), Obj.id == object_id)
|
||||||
|
else:
|
||||||
|
# object_date changed enough to move shards: remove + reinsert
|
||||||
|
self._db_for(old_shard).remove(Obj.id == object_id)
|
||||||
|
self._db_for(new_shard).insert(_to_doc(updated))
|
||||||
|
return updated
|
||||||
|
|
||||||
|
def delete(self, object_id: str) -> bool:
|
||||||
|
Obj = Query()
|
||||||
|
with self._lock:
|
||||||
|
for shard_key in self._existing_shard_keys():
|
||||||
|
db = self._db_for(shard_key)
|
||||||
|
if db.contains(Obj.id == object_id):
|
||||||
|
db.remove(Obj.id == object_id)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
# --- Search -------------------------------------------------------------
|
||||||
|
|
||||||
|
def search(self, query: SearchQuery) -> list[MemoryObject]:
|
||||||
|
shard_keys = _month_range(query.date_from, query.date_to)
|
||||||
|
if shard_keys is None:
|
||||||
|
shard_keys = self._existing_shard_keys()
|
||||||
|
else:
|
||||||
|
existing = set(self._existing_shard_keys())
|
||||||
|
shard_keys = [k for k in shard_keys if k in existing]
|
||||||
|
|
||||||
|
results: list[MemoryObject] = []
|
||||||
|
with self._lock:
|
||||||
|
for shard_key in shard_keys:
|
||||||
|
db = self._db_for(shard_key)
|
||||||
|
for doc in db.all():
|
||||||
|
obj = _from_doc(doc)
|
||||||
|
if _matches(obj, query):
|
||||||
|
results.append(obj)
|
||||||
|
|
||||||
|
results.sort(key=lambda o: o.created_at, reverse=True)
|
||||||
|
return results[query.offset : query.offset + query.limit]
|
||||||
|
|
||||||
|
|
||||||
|
def _to_doc(obj: MemoryObject) -> dict:
|
||||||
|
doc = obj.model_dump(mode="json")
|
||||||
|
return doc
|
||||||
|
|
||||||
|
|
||||||
|
def _from_doc(doc: dict) -> MemoryObject:
|
||||||
|
return MemoryObject(**doc)
|
||||||
|
|
||||||
|
|
||||||
|
def _matches(obj: MemoryObject, query: SearchQuery) -> bool:
|
||||||
|
if query.type is not None and obj.type != query.type:
|
||||||
|
return False
|
||||||
|
if query.project is not None and (obj.project or "").lower() != query.project.lower():
|
||||||
|
return False
|
||||||
|
if query.tag is not None and query.tag not in (obj.tags or []):
|
||||||
|
return False
|
||||||
|
if query.date_from is not None and obj.object_date < query.date_from:
|
||||||
|
return False
|
||||||
|
if query.date_to is not None and obj.object_date > query.date_to:
|
||||||
|
return False
|
||||||
|
if query.q:
|
||||||
|
needle = query.q.lower()
|
||||||
|
haystack = f"{obj.title}\n{obj.body}".lower()
|
||||||
|
if needle not in haystack:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
2
requirements-dev.txt
Normal file
2
requirements-dev.txt
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
pytest>=8.0
|
||||||
|
httpx>=0.27
|
||||||
4
requirements.txt
Normal file
4
requirements.txt
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
fastapi>=0.110
|
||||||
|
uvicorn[standard]>=0.29
|
||||||
|
tinydb>=4.8
|
||||||
|
pydantic>=2.6
|
||||||
135
tests/test_api.py
Normal file
135
tests/test_api.py
Normal file
@@ -0,0 +1,135 @@
|
|||||||
|
import tempfile
|
||||||
|
from datetime import date
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
import iobj.app as app_module
|
||||||
|
from iobj.storage import ObjectStore
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def client(monkeypatch):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
monkeypatch.setattr(app_module, "store", ObjectStore(tmp))
|
||||||
|
monkeypatch.setattr(app_module, "API_TOKEN", "secret-token")
|
||||||
|
with TestClient(app_module.app) as c:
|
||||||
|
yield c
|
||||||
|
|
||||||
|
|
||||||
|
AUTH = {"Authorization": "Bearer secret-token"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_health_no_auth_required(client):
|
||||||
|
resp = client.get("/health")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert resp.json() == {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_requires_auth(client):
|
||||||
|
resp = client.post("/objects", json={"type": "recap", "title": "x"})
|
||||||
|
assert resp.status_code == 401
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_with_wrong_token_rejected(client):
|
||||||
|
resp = client.post(
|
||||||
|
"/objects",
|
||||||
|
json={"type": "recap", "title": "x"},
|
||||||
|
headers={"Authorization": "Bearer wrong"},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 401
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_and_get_object(client):
|
||||||
|
resp = client.post(
|
||||||
|
"/objects",
|
||||||
|
json={
|
||||||
|
"type": "recap",
|
||||||
|
"object_date": "2026-07-01",
|
||||||
|
"project": "NordBytes",
|
||||||
|
"tags": ["frontend"],
|
||||||
|
"title": "Test",
|
||||||
|
"body": "body text",
|
||||||
|
},
|
||||||
|
headers=AUTH,
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
obj = resp.json()
|
||||||
|
assert obj["title"] == "Test"
|
||||||
|
obj_id = obj["id"]
|
||||||
|
|
||||||
|
resp = client.get(f"/objects/{obj_id}", headers=AUTH)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert resp.json()["id"] == obj_id
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_missing_object_404(client):
|
||||||
|
resp = client.get("/objects/does-not-exist", headers=AUTH)
|
||||||
|
assert resp.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_objects(client):
|
||||||
|
client.post(
|
||||||
|
"/objects",
|
||||||
|
json={"type": "git_commit", "object_date": "2026-05-01", "title": "commit a", "project": "Egmont"},
|
||||||
|
headers=AUTH,
|
||||||
|
)
|
||||||
|
client.post(
|
||||||
|
"/objects",
|
||||||
|
json={"type": "recap", "object_date": "2026-05-01", "title": "recap a", "project": "NordBytes"},
|
||||||
|
headers=AUTH,
|
||||||
|
)
|
||||||
|
resp = client.get("/objects", params={"type": "git_commit"}, headers=AUTH)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
results = resp.json()
|
||||||
|
assert len(results) == 1
|
||||||
|
assert results[0]["title"] == "commit a"
|
||||||
|
|
||||||
|
|
||||||
|
def test_objects_for_date(client):
|
||||||
|
client.post(
|
||||||
|
"/objects",
|
||||||
|
json={"type": "standup", "object_date": "2026-05-05", "title": "standup"},
|
||||||
|
headers=AUTH,
|
||||||
|
)
|
||||||
|
client.post(
|
||||||
|
"/objects",
|
||||||
|
json={"type": "standup", "object_date": "2026-05-06", "title": "other day"},
|
||||||
|
headers=AUTH,
|
||||||
|
)
|
||||||
|
resp = client.get("/objects/date/2026-05-05", headers=AUTH)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
results = resp.json()
|
||||||
|
assert len(results) == 1
|
||||||
|
assert results[0]["title"] == "standup"
|
||||||
|
|
||||||
|
|
||||||
|
def test_update_object(client):
|
||||||
|
resp = client.post(
|
||||||
|
"/objects", json={"type": "recap", "object_date": "2026-05-05", "title": "orig"}, headers=AUTH
|
||||||
|
)
|
||||||
|
obj_id = resp.json()["id"]
|
||||||
|
resp = client.patch(f"/objects/{obj_id}", json={"title": "changed"}, headers=AUTH)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert resp.json()["title"] == "changed"
|
||||||
|
|
||||||
|
|
||||||
|
def test_update_missing_object_404(client):
|
||||||
|
resp = client.patch("/objects/nope", json={"title": "x"}, headers=AUTH)
|
||||||
|
assert resp.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete_object(client):
|
||||||
|
resp = client.post(
|
||||||
|
"/objects", json={"type": "recap", "object_date": "2026-05-05", "title": "to-delete"}, headers=AUTH
|
||||||
|
)
|
||||||
|
obj_id = resp.json()["id"]
|
||||||
|
resp = client.delete(f"/objects/{obj_id}", headers=AUTH)
|
||||||
|
assert resp.status_code == 204
|
||||||
|
resp = client.get(f"/objects/{obj_id}", headers=AUTH)
|
||||||
|
assert resp.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete_missing_object_404(client):
|
||||||
|
resp = client.delete("/objects/nope", headers=AUTH)
|
||||||
|
assert resp.status_code == 404
|
||||||
142
tests/test_storage.py
Normal file
142
tests/test_storage.py
Normal file
@@ -0,0 +1,142 @@
|
|||||||
|
import tempfile
|
||||||
|
from datetime import date
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from iobj.models import ObjectCreate, ObjectType, ObjectUpdate, SearchQuery
|
||||||
|
from iobj.storage import ObjectStore
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def store():
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
s = ObjectStore(tmp)
|
||||||
|
yield s
|
||||||
|
s.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _mk(**kwargs) -> ObjectCreate:
|
||||||
|
defaults = dict(
|
||||||
|
type=ObjectType.RECAP,
|
||||||
|
object_date=date(2026, 7, 1),
|
||||||
|
project="NordBytes",
|
||||||
|
tags=["frontend"],
|
||||||
|
title="Test title",
|
||||||
|
body="Some body text about BDD tests",
|
||||||
|
)
|
||||||
|
defaults.update(kwargs)
|
||||||
|
return ObjectCreate(**defaults)
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_and_get_roundtrip(store):
|
||||||
|
obj = store.create(_mk())
|
||||||
|
fetched = store.get(obj.id)
|
||||||
|
assert fetched is not None
|
||||||
|
assert fetched.id == obj.id
|
||||||
|
assert fetched.title == "Test title"
|
||||||
|
assert fetched.project == "NordBytes"
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_missing_returns_none(store):
|
||||||
|
assert store.get("does-not-exist") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_shard_file_created_per_month(store):
|
||||||
|
store.create(_mk(object_date=date(2026, 1, 15)))
|
||||||
|
store.create(_mk(object_date=date(2026, 3, 2)))
|
||||||
|
shard_files = sorted(p.name for p in store.data_dir.glob("*.json"))
|
||||||
|
assert shard_files == ["2026-01.json", "2026-03.json"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_update_changes_fields(store):
|
||||||
|
obj = store.create(_mk())
|
||||||
|
updated = store.update(obj.id, ObjectUpdate(title="New title", tags=["backend"]))
|
||||||
|
assert updated.title == "New title"
|
||||||
|
assert updated.tags == ["backend"]
|
||||||
|
assert updated.body == obj.body # untouched field preserved
|
||||||
|
|
||||||
|
|
||||||
|
def test_update_moves_shard_when_date_changes(store):
|
||||||
|
obj = store.create(_mk(object_date=date(2026, 1, 1)))
|
||||||
|
store.update(obj.id, ObjectUpdate(object_date=date(2026, 6, 1)))
|
||||||
|
shard_files = sorted(p.name for p in store.data_dir.glob("*.json"))
|
||||||
|
assert "2026-06.json" in shard_files
|
||||||
|
moved = store.get(obj.id)
|
||||||
|
assert moved.object_date == date(2026, 6, 1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_update_missing_returns_none(store):
|
||||||
|
assert store.update("nope", ObjectUpdate(title="x")) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete_removes_object(store):
|
||||||
|
obj = store.create(_mk())
|
||||||
|
assert store.delete(obj.id) is True
|
||||||
|
assert store.get(obj.id) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete_missing_returns_false(store):
|
||||||
|
assert store.delete("nope") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_filters_by_type(store):
|
||||||
|
store.create(_mk(type=ObjectType.RECAP))
|
||||||
|
store.create(_mk(type=ObjectType.GIT_COMMIT, title="commit one"))
|
||||||
|
results = store.search(SearchQuery(type=ObjectType.GIT_COMMIT))
|
||||||
|
assert len(results) == 1
|
||||||
|
assert results[0].title == "commit one"
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_filters_by_project_case_insensitive(store):
|
||||||
|
store.create(_mk(project="NordBytes"))
|
||||||
|
store.create(_mk(project="Egmont"))
|
||||||
|
results = store.search(SearchQuery(project="nordbytes"))
|
||||||
|
assert len(results) == 1
|
||||||
|
assert results[0].project == "NordBytes"
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_filters_by_tag(store):
|
||||||
|
store.create(_mk(tags=["frontend", "bdd"]))
|
||||||
|
store.create(_mk(tags=["backend"]))
|
||||||
|
results = store.search(SearchQuery(tag="bdd"))
|
||||||
|
assert len(results) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_filters_by_date_range(store):
|
||||||
|
store.create(_mk(object_date=date(2026, 1, 5), title="jan"))
|
||||||
|
store.create(_mk(object_date=date(2026, 6, 5), title="jun"))
|
||||||
|
store.create(_mk(object_date=date(2026, 12, 5), title="dec"))
|
||||||
|
results = store.search(SearchQuery(date_from=date(2026, 2, 1), date_to=date(2026, 7, 1)))
|
||||||
|
assert len(results) == 1
|
||||||
|
assert results[0].title == "jun"
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_free_text_matches_title_and_body(store):
|
||||||
|
store.create(_mk(title="BDD patterns", body="irrelevant"))
|
||||||
|
store.create(_mk(title="irrelevant", body="mentions BDD somewhere"))
|
||||||
|
store.create(_mk(title="nothing", body="nothing"))
|
||||||
|
results = store.search(SearchQuery(q="bdd"))
|
||||||
|
assert len(results) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_respects_limit_and_offset_and_sort_order(store):
|
||||||
|
for i in range(5):
|
||||||
|
store.create(_mk(object_date=date(2026, 1, 1 + i), title=f"obj{i}"))
|
||||||
|
all_results = store.search(SearchQuery(limit=100))
|
||||||
|
assert [r.title for r in all_results] == ["obj4", "obj3", "obj2", "obj1", "obj0"]
|
||||||
|
|
||||||
|
page = store.search(SearchQuery(limit=2, offset=1))
|
||||||
|
assert [r.title for r in page] == ["obj3", "obj2"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_combines_multiple_filters(store):
|
||||||
|
store.create(_mk(project="NordBytes", tags=["bdd"], title="match", object_date=date(2026, 5, 5)))
|
||||||
|
store.create(_mk(project="NordBytes", tags=["other"], title="no-tag-match", object_date=date(2026, 5, 5)))
|
||||||
|
store.create(_mk(project="Egmont", tags=["bdd"], title="wrong-project", object_date=date(2026, 5, 5)))
|
||||||
|
results = store.search(SearchQuery(project="NordBytes", tag="bdd"))
|
||||||
|
assert len(results) == 1
|
||||||
|
assert results[0].title == "match"
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_no_shards_returns_empty(store):
|
||||||
|
assert store.search(SearchQuery()) == []
|
||||||
Reference in New Issue
Block a user