From 53d696ea732894f083965b533c07333c418503b1 Mon Sep 17 00:00:00 2001
From: Henrik Jess Nielsen <lrihni@egmont.com>
Date: Thu, 6 Aug 2026 23:22:28 +0200
Subject: [PATCH] fix(csi): survive int reboots, pin image by digest, raise
 memory
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Both plugin jobs are pinned to int with count = 1 and had no restart or
reschedule stanza. When int goes away the allocations are marked Lost and
never return — csi-nfs-node showed 1 Complete / 3 Lost / 0 Failed, so these
were never application crashes, they were the node disappearing. The plugin
has been down since 2026-06-08 as a result. Adds restart + unlimited
reschedule with exponential backoff so they recover on their own.

The image was :latest, against our own rule for Nomad. The copy cached on
int is sha256:944d0e65 and roughly seven months old, so any fresh pull on
another node would get a different build — a good way to make failures
irreproducible. Pinned to the digest that has actually been running.

Memory was 128 MB for a Node.js process. Raised to 256 with memory_max so
mount and provisioning activity can burst without permanently reserving it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 csi/controller.nomad | 26 +++++++++++++++++++++++---
 csi/node.nomad       | 31 ++++++++++++++++++++++++++++---
 2 files changed, 51 insertions(+), 6 deletions(-)

diff --git a/csi/controller.nomad b/csi/controller.nomad
index af4a114..656268e 100644
--- a/csi/controller.nomad
+++ b/csi/controller.nomad
@@ -12,11 +12,28 @@ job "csi-nfs-controller" {
   group "controller" {
     count = 1
 
+    # Pinned to int, so an int reboot takes the controller down with it.
+    # Without these the allocation is marked Lost and never returns.
+    restart {
+      attempts = 5
+      interval = "10m"
+      delay    = "30s"
+      mode     = "delay"
+    }
+
+    reschedule {
+      unlimited      = true
+      delay          = "30s"
+      delay_function = "exponential"
+      max_delay      = "5m"
+    }
+
     task "plugin" {
       driver = "docker"
 
       config {
-        image   = "democraticcsi/democratic-csi:latest"
+        # Pinned by digest rather than :latest — see node.nomad for why.
+        image   = "democraticcsi/democratic-csi@sha256:944d0e65077efbd9c1fdf23997eec8fac4b4bfb7c3de400f63e33a0a849c5ced"
         command = "/bin/democratic-csi"
         args = [
           "--csi-version=1.5.0",
@@ -60,9 +77,12 @@ TMPL
         destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml"
       }
 
+      # The controller does the provisioning work (volume create, NFS export
+      # management), so it gets more headroom than the node plugin.
       resources {
-        cpu    = 100
-        memory = 128
+        cpu        = 100
+        memory     = 256
+        memory_max = 768
       }
     }
   }
diff --git a/csi/node.nomad b/csi/node.nomad
index a6ad8f2..68b02ff 100644
--- a/csi/node.nomad
+++ b/csi/node.nomad
@@ -12,11 +12,32 @@ job "csi-nfs-node" {
   group "node" {
     count = 1
 
+    # int is both the NFS server and the only node running this plugin, so a
+    # reboot there takes it down. Without these the allocation is marked Lost
+    # and never comes back on its own — which is how it stayed dead from
+    # 2026-06-08 until someone noticed.
+    restart {
+      attempts = 5
+      interval = "10m"
+      delay    = "30s"
+      mode     = "delay"
+    }
+
+    reschedule {
+      unlimited      = true
+      delay          = "30s"
+      delay_function = "exponential"
+      max_delay      = "5m"
+    }
+
     task "plugin" {
       driver = "docker"
 
       config {
-        image        = "democraticcsi/democratic-csi:latest"
+        # Pinned by digest rather than :latest. The tag moves upstream, so a
+        # fresh pull on a new node gets a different build than the one cached
+        # on int — this digest is what has actually been running there.
+        image        = "democraticcsi/democratic-csi@sha256:944d0e65077efbd9c1fdf23997eec8fac4b4bfb7c3de400f63e33a0a849c5ced"
         command      = "/bin/democratic-csi"
         network_mode = "host"
         args = [
@@ -60,9 +81,13 @@ TMPL
         destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml"
       }
 
+      # democratic-csi is a Node.js process; 128 MB is tight enough that mount
+      # activity can push it over. memory_max lets it burst without reserving
+      # the headroom permanently.
       resources {
-        cpu    = 100
-        memory = 128
+        cpu        = 100
+        memory     = 256
+        memory_max = 512
       }
     }
   }