diff --git a/csi/controller.nomad b/csi/controller.nomad index af4a114..656268e 100644 --- a/csi/controller.nomad +++ b/csi/controller.nomad @@ -12,11 +12,28 @@ job "csi-nfs-controller" { group "controller" { count = 1 + # Pinned to int, so an int reboot takes the controller down with it. + # Without these the allocation is marked Lost and never returns. + restart { + attempts = 5 + interval = "10m" + delay = "30s" + mode = "delay" + } + + reschedule { + unlimited = true + delay = "30s" + delay_function = "exponential" + max_delay = "5m" + } + task "plugin" { driver = "docker" config { - image = "democraticcsi/democratic-csi:latest" + # Pinned by digest rather than :latest — see node.nomad for why. + image = "democraticcsi/democratic-csi@sha256:944d0e65077efbd9c1fdf23997eec8fac4b4bfb7c3de400f63e33a0a849c5ced" command = "/bin/democratic-csi" args = [ "--csi-version=1.5.0", @@ -60,9 +77,12 @@ TMPL destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml" } + # The controller does the provisioning work (volume create, NFS export + # management), so it gets more headroom than the node plugin. resources { - cpu = 100 - memory = 128 + cpu = 100 + memory = 256 + memory_max = 768 } } } diff --git a/csi/node.nomad b/csi/node.nomad index a6ad8f2..68b02ff 100644 --- a/csi/node.nomad +++ b/csi/node.nomad @@ -12,11 +12,32 @@ job "csi-nfs-node" { group "node" { count = 1 + # int is both the NFS server and the only node running this plugin, so a + # reboot there takes it down. Without these the allocation is marked Lost + # and never comes back on its own — which is how it stayed dead from + # 2026-06-08 until someone noticed. + restart { + attempts = 5 + interval = "10m" + delay = "30s" + mode = "delay" + } + + reschedule { + unlimited = true + delay = "30s" + delay_function = "exponential" + max_delay = "5m" + } + task "plugin" { driver = "docker" config { - image = "democraticcsi/democratic-csi:latest" + # Pinned by digest rather than :latest. The tag moves upstream, so a + # fresh pull on a new node gets a different build than the one cached + # on int — this digest is what has actually been running there. + image = "democraticcsi/democratic-csi@sha256:944d0e65077efbd9c1fdf23997eec8fac4b4bfb7c3de400f63e33a0a849c5ced" command = "/bin/democratic-csi" network_mode = "host" args = [ @@ -60,9 +81,13 @@ TMPL destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml" } + # democratic-csi is a Node.js process; 128 MB is tight enough that mount + # activity can push it over. memory_max lets it burst without reserving + # the headroom permanently. resources { - cpu = 100 - memory = 128 + cpu = 100 + memory = 256 + memory_max = 512 } } }