From 8969d829f28f615179c224e3b8133c5f5dbbe1e9 Mon Sep 17 00:00:00 2001
From: Joshua Moody <joshua.moody@rancher.com>
Date: Mon, 15 Jun 2020 16:43:13 -0700
Subject: [PATCH] Add quicker failover for nfs

Add tolerations so that nfs provisioner pod gets evicted from a failing
node after 60 second + 30 grace period (relevant for va recovery policy).
Add liveness + readyness probe, so that no traffic gets routed to a failed
nfs server. Disable device based fsids (major:minor) since our block device
mapping can change from node to node, which makes the id's unstable.

Signed-off-by: Joshua Moody <joshua.moody@rancher.com>
---
 examples/rwx/02-longhorn-nfs-provisioner.yaml | 30 +++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/examples/rwx/02-longhorn-nfs-provisioner.yaml b/examples/rwx/02-longhorn-nfs-provisioner.yaml
index b38bd92..fe6c7ae 100644
--- a/examples/rwx/02-longhorn-nfs-provisioner.yaml
+++ b/examples/rwx/02-longhorn-nfs-provisioner.yaml
@@ -102,6 +102,7 @@ spec:
                 - SYS_RESOURCE
           args:
             - "-provisioner=nfs.longhorn.io"
+            - "-device-based-fsids=false"
           env:
             - name: POD_IP
               valueFrom:
@@ -114,6 +115,20 @@ spec:
                 fieldRef:
                   fieldPath: metadata.namespace
           imagePullPolicy: "IfNotPresent"
+          readinessProbe:
+            exec:
+              command:
+                - ls
+                - /export
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          livenessProbe:
+            exec:
+              command:
+                - ls
+                - /export
+            initialDelaySeconds: 5
+            periodSeconds: 5
           volumeMounts:
             - name: export-volume
               mountPath: /export
@@ -121,6 +136,17 @@ spec:
         - name: export-volume
           persistentVolumeClaim:
             claimName: longhorn-nfs-provisioner
+      # we want really quick failover
+      terminationGracePeriodSeconds: 30
+      tolerations:
+        - effect: NoExecute
+          key: node.kubernetes.io/not-ready
+          operator: Exists
+          tolerationSeconds: 60
+        - effect: NoExecute
+          key: node.kubernetes.io/unreachable
+          operator: Exists
+          tolerationSeconds: 60
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -139,5 +165,5 @@ kind: StorageClass
 metadata:
   name: longhorn-nfs # workload storage class
 provisioner: nfs.longhorn.io
-parameters:
-  mountOptions: "vers=4.1"
+mountOptions:
+  - "vers=4.1"