fix: correct the version of CSI-attacher and CSI-nodeDriverRegistrar

Signed-off-by: Ray Chang <ray.chang@suse.com>

chart/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 name: longhorn
-version: 1.2.3
-appVersion: v1.2.3
-kubeVersion: ">=1.18.0-0"
+version: 1.2.6
+appVersion: v1.2.6
+kubeVersion: ">=1.18.0-0 <1.25.0-0"
 description: Longhorn is a distributed block storage system for Kubernetes.
 keywords:
 - longhorn

chart/questions.yml

@@ -17,7 +17,7 @@ questions:
     label: Longhorn Manager Image Repository
     group: "Longhorn Images Settings"
   - variable: image.longhorn.manager.tag
-    default: v1.2.3
+    default: v1.2.6
     description: "Specify Longhorn Manager Image Tag"
     type: string
     label: Longhorn Manager Image Tag
@@ -29,7 +29,7 @@ questions:
     label: Longhorn Engine Image Repository
     group: "Longhorn Images Settings"
   - variable: image.longhorn.engine.tag
-    default: v1.2.3
+    default: v1.2.6
     description: "Specify Longhorn Engine Image Tag"
     type: string
     label: Longhorn Engine Image Tag
@@ -41,7 +41,7 @@ questions:
     label: Longhorn UI Image Repository
     group: "Longhorn Images Settings"
   - variable: image.longhorn.ui.tag
-    default: v1.2.3
+    default: v1.2.6
     description: "Specify Longhorn UI Image Tag"
     type: string
     label: Longhorn UI Image Tag
@@ -53,7 +53,7 @@ questions:
     label: Longhorn Instance Manager Image Repository
     group: "Longhorn Images Settings"
   - variable: image.longhorn.instanceManager.tag
-    default: v1_20211210
+    default: v1_20220303_patch2
     description: "Specify Longhorn Instance Manager Image Tag"
     type: string
     label: Longhorn Instance Manager Image Tag
@@ -65,7 +65,7 @@ questions:
     label: Longhorn Share Manager Image Repository
     group: "Longhorn Images Settings"
   - variable: image.longhorn.shareManager.tag
-    default: v1_20211020
+    default: v1_20211020_patch2
     description: "Specify Longhorn Share Manager Image Tag"
     type: string
     label: Longhorn Share Manager Image Tag
@@ -77,7 +77,7 @@ questions:
     label: Longhorn Backing Image Manager Image Repository
     group: "Longhorn Images Settings"
   - variable: image.longhorn.backingImageManager.tag
-    default: v2_20210820
+    default: v2_20221027
     description: "Specify Longhorn Backing Image Manager Image Tag"
     type: string
     label: Longhorn Backing Image Manager Image Tag
@@ -324,6 +324,16 @@ The available volume spec options are:
     type: int
     min: 0
     default: 300
+  - variable: defaultSettings.failedBackupTTL
+    label: Failed Backup Time to Live
+    description: "In minutes. This setting determines how long Longhorn will keep the backup resource that was failed. Set to 0 to disable the auto-deletion.
+Failed backups will be checked and cleaned up during backupstore polling which is controlled by **Backupstore Poll Interval** setting.
+Hence this value determines the minimal wait interval of the cleanup. And the actual cleanup interval is multiple of **Backupstore Poll Interval**.
+Disabling **Backupstore Poll Interval** also means to disable failed backup auto-deletion."
+    group: "Longhorn Default Settings"
+    type: int
+    min: 0
+    default: 1440
   - variable: defaultSettings.autoSalvage
     label: Automatic salvage
     description: "If enabled, volumes will be automatically salvaged when all the replicas become faulty e.g. due to network disconnection. Longhorn will try to figure out which replica(s) are usable, then use them for the volume. By default true."
@@ -621,3 +631,9 @@ WARNING:
   label: Pod Security Policy
   type: boolean
   group: "Other Settings"
+- variable: global.cattle.windowsCluster.enabled
+  default: "false"
+  description: "Enable this to allow Longhorn to run on the Rancher deployed Windows cluster."
+  label: Rancher Windows Cluster
+  type: boolean
+  group: "Other Settings"

chart/templates/crds.yaml

@@ -232,7 +232,7 @@ spec:
       jsonPath: .spec.image
     - name: RefCount
       type: integer
-      description: Number of volumes are using the engine image
+      description: Number of resources using the engine image
       jsonPath: .status.refCount
     - name: BuildDate
       type: date

chart/templates/daemonset-sa.yaml

@@ -54,8 +54,6 @@ spec:
         - name: longhorn
           mountPath: /var/lib/longhorn/
           mountPropagation: Bidirectional
-        - name: longhorn-default-setting
-          mountPath: /var/lib/longhorn-setting/
         env:
         - name: POD_NAMESPACE
           valueFrom:
@@ -69,8 +67,6 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
-        - name: DEFAULT_SETTING_PATH
-          value: /var/lib/longhorn-setting/default-setting.yaml
       volumes:
       - name: dev
         hostPath:
@@ -81,23 +77,30 @@ spec:
       - name: longhorn
         hostPath:
           path: /var/lib/longhorn/
-      - name: longhorn-default-setting
-        configMap:
-          name: longhorn-default-setting
       {{- if .Values.privateRegistry.registrySecret }}
       imagePullSecrets:
       - name: {{ .Values.privateRegistry.registrySecret }}
       {{- end }}
       {{- if .Values.longhornManager.priorityClass }}
-      priorityClassName: {{ .Values.longhornManager.priorityClass | quote}}
+      priorityClassName: {{ .Values.longhornManager.priorityClass | quote }}
       {{- end }}
-      {{- if .Values.longhornManager.tolerations }}
+      {{- if or .Values.longhornManager.tolerations .Values.global.cattle.windowsCluster.enabled }}
       tolerations:
+        {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }}
+{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }}
+        {{- end }}
+        {{- if .Values.longhornManager.tolerations }}
 {{ toYaml .Values.longhornManager.tolerations | indent 6 }}
+        {{- end }}
       {{- end }}
-      {{- if .Values.longhornManager.nodeSelector }}
+      {{- if or .Values.longhornManager.nodeSelector .Values.global.cattle.windowsCluster.enabled }}
       nodeSelector:
+        {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }}
+{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }}
+        {{- end }}
+        {{- if .Values.longhornManager.nodeSelector }}
 {{ toYaml .Values.longhornManager.nodeSelector | indent 8 }}
+        {{- end }}
       {{- end }}
       serviceAccountName: longhorn-service-account
   updateStrategy:

chart/templates/default-setting.yaml

@@ -6,39 +6,58 @@ metadata:
   labels: {{- include "longhorn.labels" . | nindent 4 }}
 data:
   default-setting.yaml: |-
-    backup-target: {{ .Values.defaultSettings.backupTarget }}
-    backup-target-credential-secret: {{ .Values.defaultSettings.backupTargetCredentialSecret }}
-    allow-recurring-job-while-volume-detached: {{ .Values.defaultSettings.allowRecurringJobWhileVolumeDetached }}
-    create-default-disk-labeled-nodes: {{ .Values.defaultSettings.createDefaultDiskLabeledNodes }}
-    default-data-path: {{ .Values.defaultSettings.defaultDataPath }}
-    replica-soft-anti-affinity: {{ .Values.defaultSettings.replicaSoftAntiAffinity }}
-    replica-auto-balance: {{ .Values.defaultSettings.replicaAutoBalance }}
-    storage-over-provisioning-percentage: {{ .Values.defaultSettings.storageOverProvisioningPercentage }}
-    storage-minimal-available-percentage: {{ .Values.defaultSettings.storageMinimalAvailablePercentage }}
-    upgrade-checker: {{ .Values.defaultSettings.upgradeChecker }}
-    default-replica-count: {{ .Values.defaultSettings.defaultReplicaCount }}
-    default-data-locality: {{ .Values.defaultSettings.defaultDataLocality }}
-    default-longhorn-static-storage-class: {{ .Values.defaultSettings.defaultLonghornStaticStorageClass }}
-    backupstore-poll-interval: {{ .Values.defaultSettings.backupstorePollInterval }}
-    taint-toleration: {{ .Values.defaultSettings.taintToleration }}
-    system-managed-components-node-selector: {{ .Values.defaultSettings.systemManagedComponentsNodeSelector }}
-    priority-class: {{ .Values.defaultSettings.priorityClass }}
-    auto-salvage: {{ .Values.defaultSettings.autoSalvage }}
-    auto-delete-pod-when-volume-detached-unexpectedly: {{ .Values.defaultSettings.autoDeletePodWhenVolumeDetachedUnexpectedly }}
-    disable-scheduling-on-cordoned-node: {{ .Values.defaultSettings.disableSchedulingOnCordonedNode }}
-    replica-zone-soft-anti-affinity: {{ .Values.defaultSettings.replicaZoneSoftAntiAffinity }}
-    node-down-pod-deletion-policy: {{ .Values.defaultSettings.nodeDownPodDeletionPolicy }}
-    allow-node-drain-with-last-healthy-replica: {{ .Values.defaultSettings.allowNodeDrainWithLastHealthyReplica }}
-    mkfs-ext4-parameters: {{ .Values.defaultSettings.mkfsExt4Parameters }}
-    disable-replica-rebuild: {{ .Values.defaultSettings.disableReplicaRebuild }}
-    replica-replenishment-wait-interval: {{ .Values.defaultSettings.replicaReplenishmentWaitInterval }}
-    concurrent-replica-rebuild-per-node-limit: {{ .Values.defaultSettings.concurrentReplicaRebuildPerNodeLimit }}
-    disable-revision-counter: {{ .Values.defaultSettings.disableRevisionCounter }}
-    system-managed-pods-image-pull-policy: {{ .Values.defaultSettings.systemManagedPodsImagePullPolicy }}
-    allow-volume-creation-with-degraded-availability: {{ .Values.defaultSettings.allowVolumeCreationWithDegradedAvailability }}
-    auto-cleanup-system-generated-snapshot: {{ .Values.defaultSettings.autoCleanupSystemGeneratedSnapshot }}
-    concurrent-automatic-engine-upgrade-per-node-limit: {{ .Values.defaultSettings.concurrentAutomaticEngineUpgradePerNodeLimit }}
-    backing-image-cleanup-wait-interval: {{ .Values.defaultSettings.backingImageCleanupWaitInterval }}
-    backing-image-recovery-wait-interval: {{ .Values.defaultSettings.backingImageRecoveryWaitInterval }}
-    guaranteed-engine-manager-cpu: {{ .Values.defaultSettings.guaranteedEngineManagerCPU }}
-    guaranteed-replica-manager-cpu: {{ .Values.defaultSettings.guaranteedReplicaManagerCPU }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.backupTarget) }}backup-target: {{ .Values.defaultSettings.backupTarget }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.backupTargetCredentialSecret) }}backup-target-credential-secret: {{ .Values.defaultSettings.backupTargetCredentialSecret }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.allowRecurringJobWhileVolumeDetached) }}allow-recurring-job-while-volume-detached: {{ .Values.defaultSettings.allowRecurringJobWhileVolumeDetached }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.createDefaultDiskLabeledNodes) }}create-default-disk-labeled-nodes: {{ .Values.defaultSettings.createDefaultDiskLabeledNodes }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.defaultDataPath) }}default-data-path: {{ .Values.defaultSettings.defaultDataPath }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.replicaSoftAntiAffinity) }}replica-soft-anti-affinity: {{ .Values.defaultSettings.replicaSoftAntiAffinity }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.replicaAutoBalance) }}replica-auto-balance: {{ .Values.defaultSettings.replicaAutoBalance }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.storageOverProvisioningPercentage) }}storage-over-provisioning-percentage: {{ .Values.defaultSettings.storageOverProvisioningPercentage }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.storageMinimalAvailablePercentage) }}storage-minimal-available-percentage: {{ .Values.defaultSettings.storageMinimalAvailablePercentage }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.upgradeChecker) }}upgrade-checker: {{ .Values.defaultSettings.upgradeChecker }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.defaultReplicaCount) }}default-replica-count: {{ .Values.defaultSettings.defaultReplicaCount }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.defaultDataLocality) }}default-data-locality: {{ .Values.defaultSettings.defaultDataLocality }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.defaultLonghornStaticStorageClass) }}default-longhorn-static-storage-class: {{ .Values.defaultSettings.defaultLonghornStaticStorageClass }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.backupstorePollInterval) }}backupstore-poll-interval: {{ .Values.defaultSettings.backupstorePollInterval }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.failedBackupTTL) }}failed-backup-ttl: {{ .Values.defaultSettings.failedBackupTTL }}{{ end }}
+    {{- if or (not (kindIs "invalid" .Values.defaultSettings.taintToleration)) (.Values.global.cattle.windowsCluster.enabled) }}
+    taint-toleration: {{ $windowsDefaultSettingTaintToleration := list }}{{ $defaultSettingTaintToleration := list -}}
+      {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.defaultSetting.taintToleration -}}
+        {{- $windowsDefaultSettingTaintToleration = .Values.global.cattle.windowsCluster.defaultSetting.taintToleration -}}
+      {{- end -}}
+      {{- if not (kindIs "invalid" .Values.defaultSettings.taintToleration) -}}
+        {{- $defaultSettingTaintToleration = .Values.defaultSettings.taintToleration -}}
+      {{- end -}}
+      {{- $taintToleration := list $windowsDefaultSettingTaintToleration $defaultSettingTaintToleration }}{{ join ";" (compact $taintToleration) -}}
+    {{- end }}
+    {{- if or (not (kindIs "invalid" .Values.defaultSettings.systemManagedComponentsNodeSelector)) (.Values.global.cattle.windowsCluster.enabled) }}
+    system-managed-components-node-selector: {{ $windowsDefaultSettingNodeSelector := list }}{{ $defaultSettingNodeSelector := list -}}
+      {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.defaultSetting.systemManagedComponentsNodeSelector -}}
+        {{ $windowsDefaultSettingNodeSelector = .Values.global.cattle.windowsCluster.defaultSetting.systemManagedComponentsNodeSelector -}}
+      {{- end -}}
+      {{- if not (kindIs "invalid" .Values.defaultSettings.systemManagedComponentsNodeSelector) -}}
+        {{- $defaultSettingNodeSelector = .Values.defaultSettings.systemManagedComponentsNodeSelector -}}
+      {{- end -}}
+      {{- $nodeSelector := list $windowsDefaultSettingNodeSelector $defaultSettingNodeSelector }}{{ join ";" (compact $nodeSelector) -}}
+    {{- end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.priorityClass) }}priority-class: {{ .Values.defaultSettings.priorityClass }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.autoSalvage) }}auto-salvage: {{ .Values.defaultSettings.autoSalvage }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.autoDeletePodWhenVolumeDetachedUnexpectedly) }}auto-delete-pod-when-volume-detached-unexpectedly: {{ .Values.defaultSettings.autoDeletePodWhenVolumeDetachedUnexpectedly }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.disableSchedulingOnCordonedNode) }}disable-scheduling-on-cordoned-node: {{ .Values.defaultSettings.disableSchedulingOnCordonedNode }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.replicaZoneSoftAntiAffinity) }}replica-zone-soft-anti-affinity: {{ .Values.defaultSettings.replicaZoneSoftAntiAffinity }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.nodeDownPodDeletionPolicy) }}node-down-pod-deletion-policy: {{ .Values.defaultSettings.nodeDownPodDeletionPolicy }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.allowNodeDrainWithLastHealthyReplica) }}allow-node-drain-with-last-healthy-replica: {{ .Values.defaultSettings.allowNodeDrainWithLastHealthyReplica }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.mkfsExt4Parameters) }}mkfs-ext4-parameters: {{ .Values.defaultSettings.mkfsExt4Parameters }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.disableReplicaRebuild) }}disable-replica-rebuild: {{ .Values.defaultSettings.disableReplicaRebuild }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.replicaReplenishmentWaitInterval) }}replica-replenishment-wait-interval: {{ .Values.defaultSettings.replicaReplenishmentWaitInterval }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.concurrentReplicaRebuildPerNodeLimit) }}concurrent-replica-rebuild-per-node-limit: {{ .Values.defaultSettings.concurrentReplicaRebuildPerNodeLimit }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.disableRevisionCounter) }}disable-revision-counter: {{ .Values.defaultSettings.disableRevisionCounter }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.systemManagedPodsImagePullPolicy) }}system-managed-pods-image-pull-policy: {{ .Values.defaultSettings.systemManagedPodsImagePullPolicy }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.allowVolumeCreationWithDegradedAvailability) }}allow-volume-creation-with-degraded-availability: {{ .Values.defaultSettings.allowVolumeCreationWithDegradedAvailability }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.autoCleanupSystemGeneratedSnapshot) }}auto-cleanup-system-generated-snapshot: {{ .Values.defaultSettings.autoCleanupSystemGeneratedSnapshot }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.concurrentAutomaticEngineUpgradePerNodeLimit) }}concurrent-automatic-engine-upgrade-per-node-limit: {{ .Values.defaultSettings.concurrentAutomaticEngineUpgradePerNodeLimit }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.backingImageCleanupWaitInterval) }}backing-image-cleanup-wait-interval: {{ .Values.defaultSettings.backingImageCleanupWaitInterval }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.backingImageRecoveryWaitInterval) }}backing-image-recovery-wait-interval: {{ .Values.defaultSettings.backingImageRecoveryWaitInterval }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.guaranteedEngineManagerCPU) }}guaranteed-engine-manager-cpu: {{ .Values.defaultSettings.guaranteedEngineManagerCPU }}{{ end }}
+    {{ if not (kindIs "invalid" .Values.defaultSettings.guaranteedReplicaManagerCPU) }}guaranteed-replica-manager-cpu: {{ .Values.defaultSettings.guaranteedReplicaManagerCPU }}{{ end }}

chart/templates/deployment-driver.yaml

@@ -89,15 +89,25 @@ spec:
       - name: {{ .Values.privateRegistry.registrySecret }}
       {{- end }}
       {{- if .Values.longhornDriver.priorityClass }}
-      priorityClassName: {{ .Values.longhornDriver.priorityClass | quote}}
+      priorityClassName: {{ .Values.longhornDriver.priorityClass | quote }}
       {{- end }}
-      {{- if .Values.longhornDriver.tolerations }}
+      {{- if or .Values.longhornDriver.tolerations .Values.global.cattle.windowsCluster.enabled }}
       tolerations:
+        {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }}
+{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }}
+        {{- end }}
+        {{- if .Values.longhornDriver.tolerations }}
 {{ toYaml .Values.longhornDriver.tolerations | indent 6 }}
+        {{- end }}
       {{- end }}
-      {{- if .Values.longhornDriver.nodeSelector }}
+      {{- if or .Values.longhornDriver.nodeSelector .Values.global.cattle.windowsCluster.enabled }}
       nodeSelector:
+        {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }}
+{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }}
+        {{- end }}
+        {{- if .Values.longhornDriver.nodeSelector }}
 {{ toYaml .Values.longhornDriver.nodeSelector | indent 8 }}
+        {{- end }}
       {{- end }}
       serviceAccountName: longhorn-service-account
       securityContext:

chart/templates/deployment-ui.yaml

@@ -19,28 +19,50 @@ spec:
       - name: longhorn-ui
         image: {{ template "registry_url" . }}{{ .Values.image.longhorn.ui.repository }}:{{ .Values.image.longhorn.ui.tag }}
         imagePullPolicy: IfNotPresent
-        securityContext:
-          runAsUser: 0
+        volumeMounts:
+        - name : nginx-cache
+          mountPath: /var/cache/nginx/
+        - name : nginx-config
+          mountPath: /var/config/nginx/
+        - name: var-run
+          mountPath: /var/run/
         ports:
         - containerPort: 8000
           name: http
         env:
           - name: LONGHORN_MANAGER_IP
             value: "http://longhorn-backend:9500"
+      volumes:
+      - emptyDir: {}
+        name: nginx-cache
+      - emptyDir: {}
+        name: nginx-config
+      - emptyDir: {}
+        name: var-run
       {{- if .Values.privateRegistry.registrySecret }}
       imagePullSecrets:
       - name: {{ .Values.privateRegistry.registrySecret }}
       {{- end }}
       {{- if .Values.longhornUI.priorityClass }}
-      priorityClassName: {{ .Values.longhornUI.priorityClass | quote}}
+      priorityClassName: {{ .Values.longhornUI.priorityClass | quote }}
       {{- end }}
-      {{- if .Values.longhornUI.tolerations }}
+      {{- if or .Values.longhornUI.tolerations .Values.global.cattle.windowsCluster.enabled }}
       tolerations:
+        {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }}
+{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }}
+        {{- end }}
+        {{- if .Values.longhornUI.tolerations }}
 {{ toYaml .Values.longhornUI.tolerations | indent 6 }}
+        {{- end }}
       {{- end }}
-      {{- if .Values.longhornUI.nodeSelector }}
+      {{- if or .Values.longhornUI.nodeSelector .Values.global.cattle.windowsCluster.enabled }}
       nodeSelector:
+        {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }}
+{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }}
+        {{- end }}
+        {{- if .Values.longhornUI.nodeSelector }}
 {{ toYaml .Values.longhornUI.nodeSelector | indent 8 }}
+        {{- end }}
       {{- end }}
 ---
 kind: Service

chart/templates/postupgrade-job.yaml

@@ -35,14 +35,24 @@ spec:
       - name: {{ .Values.privateRegistry.registrySecret }}
       {{- end }}
       {{- if .Values.longhornManager.priorityClass }}
-      priorityClassName: {{ .Values.longhornManager.priorityClass | quote}}
+      priorityClassName: {{ .Values.longhornManager.priorityClass | quote }}
       {{- end }}
       serviceAccountName: longhorn-service-account
-      {{- if .Values.longhornManager.tolerations }}
+      {{- if or .Values.longhornManager.tolerations .Values.global.cattle.windowsCluster.enabled }}
       tolerations:
+        {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }}
+{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }}
+        {{- end }}
+        {{- if .Values.longhornManager.tolerations }}
 {{ toYaml .Values.longhornManager.tolerations | indent 6 }}
+        {{- end }}
       {{- end }}
-      {{- if .Values.longhornManager.nodeSelector }}
+      {{- if or .Values.longhornManager.nodeSelector .Values.global.cattle.windowsCluster.enabled }}
       nodeSelector:
+        {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }}
+{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }}
+        {{- end }}
+        {{- if .Values.longhornManager.nodeSelector }}
 {{ toYaml .Values.longhornManager.nodeSelector | indent 8 }}
+        {{- end }}
       {{- end }}

chart/templates/serviceaccount.yaml

@@ -4,3 +4,7 @@ metadata:
   name: longhorn-service-account
   namespace: {{ include "release_namespace" . }}
   labels: {{- include "longhorn.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}

chart/templates/services.yaml

@@ -4,7 +4,7 @@ kind: Service
 metadata:
   labels: {{- include "longhorn.labels" . | nindent 4 }}
   name: longhorn-engine-manager
-  namespace: longhorn-system
+  namespace: {{ include "release_namespace" . }}
 spec:
   clusterIP: None
   selector:
@@ -16,7 +16,7 @@ kind: Service
 metadata:
   labels: {{- include "longhorn.labels" . | nindent 4 }}
   name: longhorn-replica-manager
-  namespace: longhorn-system
+  namespace: {{ include "release_namespace" . }}
 spec:
   clusterIP: None
   selector:

chart/templates/storageclass.yaml

@@ -21,7 +21,10 @@ data:
       staleReplicaTimeout: "30"
       fromBackup: ""
       {{- if .Values.persistence.defaultFsType }}
-      fsType: "{{.Values.persistence.defaultFsType}}"
+      fsType: "{{ .Values.persistence.defaultFsType }}"
+      {{- end }}
+      {{- if .Values.persistence.migratable }}
+      migratable: "{{ .Values.persistence.migratable }}"
       {{- end }}
       {{- if .Values.persistence.backingImage.enable }}
       backingImage: {{ .Values.persistence.backingImage.name }}

chart/templates/uninstall-job.yaml

@@ -36,14 +36,24 @@ spec:
       - name: {{ .Values.privateRegistry.registrySecret }}
       {{- end }}
       {{- if .Values.longhornManager.priorityClass }}
-      priorityClassName: {{ .Values.longhornManager.priorityClass | quote}}
+      priorityClassName: {{ .Values.longhornManager.priorityClass | quote }}
       {{- end }}
       serviceAccountName: longhorn-service-account
-      {{- if .Values.longhornManager.tolerations }}
+      {{- if or .Values.longhornManager.tolerations .Values.global.cattle.windowsCluster.enabled }}
       tolerations:
+        {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }}
+{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }}
+        {{- end }}
+        {{- if .Values.longhornManager.tolerations }}
 {{ toYaml .Values.longhornManager.tolerations | indent 6 }}
+        {{- end }}
       {{- end }}
-      {{- if .Values.longhornManager.nodeSelector }}
+      {{- if or .Values.longhornManager.nodeSelector .Values.global.cattle.windowsCluster.enabled }}
       nodeSelector:
+        {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }}
+{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }}
+        {{- end }}
+        {{- if or .Values.longhornManager.nodeSelector }}
 {{ toYaml .Values.longhornManager.nodeSelector | indent 8 }}
+        {{- end }}
       {{- end }}

chart/values.yaml

@@ -4,27 +4,43 @@
 global:
   cattle:
     systemDefaultRegistry: ""
+    windowsCluster:
+      # Enable this to allow Longhorn to run on the Rancher deployed Windows cluster
+      enabled: false
+      # Tolerate Linux node taint
+      tolerations:
+      - key: "cattle.io/os"
+        value: "linux"
+        effect: "NoSchedule"
+        operator: "Equal"
+      # Select Linux nodes
+      nodeSelector:
+        kubernetes.io/os: "linux"
+      # Recognize toleration and node selector for Longhorn run-time created components
+      defaultSetting:
+        taintToleration: cattle.io/os=linux:NoSchedule
+        systemManagedComponentsNodeSelector: kubernetes.io/os:linux
 
 image:
   longhorn:
     engine:
       repository: longhornio/longhorn-engine
-      tag: v1.2.3
+      tag: v1.2.6
     manager:
       repository: longhornio/longhorn-manager
-      tag: v1.2.3
+      tag: v1.2.6
     ui:
       repository: longhornio/longhorn-ui
-      tag: v1.2.3
+      tag: v1.2.6
     instanceManager:
       repository: longhornio/longhorn-instance-manager
-      tag: v1_20211210
+      tag: v1_20220303_patch2
     shareManager:
       repository: longhornio/longhorn-share-manager
-      tag: v1_20211020
+      tag: v1_20211020_patch2
     backingImageManager:
       repository: longhornio/backing-image-manager
-      tag: v2_20210820
+      tag: v2_20221027
   csi:
     attacher:
       repository: longhornio/csi-attacher
@@ -56,6 +72,7 @@ persistence:
   defaultFsType: ext4
   defaultClassReplicaCount: 3
   reclaimPolicy: Delete
+  migratable: false
   recurringJobSelector:
     enable: false
     jobList: []
@@ -88,6 +105,7 @@ defaultSettings:
   defaultReplicaCount: ~
   defaultLonghornStaticStorageClass: ~
   backupstorePollInterval: ~
+  failedBackupTTL: ~
   taintToleration: ~
   systemManagedComponentsNodeSelector: ~
   priorityClass: ~
@@ -161,19 +179,6 @@ longhornUI:
   #  label-key1: "label-value1"
   #  label-key2: "label-value2"
 
-resources: {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-  # limits:
-  #  cpu: 100m
-  #  memory: 128Mi
-  # requests:
-  #  cpu: 100m
-  #  memory: 128Mi
-  #
-
 ingress:
   ## Set to true to enable ingress record generation
   enabled: false
@@ -226,3 +231,7 @@ namespaceOverride: ""
 
 # Annotations to add to the Longhorn Manager DaemonSet Pods. Optional.
 annotations: {}
+
+serviceAccount:
+  # Annotations to add to the service account
+  annotations: {}

deploy/backupstores/minio-backupstore.yaml

@@ -46,15 +46,15 @@ spec:
 
   containers:
   - name: minio
-    image: longhornio/minio:RELEASE.2020-10-18T21-54-12Z
-    command: ["sh", "-c", "mkdir -p /storage/backupbucket && mkdir -p /root/.minio/certs && ln -s /root/certs/private.key /root/.minio/certs/private.key && ln -s /root/certs/public.crt /root/.minio/certs/public.crt && exec /usr/bin/minio server /storage"]
+    image: minio/minio:RELEASE.2022-02-01T18-00-14Z
+    command: ["sh", "-c", "mkdir -p /storage/backupbucket && mkdir -p /root/.minio/certs && ln -s /root/certs/private.key /root/.minio/certs/private.key && ln -s /root/certs/public.crt /root/.minio/certs/public.crt && exec minio server /storage"]
     env:
-    - name: MINIO_ACCESS_KEY
+    - name: MINIO_ROOT_USER
       valueFrom:
         secretKeyRef:
           name: minio-secret
           key: AWS_ACCESS_KEY_ID
-    - name: MINIO_SECRET_KEY
+    - name: MINIO_ROOT_PASSWORD
       valueFrom:
         secretKeyRef:
           name: minio-secret

deploy/longhorn-images.txt

@@ -3,9 +3,9 @@ longhornio/csi-provisioner:v2.1.2
 longhornio/csi-resizer:v1.2.0
 longhornio/csi-snapshotter:v3.0.3
 longhornio/csi-node-driver-registrar:v2.3.0
-longhornio/backing-image-manager:v2_20210820
-longhornio/longhorn-engine:v1.2.3
-longhornio/longhorn-instance-manager:v1_20211210
-longhornio/longhorn-manager:v1.2.3
-longhornio/longhorn-share-manager:v1_20211020
-longhornio/longhorn-ui:v1.2.3
+longhornio/backing-image-manager:v2_20221027
+longhornio/longhorn-engine:v1.2.6
+longhornio/longhorn-instance-manager:v1_20220303_patch2
+longhornio/longhorn-manager:v1.2.6
+longhornio/longhorn-share-manager:v1_20211020_patch2
+longhornio/longhorn-ui:v1.2.6

deploy/longhorn.yaml

@@ -306,7 +306,7 @@ spec:
       jsonPath: .spec.image
     - name: RefCount
       type: integer
-      description: Number of volumes are using the engine image
+      description: Number of resources are using the engine image
       jsonPath: .status.refCount
     - name: BuildDate
       type: date
@@ -838,42 +838,43 @@ metadata:
   namespace: longhorn-system
 data:
   default-setting.yaml: |-
-    backup-target:
-    backup-target-credential-secret:
-    allow-recurring-job-while-volume-detached:
-    create-default-disk-labeled-nodes:
-    default-data-path:
-    replica-soft-anti-affinity:
-    replica-auto-balance:
-    storage-over-provisioning-percentage:
-    storage-minimal-available-percentage:
-    upgrade-checker:
-    default-replica-count:
-    default-data-locality:
-    default-longhorn-static-storage-class:
-    backupstore-poll-interval:
-    taint-toleration:
-    system-managed-components-node-selector:
-    priority-class:
-    auto-salvage:
-    auto-delete-pod-when-volume-detached-unexpectedly:
-    disable-scheduling-on-cordoned-node:
-    replica-zone-soft-anti-affinity:
-    node-down-pod-deletion-policy:
-    allow-node-drain-with-last-healthy-replica:
-    mkfs-ext4-parameters:
-    disable-replica-rebuild:
-    replica-replenishment-wait-interval:
-    concurrent-replica-rebuild-per-node-limit:
-    disable-revision-counter:
-    system-managed-pods-image-pull-policy:
-    allow-volume-creation-with-degraded-availability:
-    auto-cleanup-system-generated-snapshot:
-    concurrent-automatic-engine-upgrade-per-node-limit:
-    backing-image-cleanup-wait-interval:
-    backing-image-recovery-wait-interval:
-    guaranteed-engine-manager-cpu:
-    guaranteed-replica-manager-cpu:
+    #backup-target:
+    #backup-target-credential-secret:
+    #allow-recurring-job-while-volume-detached:
+    #create-default-disk-labeled-nodes:
+    #default-data-path:
+    #replica-soft-anti-affinity:
+    #replica-auto-balance:
+    #storage-over-provisioning-percentage:
+    #storage-minimal-available-percentage:
+    #upgrade-checker:
+    #default-replica-count:
+    #default-data-locality:
+    #default-longhorn-static-storage-class:
+    #backupstore-poll-interval:
+    #taint-toleration:
+    #system-managed-components-node-selector:
+    #priority-class:
+    #auto-salvage:
+    #auto-delete-pod-when-volume-detached-unexpectedly:
+    #disable-scheduling-on-cordoned-node:
+    #replica-zone-soft-anti-affinity:
+    #node-down-pod-deletion-policy:
+    #allow-node-drain-with-last-healthy-replica:
+    #mkfs-ext4-parameters:
+    #disable-replica-rebuild:
+    #replica-replenishment-wait-interval:
+    #concurrent-replica-rebuild-per-node-limit:
+    #disable-revision-counter:
+    #system-managed-pods-image-pull-policy:
+    #allow-volume-creation-with-degraded-availability:
+    #auto-cleanup-system-generated-snapshot:
+    #concurrent-automatic-engine-upgrade-per-node-limit:
+    #backing-image-cleanup-wait-interval:
+    #backing-image-recovery-wait-interval:
+    #guaranteed-engine-manager-cpu:
+    #guaranteed-replica-manager-cpu:
+    #failed-backup-ttl:
 ---
 apiVersion: policy/v1beta1
 kind: PodSecurityPolicy
@@ -984,7 +985,7 @@ spec:
     spec:
       containers:
       - name: longhorn-manager
-        image: longhornio/longhorn-manager:v1.2.3
+        image: longhornio/longhorn-manager:v1.2.6
         imagePullPolicy: IfNotPresent
         securityContext:
           privileged: true
@@ -993,15 +994,15 @@ spec:
         - -d
         - daemon
         - --engine-image
-        - longhornio/longhorn-engine:v1.2.3
+        - longhornio/longhorn-engine:v1.2.6
         - --instance-manager-image
-        - longhornio/longhorn-instance-manager:v1_20211210
+        - longhornio/longhorn-instance-manager:v1_20220303_patch2
         - --share-manager-image
-        - longhornio/longhorn-share-manager:v1_20211020
+        - longhornio/longhorn-share-manager:v1_20211020_patch2
         - --backing-image-manager-image
-        - longhornio/backing-image-manager:v2_20210820
+        - longhornio/backing-image-manager:v2_20221027
         - --manager-image
-        - longhornio/longhorn-manager:v1.2.3
+        - longhornio/longhorn-manager:v1.2.6
         - --service-account
         - longhorn-service-account
         ports:
@@ -1018,8 +1019,6 @@ spec:
         - name: longhorn
           mountPath: /var/lib/longhorn/
           mountPropagation: Bidirectional
-        - name: longhorn-default-setting
-          mountPath: /var/lib/longhorn-setting/
         env:
         - name: POD_NAMESPACE
           valueFrom:
@@ -1033,9 +1032,6 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
-        # Should be: mount path of the volume longhorn-default-setting + the key of the configmap data in 04-default-setting.yaml
-        - name: DEFAULT_SETTING_PATH
-          value: /var/lib/longhorn-setting/default-setting.yaml
       volumes:
       - name: dev
         hostPath:
@@ -1046,9 +1042,6 @@ spec:
       - name: longhorn
         hostPath:
           path: /var/lib/longhorn/
-      - name: longhorn-default-setting
-        configMap:
-          name: longhorn-default-setting
 #      imagePullSecrets:
 #      - name: ""
 #      priorityClassName:
@@ -1123,16 +1116,28 @@ spec:
     spec:
       containers:
       - name: longhorn-ui
-        image: longhornio/longhorn-ui:v1.2.3
+        image: longhornio/longhorn-ui:v1.2.6
         imagePullPolicy: IfNotPresent
-        securityContext:
-          runAsUser: 0
+        volumeMounts:
+        - name : nginx-cache
+          mountPath: /var/cache/nginx/
+        - name : nginx-config
+          mountPath: /var/config/nginx/
+        - name: var-run
+          mountPath: /var/run/
         ports:
         - containerPort: 8000
           name: http
         env:
           - name: LONGHORN_MANAGER_IP
             value: "http://longhorn-backend:9500"
+      volumes:
+      - emptyDir: {}
+        name: nginx-cache
+      - emptyDir: {}
+        name: nginx-config
+      - emptyDir: {}
+        name: var-run
 #      imagePullSecrets:
 #        - name: ""
 #      priorityClassName:
@@ -1179,18 +1184,18 @@ spec:
     spec:
       initContainers:
         - name: wait-longhorn-manager
-          image: longhornio/longhorn-manager:v1.2.3
+          image: longhornio/longhorn-manager:v1.2.6
           command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done']
       containers:
         - name: longhorn-driver-deployer
-          image: longhornio/longhorn-manager:v1.2.3
+          image: longhornio/longhorn-manager:v1.2.6
           imagePullPolicy: IfNotPresent
           command:
           - longhorn-manager
           - -d
           - deploy-driver
           - --manager-image
-          - longhornio/longhorn-manager:v1.2.3
+          - longhornio/longhorn-manager:v1.2.6
           - --manager-url
           - http://longhorn-backend:9500/v1
           env:

enhancements/20220801-failed-backups-cleanup.md

@@ -0,0 +1,110 @@
+# Failed Backup Clean Up
+
+## Summary
+
+Longhorn will leave the failed backups behind and will not delete the backups automatically either until the backup target is removed. Failed backup cleanup will be occurred when making a backup to remote backup target failed. This LEP will trigger the deletion of failed backups automatically.
+
+### Related Issues
+
+[[IMPROVEMENT] Support failed/obsolete orphaned backup cleanup](https://github.com/longhorn/longhorn/issues/3898)
+
+## Motivation
+
+### Goals
+
+- Support the auto-deletion of failed backups that exceeded the TTL.
+- Support the global auto-deletion option of failed backups cleanup for users.
+- The process should not be stuck in the reconciliation of the controllers.
+
+### Non-goals [optional]
+
+- Clean up unknown files or directories on the remote backup target.
+
+## Proposal
+
+1. The `backup_volume_controller` will be responsible for deleting Backup CR when there is a backup which state is in `Error` or `Unknown`.
+
+    The reconciliation procedure of the `backup_volume_controller` gets the latest failed backups from the datastore and delete the failed backups.
+
+    ```text
+
+          queue           ┌───────────────┐
+         ┌┐ ┌┐ ┌┐         │               │
+     ... ││ ││ ││ ──────► │ syncHandler() |
+         └┘ └┘ └┘         │               │
+                          └───────┬───────┘
+                                  │
+                       ┌──────────▼───────────┐
+                       │                      │
+                       │     reconcile()      |
+                       │                      │
+                       └──────────┬───────────┘
+                                  │
+                       ┌──────────▼───────────┐
+                       │                      │
+                       │  get failed backups  │
+                       │                      |
+                       |  then delete them    │
+                       │                      │
+                       └──────────────────────┘
+
+
+
+    ```
+
+### User Stories
+
+When a user or recurring job tries to make a backup and store it in the remote backup target, many situations will cause the backup procedure failed. In some cases, there will be some failed backups still staying in the Longhorn system and this kind of backups are not handled by the Longhorn system until user removes the backup target. Or users can manage the failed backups via Longhorn GUI or command line tools manually.
+
+After the enhancement, Longhorn can delete the failed backups automatically after enabling auto-deletion.
+
+### User Experience In Detail
+
+- Via Longhorn GUI
+  - Users can be aware of that backup was failed if auto-deletion is disabled.
+  - Users can check the event log to understand why the backup failed and deleted.
+
+- Via `kubectl`
+  - Users can list the failed backups by `kubectl -n longhorn-system get backups` if auto-deletion is disabled.
+
+## Design
+
+### Implementation Overview
+
+**Settings**
+
+- Add setting `failed-backup-ttl`. Default value is `1440` minutes and set to `0` to disable the auto-deletion.
+
+**Failed Backup**
+
+- Backups in the state `longhorn.BackupStateError` or `longhorn.BackupStateUnknown`.
+
+**Backup Controller**
+
+- Start the monitor and sync the backup status with the monitor in each reconcile loop.
+- Update the backup status.
+- Trigger `backup_volume_controller` to delete the failed backups.
+
+**Backup Volume controller**
+
+- Reconcile loop usually is triggered after backupstore polling which is controlled by **Backupstore Poll Interval** setting.
+- Start to get all backups in each reconcile loop
+- Tell failed backups from all backups and try to delete failed backups by default.
+- Update the backup volume CR status.
+
+### Test plan
+
+**Integration tests**
+
+- `backups` CRs with `Error` or `Unknown` state will be removed by `backup_volume_controller` triggered by backupstore polling when the `backup_monitor` detects the backup failed.
+- `backups` CRs with `Error` or `Unknown` state will not be handled if the auto-deletion is disabled.
+
+## Note [optional]
+
+### Why not leverage the current orphan framework
+
+1. We already have the backup CR to handle the backup resources and failed backup is not like orphaned replica which is not owned by any volume at the beginning.
+
+2. Cascading deletion of orphaned CR and backup CR would be more complicated than we just handle the failed backups immediately when backup procedure failed. Both in this LEP or orphan framework we would delete the failed backups by `backup_volume_controller`.
+
+3. Listing orphaned backups and failed backups on both two UI pages `Orphaned Data` and `Backup` might be a bit confusing for users. Deleting items manually on either of two pages would be involved in what it mentioned at statement 2.

examples/restore_to_file.yaml.template

@@ -16,7 +16,7 @@ spec:
       --output-file '/tmp/restore/<OUTPUT_FILE>'
       --output-format <OUTPUT_FORMAT>
     # the version of longhorn engine should be v0.4.1 or higher
-    image: rancher/longhorn-engine:v0.4.1
+    image: longhornio/longhorn-engine:<LONGHORN_VERSION>
     imagePullPolicy: IfNotPresent
     securityContext:
       privileged: true

scripts/environment_check.sh

@@ -1,18 +1,131 @@
 #!/bin/bash
 
-dependencies() {
+export RED='\x1b[0;31m'
+export GREEN='\x1b[38;5;22m'
+export CYAN='\x1b[36m'
+export YELLOW='\x1b[33m'
+export NO_COLOR='\x1b[0m'
+
+if [ -z "${LOG_TITLE}" ]; then
+  LOG_TITLE=''
+fi
+if [ -z "${LOG_LEVEL}" ]; then
+  LOG_LEVEL="INFO"
+fi
+
+debug() {
+  if [[ "${LOG_LEVEL}" == "DEBUG" ]]; then
+    local log_title
+    if [ -n "${LOG_TITLE}" ]; then
+     log_title="(${LOG_TITLE})"
+    else
+     log_title=''
+    fi
+    echo -e "${GREEN}[DEBUG]${log_title} ${NO_COLOR}$1"
+  fi
+}
+
+info() {
+  if [[ "${LOG_LEVEL}" == "DEBUG" ]] ||\
+     [[ "${LOG_LEVEL}" == "INFO" ]]; then
+    local log_title
+    if [ -n "${LOG_TITLE}" ]; then
+     log_title="(${LOG_TITLE})"
+    else
+     log_title=''
+    fi
+    echo -e "${CYAN}[INFO] ${log_title} ${NO_COLOR}$1"
+  fi
+}
+
+warn() {
+  if [[ "${LOG_LEVEL}" == "DEBUG" ]] ||\
+     [[ "${LOG_LEVEL}" == "INFO" ]] ||\
+     [[ "${LOG_LEVEL}" == "WARN" ]]; then
+    local log_title
+    if [ -n "${LOG_TITLE}" ]; then
+     log_title="(${LOG_TITLE})"
+    else
+     log_title=''
+    fi
+    echo -e "${YELLOW}[WARN] ${log_title} ${NO_COLOR}$1"
+  fi
+}
+
+error() {
+  if [[ "${LOG_LEVEL}" == "DEBUG" ]] ||\
+     [[ "${LOG_LEVEL}" == "INFO" ]] ||\
+     [[ "${LOG_LEVEL}" == "WARN" ]] ||\
+     [[ "${LOG_LEVEL}" == "ERROR" ]]; then
+    local log_title
+    if [ -n "${LOG_TITLE}" ]; then
+     log_title="(${LOG_TITLE})"
+    else
+     log_title=''
+    fi
+    echo -e "${RED}[ERROR]${log_title} ${NO_COLOR}$1"
+  fi
+}
+
+detect_node_os()
+{
+  local pod="$1"
+
+  OS=`kubectl exec -i $pod -- nsenter --mount=/proc/1/ns/mnt -- bash -c 'grep -E "^ID_LIKE=" /etc/os-release | cut -d= -f2'`
+  if [[ -z "${OS}" ]]; then
+    OS=`kubectl exec -i $pod -- nsenter --mount=/proc/1/ns/mnt -- bash -c 'grep -E "^ID=" /etc/os-release | cut -d= -f2'`
+  fi
+  echo "$OS"
+}
+
+set_packages_and_check_cmd()
+{
+  case $OS in
+  *"debian"* | *"ubuntu"* )
+    CHECK_CMD='dpkg -l | grep -w'
+    PACKAGES=(nfs-common open-iscsi)
+    ;;
+  *"centos"* | *"fedora"* | *"rocky"* | *"ol"* )
+    CHECK_CMD='rpm -q'
+    PACKAGES=(nfs-utils iscsi-initiator-utils)
+    ;;
+  *"suse"* )
+    CHECK_CMD='rpm -q'
+    PACKAGES=(nfs-client open-iscsi)
+    ;;
+  *"arch"* )
+    CHECK_CMD='pacman -Q'
+    PACKAGES=(nfs-utils open-iscsi)
+    ;;
+  *"gentoo"* )
+    CHECK_CMD='qlist -I'
+    PACKAGES=(net-fs/nfs-utils sys-block/open-iscsi)
+    ;;
+  *)
+    CHECK_CMD=''
+    PACKAGES=()
+    warn "Stop the environment check because '$OS' is not supported in the environment check script."
+    exit 1
+    ;;
+   esac
+}
+
+check_dependencies() {
   local targets=($@)
+
   local allFound=true
   for ((i=0; i<${#targets[@]}; i++)); do
     local target=${targets[$i]}
     if [ "$(which $target)" == "" ]; then
       allFound=false
-      echo Not found: $target
+      error "Not found: $target"
     fi
   done
   if [ "$allFound" == "false" ]; then
-    echo "Please install missing dependencies."
+    error "Please install missing dependencies."
     exit 2
+  else
+    info "Required dependencies are installed."
   fi
 }
 
@@ -33,9 +146,10 @@ spec:
       labels:
         app: longhorn-environment-check
     spec:
+      hostPID: true
       containers:
       - name: longhorn-environment-check
-        image: busybox
+        image: alpine:3.12
         args: ["/bin/sh", "-c", "sleep 1000000000"]
         volumeMounts:
         - name: mountpoint
@@ -48,14 +162,14 @@ spec:
         hostPath:
             path: /tmp/longhorn-environment-check
 EOF
-  kubectl create -f $TEMP_DIR/environment_check.yaml
+  kubectl create -f $TEMP_DIR/environment_check.yaml > /dev/null
 }
 
 cleanup() {
-  echo "cleaning up..."
-  kubectl delete -f $TEMP_DIR/environment_check.yaml
+  info "Cleaning up longhorn-environment-check pods..."
+  kubectl delete -f $TEMP_DIR/environment_check.yaml > /dev/null
   rm -rf $TEMP_DIR
-  echo "clean up complete"
+  info "Cleanup completed."
 }
 
 wait_ds_ready() {
@@ -65,16 +179,16 @@ wait_ds_ready() {
     local desiredNumberScheduled=$(echo $ds | jq .status.desiredNumberScheduled)
 
     if [ "$desiredNumberScheduled" == "$numberReady" ] && [ "$desiredNumberScheduled" != "0" ]; then
-      echo "all pods ready ($numberReady/$desiredNumberScheduled)"
+      info "All longhorn-environment-check pods are ready ($numberReady/$desiredNumberScheduled)."
       return
     fi
 
-    echo "waiting for pods to become ready ($numberReady/$desiredNumberScheduled)"
+    info "Waiting for longhorn-environment-check pods to become ready ($numberReady/$desiredNumberScheduled)..."
     sleep 3
   done
 }
 
-validate_ds() {
+check_mount_propagation() {
   local allSupported=true
   local pods=$(kubectl -l app=longhorn-environment-check get po -o json)
 
@@ -88,25 +202,101 @@ validate_ds() {
 
     if [ "$mountPropagation" != "Bidirectional" ]; then
       allSupported=false
-      echo "node $nodeName: MountPropagation DISABLED"
+      error "node $nodeName: MountPropagation is disabled"
     fi
   done
 
   if [ "$allSupported" != "true" ]; then
-    echo
-    echo "  MountPropagation is disabled on at least one node."
-    echo "  As a result, CSI driver and Base image cannot be supported."
-    echo
+    error "MountPropagation is disabled on at least one node. As a result, CSI driver and Base image cannot be supported."
     exit 1
   else
-    echo -e "\n  MountPropagation is enabled!\n"
+    info "MountPropagation is enabled."
   fi
 }
 
-dependencies kubectl jq mktemp
+check_package_installed() {
+  local pods=$(kubectl get pods -o name | grep longhorn-environment-check)
+
+  local allFound=true
+
+  for pod in ${pods}; do
+    OS=`detect_node_os $pod`
+    if [ x"$OS" == x"" ]; then
+      error "Unable to detect OS on node $node."
+      exit 2
+    fi
+
+    set_packages_and_check_cmd "$OS"
+
+    for ((i=0; i<${#PACKAGES[@]}; i++)); do
+      local package=${PACKAGES[$i]}
+
+      kubectl exec -i $pod -- nsenter --mount=/proc/1/ns/mnt -- timeout 30 bash -c "$CHECK_CMD $package" > /dev/null 2>&1
+      if [ $? != 0 ]; then
+        allFound=false
+        node=`kubectl get ${pod} --no-headers -o=custom-columns=:.spec.nodeName`
+        error "$package is not found in $node."
+      fi
+    done
+  done
+
+  if [ "$allFound" == "false" ]; then
+    error "Please install missing packages."
+    exit 2
+  else
+    info "Required packages are installed."
+  fi
+}
+
+check_multipathd() {
+  local pods=$(kubectl get pods -o name | grep longhorn-environment-check)
+  local allNotFound=true
+
+  for pod in ${pods}; do
+    kubectl exec -t $pod -- nsenter --mount=/proc/1/ns/mnt -- bash -c "systemctl status --no-pager multipathd.service" > /dev/null 2>&1
+    if [ $? = 0 ]; then
+      allNotFound=false
+      node=`kubectl get ${pod} --no-headers -o=custom-columns=:.spec.nodeName`
+      warn "multipathd is running on $node."
+    fi
+  done
+
+  if [ "$allNotFound" == "false" ]; then
+    warn "multipathd would probably result in the Longhorn volume mount failure. Please refer to https://longhorn.io/kb/troubleshooting-volume-with-multipath for more information."
+  fi
+}
+
+check_iscsid() {
+  local pods=$(kubectl get pods -o name | grep longhorn-environment-check)
+  local allFound=true
+
+  for pod in ${pods}; do
+    kubectl exec -t $pod -- nsenter --mount=/proc/1/ns/mnt -- bash -c "systemctl status --no-pager iscsid.service" > /dev/null 2>&1
+
+    if [ $? != 0 ]; then
+      allFound=false
+      node=`kubectl get ${pod} --no-headers -o=custom-columns=:.spec.nodeName`
+      error "iscsid is not running on $node."
+    fi
+  done
+
+  if [ "$allFound" == "false" ]; then
+    exit 2
+  fi
+}
+
+DEPENDENCIES=(kubectl jq mktemp)
+check_dependencies ${DEPENDENCIES[@]}
+
 TEMP_DIR=$(mktemp -d)
+
 trap cleanup EXIT
 create_ds
 wait_ds_ready
-validate_ds
+check_package_installed
+check_iscsid
+check_multipathd
+check_mount_propagation
+
 exit 0
+

scripts/restore-backup-to-file.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+export RED='\x1b[0;31m'
+export NO_COLOR='\x1b[0m'
+
+usage () {
+   echo "USAGE: $0 --aws-access-key <your_aws_access_key>  \ "
+   echo "          --aws-secret-access-key <your_aws_secret_access_key> \ "
+   echo "          --backup-url s3://backupbucket@ap-northeast-1/backupstore?backup=<backup_name>&volume=<volume_name> \ "
+   echo "          --output-file volume.raw \ "
+   echo "          --output-format raw \ "
+   echo "          --version <longhorn_version>"
+   echo "          --backing-file <backing_file_path>"
+   echo "Restore a Longhorn backup to a raw image or a qcow2 image."
+   echo ""
+   echo "  -u, --backup-url             (Required) Backups S3/NFS URL. e.g., s3://backupbucket@us-east-1/backupstore?backup=backup-bd326da2c4414b02&volume=volumeexamplename"
+   echo "  -o, --output-file            (Required) Output file, e.g., /tmp/restore/volume.raw"
+   echo "  -f, --output-format          (Required) Output file format, e.g., raw or qcow2"
+   echo "  -v, --version                (Required) Longhorn version, e.g., v1.3.2"
+   echo "      --aws-access-key         (Optional) AWS credentials access key"
+   echo "      --aws-secret-access-key  (Optional) AWS credentials access secret key"
+   echo "  -b, --backing-file           (Optional) backing image. e.g., /tmp/backingfile.qcow2"
+   echo "  -h, --help                   Usage message"
+}
+
+error_invalid_params() {
+  echo -e "${RED}[ERROR]Invalid params. Check the required params.${NO_COLOR}"
+  usage
+  exit 1
+}
+
+while [[ "$#" -gt 0 ]]; do
+  key="$1"
+  case $key in
+    --aws-access-key)
+    aws_access_key="$2"
+    shift # past argument
+    shift # past value
+    ;;
+    --aws-secret-access-key)
+    aws_secret_access_key="$2"
+    shift # past argument
+    shift # past value
+    ;;
+    -u|--backup-url)
+    backup_url="$2"
+    shift # past argument
+    shift # past value
+    ;;
+    -o|--output-file)
+    output_file="$2"
+    shift # past argument
+    shift # past value
+    ;;
+    -f|--output-format)
+    output_format="$2"
+    shift # past argument
+    shift # past value
+    ;;
+    -b|--backing-file)
+    backing_file="$2"
+    shift # past argument
+    shift # past value
+    ;;
+    -v|--version)
+    version="$2"
+    shift # past argument
+    shift # past value
+    ;;
+    -h|--help)
+    usage
+    exit 0
+    shift
+    ;;
+    *)
+    error_invalid_params
+    ;;
+    esac
+done
+
+# Check the required parameters exits
+if [ -z "${backup_url}" ] || [ -z "${output_file}" ] || [ -z "${output_format}" ] || [ -z "${version}" ]; then
+      error_invalid_params
+fi
+if [[ "${backup_url}" =~ ^[Ss]3 ]]; then
+  if [ -z "${aws_access_key}" ] || [ -z "${aws_secret_access_key}" ]; then
+      error_invalid_params
+  fi
+fi
+
+# Compose the docker arguments
+if [[ "${backup_url}" =~ ^[Ss]3 ]]; then
+  CUSTOMIZED_ARGS="-e AWS_ACCESS_KEY_ID="${aws_access_key}" -e AWS_SECRET_ACCESS_KEY="${aws_secret_access_key}" "
+else
+  CUSTOMIZED_ARGS="--cap-add SYS_ADMIN --security-opt apparmor:unconfined"
+fi
+
+# Start restoring a backup to an image file. 
+docker run ${CUSTOMIZED_ARGS} -v /tmp/restore:/tmp/restore \
+            longhornio/longhorn-engine:"${version}" longhorn backup \
+            restore-to-file ""${backup_url}"" \
+            --output-file "/tmp/restore/${output_file}" \
+            --output-format "${output_format}" \
+            --backing-file "${backing_file}"

uninstall/uninstall.yaml

@@ -61,7 +61,7 @@ rules:
     resources: ["priorityclasses"]
     verbs: ["watch", "list"]
   - apiGroups: ["storage.k8s.io"]
-    resources: ["csidrivers", "storageclasses"]
+    resources: ["csidrivers", "storageclasses", "volumeattachments"]
     verbs: ["*"]
   - apiGroups: ["longhorn.io"]
     resources: ["volumes", "engines", "replicas", "settings", "engineimages", "nodes", "instancemanagers", "sharemanagers", "backingimages", "backingimagemanagers", "backingimagedatasources", "backuptargets", "backupvolumes", "backups", "recurringjobs"]
@@ -101,7 +101,7 @@ spec:
     spec:
       containers:
       - name: longhorn-uninstall
-        image: longhornio/longhorn-manager:v1.2.3
+        image: longhornio/longhorn-manager:v1.2.6
         imagePullPolicy: Always
         securityContext:
           privileged: true