Merge pull request #2117 from longhorn/v1.1.0

V1.1.0

README.md

@@ -33,7 +33,7 @@ You can read more about the community and its events here: https://github.com/lo
 
 ## Current status
 
-The latest release of Longhorn is **v1.0.2**.
+The latest release of Longhorn is **v1.1.0**.
 
 ## Source code
 Longhorn is 100% open source software. Project source code is spread across a number of repos:

chart/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 name: longhorn
-version: 1.0.2
+version: 1.1.0
-appVersion: v1.0.2
+appVersion: v1.1.0
-kubeVersion: ">=v1.14.0-r0"
+kubeVersion: ">=v1.16.0-r0"
 description: Longhorn is a distributed block storage system for Kubernetes.
 keywords:
 - longhorn
@@ -16,6 +16,7 @@ sources:
 - https://github.com/longhorn/longhorn
 - https://github.com/longhorn/longhorn-engine
 - https://github.com/longhorn/longhorn-instance-manager
+- https://github.com/longhorn/longhorn-share-manager
 - https://github.com/longhorn/longhorn-manager
 - https://github.com/longhorn/longhorn-ui
 - https://github.com/longhorn/longhorn-tests

chart/README.md

@@ -10,13 +10,14 @@ Longhorn is 100% open source software. Project source code is spread across a nu
 
 1. Longhorn Engine -- Core controller/replica logic https://github.com/longhorn/longhorn-engine
 2. Longhorn Instance Manager -- Controller/replica instance lifecycle management https://github.com/longhorn/longhorn-instance-manager
-3. Longhorn Manager -- Longhorn orchestration, includes CSI driver for Kubernetes https://github.com/longhorn/longhorn-manager
+3. Longhorn Share Manager -- NFS provisioner that exposes Longhorn volumes as ReadWriteMany volumes. https://github.com/longhorn/longhorn-share-manager
+4. Longhorn Manager -- Longhorn orchestration, includes CSI driver for Kubernetes https://github.com/longhorn/longhorn-manager
 4. Longhorn UI -- Dashboard https://github.com/longhorn/longhorn-ui
 
 ## Prerequisites
 
 1. Docker v1.13+
-2. Kubernetes v1.15+
+2. Kubernetes v1.16+
 3. Make sure `curl`, `findmnt`, `grep`, `awk` and `blkid` has been installed in all nodes of the Kubernetes cluster.
 4. Make sure `open-iscsi` has been installed in all nodes of the Kubernetes cluster. For GKE, recommended Ubuntu as guest OS image since it contains `open-iscsi` already.
 

chart/questions.yml

@@ -17,7 +17,7 @@ questions:
     label: Longhorn Manager Image Repository
     group: "Longhorn Images Settings"
   - variable: image.longhorn.manager.tag
-    default: v1.0.2
+    default: v1.1.0
     description: "Specify Longhorn Manager Image Tag"
     type: string
     label: Longhorn Manager Image Tag
@@ -29,7 +29,7 @@ questions:
     label: Longhorn Engine Image Repository
     group: "Longhorn Images Settings"
   - variable: image.longhorn.engine.tag
-    default: v1.0.2
+    default: v1.1.0
     description: "Specify Longhorn Engine Image Tag"
     type: string
     label: Longhorn Engine Image Tag
@@ -41,7 +41,7 @@ questions:
     label: Longhorn UI Image Repository
     group: "Longhorn Images Settings"
   - variable: image.longhorn.ui.tag
-    default: v1.0.2
+    default: v1.1.0
     description: "Specify Longhorn UI Image Tag"
     type: string
     label: Longhorn UI Image Tag
@@ -53,11 +53,23 @@ questions:
     label: Longhorn Instance Manager Image Repository
     group: "Longhorn Images Settings"
   - variable: image.longhorn.instanceManager.tag
-    default: v1_20200514
+    default: v1_20201216
     description: "Specify Longhorn Instance Manager Image Tag"
     type: string
     label: Longhorn Instance Manager Image Tag
     group: "Longhorn Images Settings"
+  - variable: image.longhorn.shareManager.repository
+    default: longhornio/longhorn-share-manager
+    description: "Specify Longhorn Share Manager Image Repository"
+    type: string
+    label: Longhorn Share Manager Image Repository
+    group: "Longhorn Images Settings"
+  - variable: image.longhorn.shareManager.tag
+    default: v1_20201204
+    description: "Specify Longhorn Share Manager Image Tag"
+    type: string
+    label: Longhorn Share Manager Image Tag
+    group: "Longhorn Images Settings"
   - variable: image.csi.attacher.repository
     default: longhornio/csi-attacher
     description: "Specify CSI attacher image repository. Leave blank to autodetect."
@@ -65,7 +77,7 @@ questions:
     label: Longhorn CSI Attacher Image Repository
     group: "Longhorn CSI Driver Images"
   - variable: image.csi.attacher.tag
-    default: v2.0.0
+    default: v2.2.1-lh1
     description: "Specify CSI attacher image tag. Leave blank to autodetect."
     type: string
     label: Longhorn CSI Attacher Image Tag
@@ -77,7 +89,7 @@ questions:
     label: Longhorn CSI Provisioner Image Repository
     group: "Longhorn CSI Driver Images"
   - variable: image.csi.provisioner.tag
-    default: v1.4.0
+    default: v1.6.0-lh1
     description: "Specify CSI provisioner image tag. Leave blank to autodetect."
     type: string
     label: Longhorn CSI Provisioner Image Tag
@@ -89,7 +101,7 @@ questions:
     label: Longhorn CSI Node Driver Registrar Image Repository
     group: "Longhorn CSI Driver Images"
   - variable: image.csi.nodeDriverRegistrar.tag
-    default: v1.2.0
+    default: v1.2.0-lh1
     description: "Specify CSI Node Driver Registrar image tag. Leave blank to autodetect."
     type: string
     label: Longhorn CSI Node Driver Registrar Image Tag
@@ -101,13 +113,19 @@ questions:
     label: Longhorn CSI Driver Resizer Image Repository
     group: "Longhorn CSI Driver Images"
   - variable: image.csi.resizer.tag
-    default: v0.3.0
+    default: v0.5.1-lh1
     description: "Specify CSI Driver Resizer image tag. Leave blank to autodetect."
     type: string
     label: Longhorn CSI Driver Resizer Image Tag
     group: "Longhorn CSI Driver Images"
+  - variable: image.csi.snapshotter.repository
+    default: longhornio/csi-snapshotter
+    description: "Specify CSI Driver Snapshotter image repository. Leave blank to autodetect."
+    type: string
+    label: Longhorn CSI Driver Snapshotter Image Repository
+    group: "Longhorn CSI Driver Images"
   - variable: image.csi.snapshotter.tag
-    default: v2.1.1
+    default: v2.1.1-lh1
     description: "Specify CSI Driver Snapshotter image tag. Leave blank to autodetect."
     type: string
     label: Longhorn CSI Driver Snapshotter Image Tag
@@ -130,6 +148,12 @@ questions:
   group: "Private Registry Settings"
   type: password
   default: ""
+- variable: privateRegistry.registrySecret
+  label: Private registry secret name
+  description: "Longhorn will automatically generate a Kubernetes secret with this name and use it to pull images from your private registry."
+  group: "Private Registry Settings"
+  type: string
+  default: ""
 - variable: longhorn.default_setting
   default: "false"
   description: "Customize the default settings before installing Longhorn for the first time. This option will only work if the cluster hasn't installed Longhorn."
@@ -138,12 +162,6 @@ questions:
   show_subquestion_if: true
   group: "Longhorn Default Settings"
   subquestions:
-  - variable: defaultSettings.registrySecret
-    label: Private registry secret
-    description: "The Kubernetes Secret name"
-    group: "Longhorn Default Settings"
-    type: string
-    default: ""
   - variable: csi.kubeletRootDir
     default:
     description: "Specify kubelet root-dir. Leave blank to autodetect."
@@ -152,7 +170,7 @@ questions:
     group: "Longhorn CSI Driver Settings"
   - variable: csi.attacherReplicaCount
     type: int
-    default:
+    default: 3
     min: 1
     max: 10
     description: "Specify replica count of CSI Attacher. By default 3."
@@ -160,7 +178,7 @@ questions:
     group: "Longhorn CSI Driver Settings"
   - variable: csi.provisionerReplicaCount
     type: int
-    default:
+    default: 3
     min: 1
     max: 10
     description: "Specify replica count of CSI Provisioner. By default 3."
@@ -168,7 +186,7 @@ questions:
     group: "Longhorn CSI Driver Settings"
   - variable: csi.resizerReplicaCount
     type: int
-    default:
+    default: 3
     min: 1
     max: 10
     description: "Specify replica count of CSI Resizer. By default 3."
@@ -176,34 +194,12 @@ questions:
     group: "Longhorn CSI Driver Settings"
   - variable: csi.snapshotterReplicaCount
     type: int
-    default:
+    default: 3
     min: 1
     max: 10
     description: "Specify replica count of CSI Snapshotter. By default 3."
     label: Longhorn CSI Snapshotter replica count
     group: "Longhorn CSI Driver Settings"
-  - variable: persistence.defaultClass
-    default: "true"
-    description: "Set as default StorageClass"
-    group: "Longhorn CSI Driver Settings"
-    type: boolean
-    required: true
-    label: Default Storage Class
-  - variable: persistence.reclaimPolicy
-    default: "Delete"
-    description: "Define reclaim policy (Retain or Delete)"
-    group: "Longhorn CSI Driver Settings"
-    type: string
-    required: true
-    label: Storage Class Retain Policy
-  - variable: persistence.defaultClassReplicaCount
-    description: "Set replica count for default StorageClass"
-    group: "Longhorn CSI Driver Settings"
-    type: int
-    default: 3
-    min: 1
-    max: 10
-    label: Default Storage Class Replica Count
   - variable: defaultSettings.backupTarget
     label: Backup Target
     description: "The endpoint used to access the backupstore. NFS and S3 are supported."
@@ -216,6 +212,13 @@ questions:
     group: "Longhorn Default Settings"
     type: string
     default:
+  - variable: defaultSettings.allowRecurringJobWhileVolumeDetached
+    label: Allow Recurring Job While Volume Is Detached
+    description: 'If this setting is enabled, Longhorn will automatically attaches the volume and takes snapshot/backup when it is the time to do recurring snapshot/backup.
+Note that the volume is not ready for workload during the period when the volume was automatically attached. Workload will have to wait until the recurring job finishes.'
+    group: "Longhorn Default Settings"
+    type: boolean
+    default: "false"
   - variable: defaultSettings.createDefaultDiskLabeledNodes
     label: Create Default Disk on Labeled Nodes
     description: 'Create default Disk automatically only on Nodes with the label "node.longhorn.io/create-default-disk=true" if no other disks exist. If disabled, the default disk will be created on all new nodes when each node is first added.'
@@ -228,6 +231,19 @@ questions:
     group: "Longhorn Default Settings"
     type: string
     default: "/var/lib/longhorn/"
+  - variable: defaultSettings.defaultDataLocality
+    label: Default Data Locality
+    description: 'We say a Longhorn volume has data locality if there is a local replica of the volume on the same node as the pod which is using the volume.
+This setting specifies the default data locality when a volume is created from the Longhorn UI. For Kubernetes configuration, update the `dataLocality` in the StorageClass
+The available modes are:
+- **disabled**. This is the default option. There may or may not be a replica on the same node as the attached volume (workload)
+- **best-effort**. This option instructs Longhorn to try to keep a replica on the same node as the attached volume (workload). Longhorn will not stop the volume, even if it cannot keep a replica local to the attached volume (workload) due to environment limitation, e.g. not enough disk space, incompatible disk tags, etc.'
+    group: "Longhorn Default Settings"
+    type: enum
+    options:
+    - "disabled"
+    - "best-effort"
+    default: "disabled"
   - variable: defaultSettings.replicaSoftAntiAffinity
     label: Replica Node Level Soft Anti-Affinity
     description: 'Allow scheduling on nodes with existing healthy replicas of the same volume. By default false.'
@@ -265,10 +281,13 @@ questions:
     default: 3
   - variable: defaultSettings.guaranteedEngineCPU
     label: Guaranteed Engine CPU
-    description: 'Allow Longhorn Instance Managers to have guaranteed CPU allocation. The value is how many CPUs should be reserved for each Engine/Replica Instance Manager Pod created by Longhorn. For example, 0.1 means one-tenth of a CPU. This will help maintain engine stability during high node workload. It only applies to the Engine/Replica Manager Pods created after the setting took effect.
+    description: "Allow Longhorn Instance Managers to have guaranteed CPU allocation. By default 0.25. The value is how many CPUs should be reserved for each Engine/Replica Instance Manager Pod created by Longhorn. For example, 0.1 means one-tenth of a CPU. This will help maintain engine stability during high node workload. It only applies to the Engine/Replica Instance Manager Pods created after the setting took effect.
-WARNING: After this setting is changed, all the instance managers on all the nodes will be automatically restarted.
+In order to prevent unexpected volume crash, you can use the following formula to calculate an appropriate value for this setting:
-WARNING: DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES.
+'Guaranteed Engine CPU = The estimated max Longhorn volume/replica count on a node * 0.1'.
-By default 0.25.'
+The result of above calculation doesn't mean that's the maximum CPU resources the Longhorn workloads require. To fully exploit the Longhorn volume I/O performance, you can allocate/guarantee more CPU resources via this setting.
+If it's hard to estimate the volume/replica count now, you can leave it with the default value, or allocate 1/8 of total CPU of a node. Then you can tune it when there is no running workload using Longhorn volumes.
+WARNING: After this setting is changed, all the instance managers on all the nodes will be automatically restarted
+WARNING: DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES."
     group: "Longhorn Default Settings"
     type: float
     default: 0.25
@@ -287,8 +306,15 @@ By default 0.25.'
     default: 300
   - variable: defaultSettings.taintToleration
     label: Kubernetes Taint Toleration
-    description: 'To dedicate nodes to store Longhorn replicas and reject other general workloads, set tolerations for Longhorn and add taints for the storage nodes. All Longhorn volumes should be detached before modifying toleration settings. We recommend setting tolerations during Longhorn deployment because the Longhorn system cannot be operated during the update. Multiple tolerations can be set here, and these tolerations are separated by semicolon. For example, `key1=value1:NoSchedule; key2:NoExecute`. Because `kubernetes.io` is used as the key of all Kubernetes default tolerations, it should not be used in the toleration settings.
+    description: "To dedicate nodes to store Longhorn replicas and reject other general workloads, set tolerations for Longhorn and add taints for the storage nodes.
-WARNING: DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES.'
+All Longhorn volumes should be detached before modifying toleration settings.
+We recommend setting tolerations during Longhorn deployment because the Longhorn system cannot be operated during the update.
+Multiple tolerations can be set here, and these tolerations are separated by semicolon. For example:
+* `key1=value1:NoSchedule; key2:NoExecute`
+* `:` this toleration tolerates everything because an empty key with operator `Exists` matches all keys, values and effects
+* `key1=value1:`  this toleration has empty effect. It matches all effects with key `key1`
+Because `kubernetes.io` is used as the key of all Kubernetes default tolerations, it should not be used in the toleration settings.
+WARNING: DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES!"
     group: "Longhorn Default Settings"
     type: string
     default: ""
@@ -304,6 +330,14 @@ WARNING: DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES.'
     group: "Longhorn Default Settings"
     type: boolean
     default: "true"
+  - variable: defaultSettings.autoDeletePodWhenVolumeDetachedUnexpectedly
+    label: Automatically Delete Workload Pod when The Volume Is Detached Unexpectedly
+    description: 'If enabled, Longhorn will automatically delete the workload pod that is managed by a controller (e.g. deployment, statefulset, daemonset, etc...) when Longhorn volume is detached unexpectedly (e.g. during Kubernetes upgrade, Docker reboot, or network disconnect). By deleting the pod, its controller restarts the pod and Kubernetes handles volume reattachment and remount.
+If disabled, Longhorn will not delete the workload pod that is managed by a controller. You will have to manually restart the pod to reattach and remount the volume.
+**Note:** This setting does not apply to the workload pods that do not have a controller. Longhorn never deletes them.'
+    group: "Longhorn Default Settings"
+    type: boolean
+    default: "true"
   - variable: defaultSettings.disableSchedulingOnCordonedNode
     label: Disable Scheduling On Cordoned Node
     description: "Disable Longhorn manager to schedule replica on Kubernetes cordoned node. By default true."
@@ -326,11 +360,114 @@ WARNING: DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES.'
     - "never"
     - "immediate"
     default: "wait"
+  - variable: defaultSettings.nodeDownPodDeletionPolicy
+    label: Pod Deletion Policy When Node is Down
+    description: "Defines the Longhorn action when a Volume is stuck with a StatefulSet/Deployment Pod on a node that is down.
+- **do-nothing** is the default Kubernetes behavior of never force deleting StatefulSet/Deployment terminating pods. Since the pod on the node that is down isn't removed, Longhorn volumes are stuck on nodes that are down.
+- **delete-statefulset-pod** Longhorn will force delete StatefulSet terminating pods on nodes that are down to release Longhorn volumes so that Kubernetes can spin up replacement pods.
+- **delete-deployment-pod** Longhorn will force delete Deployment terminating pods on nodes that are down to release Longhorn volumes so that Kubernetes can spin up replacement pods.
+- **delete-both-statefulset-and-deployment-pod** Longhorn will force delete StatefulSet/Deployment terminating pods on nodes that are down to release Longhorn volumes so that Kubernetes can spin up replacement pods."
+    group: "Longhorn Default Settings"
+    type: enum
+    options:
+    - "do-nothing"
+    - "delete-statefulset-pod"
+    - "delete-deployment-pod"
+    - "delete-both-statefulset-and-deployment-pod"
+    default: "do-nothing"
+  - variable: defaultSettings.allowNodeDrainWithLastHealthyReplica
+    label: Allow Node Drain with the Last Healthy Replica
+    description: "By default, Longhorn will block `kubectl drain` action on a node if the node contains the last healthy replica of a volume.
+If this setting is enabled, Longhorn will **not** block `kubectl drain` action on a node even if the node contains the last healthy replica of a volume."
+    group: "Longhorn Default Settings"
+    type: boolean
+    default: "false"
   - variable: defaultSettings.mkfsExt4Parameters
     label: Custom mkfs.ext4 parameters
     description: "Allows setting additional filesystem creation parameters for ext4. For older host kernels it might be necessary to disable the optional ext4 metadata_csum feature by specifying `-O ^64bit,^metadata_csum`."
     group: "Longhorn Default Settings"
     type: string
+  - variable: defaultSettings.disableReplicaRebuild
+    label: Disable Replica Rebuild
+    description: "This setting disable replica rebuild cross the whole cluster, eviction and data locality feature won't work if this setting is true. But doesn't have any impact to any current replica rebuild and restore disaster recovery volume."
+    group: "Longhorn Default Settings"
+    type: boolean
+    default: "false"
+  - variable: defaultSettings.replicaReplenishmentWaitInterval
+    label: Replica Replenishment Wait Interval
+    description: "In seconds. The interval determines how long Longhorn will wait at least in order to reuse the existing data on a failed replica rather than directly creating a new replica for a degraded volume.
+Warning: This option works only when there is a failed replica in the volume. And this option may block the rebuilding for a while in the case."
+    group: "Longhorn Default Settings"
+    type: int
+    min: 0
+    default: 600
+  - variable: defaultSettings.disableRevisionCounter
+    label: Disable Revision Counter
+    description: "This setting is only for volumes created by UI. By default, this is false meaning there will be a reivision counter file to track every write to the volume. During salvage recovering Longhorn will pick the repica with largest reivision counter as candidate to recover the whole volume. If revision counter is disabled, Longhorn will not track every write to the volume. During the salvage recovering, Longhorn will use the 'volume-head-xxx.img' file last modification time and file size to pick the replica candidate to recover the whole volume."
+    group: "Longhorn Default Settings"
+    type: boolean
+    default: "false"
+  - variable: defaultSettings.systemManagedPodsImagePullPolicy
+    label: System Managed Pod Image Pull Policy
+    description: "This setting defines the Image Pull Policy of Longhorn system managed pods, e.g. instance manager, engine image, CSI driver, etc. The new Image Pull Policy will only apply after the system managed pods restart."
+    group: "Longhorn Default Settings"
+    type: enum
+    options:
+    - "if-not-present"
+    - "always"
+    - "never"
+    default: "if-not-present"
+  - variable: defaultSettings.allowVolumeCreationWithDegradedAvailability
+    label: Allow Volume Creation with Degraded Availability
+    description: "This setting allows user to create and attach a volume that doesn't have all the replicas scheduled at the time of creation."
+    group: "Longhorn Default Settings"
+    type: boolean
+    default: "true"
+  - variable: defaultSettings.autoCleanupSystemGeneratedSnapshot
+    label: Automatically Cleanup System Generated Snapshot
+    description: "This setting enables Longhorn to automatically cleanup the system generated snapshot after replica rebuild is done."
+    group: "Longhorn Default Settings"
+    type: boolean
+    default: "true"
+- variable: persistence.defaultClass
+  default: "true"
+  description: "Set as default StorageClass for Longhorn"
+  label: Default Storage Class
+  group: "Longhorn Storage Class Settings"
+  required: true
+  type: boolean
+- variable: persistence.reclaimPolicy
+  label: Storage Class Retain Policy
+  description: "Define reclaim policy (Retain or Delete)"
+  group: "Longhorn Storage Class Settings"
+  required: true
+  type: enum
+  options:
+  - "Delete"
+  - "Retain"
+  default: "Delete"
+- variable: persistence.defaultClassReplicaCount
+  description: "Set replica count for Longhorn StorageClass"
+  label: Default Storage Class Replica Count
+  group: "Longhorn Storage Class Settings"
+  type: int
+  min: 1
+  max: 10
+  default: 3
+- variable: persistence.recurringJobs.enable
+  description: "Enable recurring job for Longhorn StorageClass"
+  group: "Longhorn Storage Class Settings"
+  label: Enable Storage Class Recurring Job
+  type: boolean
+  default: false
+  show_subquestion_if: true
+  subquestions:
+  - variable: persistence.recurringJobs.jobList
+    description: 'Recurring job list for Longhorn StorageClass. Please be careful of quotes of input. e.g., [{"name":"backup", "task":"backup", "cron":"*/2 * * * *", "retain":1,"labels": {"interval":"2m"}}]'
+    label: Storage Class Recurring Job List
+    group: "Longhorn Storage Class Settings"
+    type: string
+    default:
 - variable: ingress.enabled
   default: "false"
   description: "Expose app using Layer 7 Load Balancer - ingress"

chart/templates/_helpers.tpl

@@ -53,3 +53,14 @@ app.kubernetes.io/version: {{ .Chart.AppVersion }}
 {{ include "system_default_registry" . }}
 {{- end -}}
 {{- end -}}
+
+{{- /*
+ define the longhorn release namespace
+*/ -}}
+{{- define "release_namespace" -}}
+{{- if .Values.namespaceOverride -}}
+{{- .Values.namespaceOverride -}}
+{{- else -}}
+{{- .Release.Namespace -}}
+{{- end -}}
+{{- end -}}

chart/templates/clusterrole.yaml

@@ -22,16 +22,26 @@ rules:
 - apiGroups: ["batch"]
   resources: ["jobs", "cronjobs"]
   verbs: ["*"]
+- apiGroups: ["policy"]
+  resources: ["poddisruptionbudgets"]
+  verbs: ["*"]
 - apiGroups: ["scheduling.k8s.io"]
   resources: ["priorityclasses"]
   verbs: ["watch", "list"]
 - apiGroups: ["storage.k8s.io"]
   resources: ["storageclasses", "volumeattachments", "csinodes", "csidrivers"]
   verbs: ["*"]
-- apiGroups: ["coordination.k8s.io"]
+- apiGroups: ["snapshot.storage.k8s.io"]
-  resources: ["leases"]
+  resources: ["volumesnapshotclasses", "volumesnapshots", "volumesnapshotcontents", "volumesnapshotcontents/status"]
   verbs: ["*"]
 - apiGroups: ["longhorn.io"]
   resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings",
-              "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status"]
+              "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status",
+              "sharemanagers", "sharemanagers/status"]
   verbs: ["*"]
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["*"]
+- apiGroups: ["metrics.k8s.io"]
+  resources: ["pods", "nodes"]
+  verbs: ["get", "list"]

chart/templates/clusterrolebinding.yaml

@@ -10,4 +10,4 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: longhorn-service-account
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ include "release_namespace" . }}

chart/templates/crds.yaml

@@ -1,10 +1,8 @@
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   labels: {{- include "longhorn.labels" . | nindent 4 }}
     longhorn-manager: Engine
-  annotations:
-    helm.sh/resource-policy: keep
   name: engines.longhorn.io
 spec:
   group: longhorn.io
@@ -16,17 +14,46 @@ spec:
     - lhe
     singular: engine
   scope: Namespaced
-  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
     subresources:
       status: {}
+    additionalPrinterColumns:
+    - name: State
+      type: string
+      description: The current state of the engine
+      jsonPath: .status.currentState
+    - name: Node
+      type: string
+      description: The node that the engine is on
+      jsonPath: .spec.nodeID
+    - name: InstanceManager
+      type: string
+      description: The instance manager of the engine
+      jsonPath: .status.instanceManagerName
+    - name: Image
+      type: string
+      description: The current image of the engine
+      jsonPath: .status.currentImage
+    - name: Age
+      type: date
+      jsonPath: .metadata.creationTimestamp
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   labels: {{- include "longhorn.labels" . | nindent 4 }}
     longhorn-manager: Replica
-  annotations:
-    helm.sh/resource-policy: keep
   name: replicas.longhorn.io
 spec:
   group: longhorn.io
@@ -38,17 +65,50 @@ spec:
     - lhr
     singular: replica
   scope: Namespaced
-  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
     subresources:
       status: {}
+    additionalPrinterColumns:
+    - name: State
+      type: string
+      description: The current state of the replica
+      jsonPath: .status.currentState
+    - name: Node
+      type: string
+      description: The node that the replica is on
+      jsonPath: .spec.nodeID
+    - name: Disk
+      type: string
+      description: The disk that the replica is on
+      jsonPath: .spec.diskID
+    - name: InstanceManager
+      type: string
+      description: The instance manager of the replica
+      jsonPath: .status.instanceManagerName
+    - name: Image
+      type: string
+      description: The current image of the replica
+      jsonPath: .status.currentImage
+    - name: Age
+      type: date
+      jsonPath: .metadata.creationTimestamp
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   labels: {{- include "longhorn.labels" . | nindent 4 }}
     longhorn-manager: Setting
-  annotations:
-    helm.sh/resource-policy: keep
   name: settings.longhorn.io
 spec:
   group: longhorn.io
@@ -60,15 +120,27 @@ spec:
     - lhs
     singular: setting
   scope: Namespaced
-  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        x-kubernetes-preserve-unknown-fields: true
+    additionalPrinterColumns:
+    - name: Value
+      type: string
+      description: The value of the setting
+      jsonPath: .value
+    - name: Age
+      type: date
+      jsonPath: .metadata.creationTimestamp
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   labels: {{- include "longhorn.labels" . | nindent 4 }}
     longhorn-manager: Volume
-  annotations:
-    helm.sh/resource-policy: keep
   name: volumes.longhorn.io
 spec:
   group: longhorn.io
@@ -80,17 +152,50 @@ spec:
     - lhv
     singular: volume
   scope: Namespaced
-  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
     subresources:
       status: {}
+    additionalPrinterColumns:
+    - name: State
+      type: string
+      description: The state of the volume
+      jsonPath: .status.state
+    - name: Robustness
+      type: string
+      description: The robustness of the volume
+      jsonPath: .status.robustness
+    - name: Scheduled
+      type: string
+      description: The scheduled condition of the volume
+      jsonPath: .status.conditions['scheduled']['status']
+    - name: Size
+      type: string
+      description: The size of the volume
+      jsonPath: .spec.size
+    - name: Node
+      type: string
+      description: The node that the volume is currently attaching to
+      jsonPath: .status.currentNodeID
+    - name: Age
+      type: date
+      jsonPath: .metadata.creationTimestamp
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   labels: {{- include "longhorn.labels" . | nindent 4 }}
     longhorn-manager: EngineImage
-  annotations:
-    helm.sh/resource-policy: keep
   name: engineimages.longhorn.io
 spec:
   group: longhorn.io
@@ -102,17 +207,46 @@ spec:
     - lhei
     singular: engineimage
   scope: Namespaced
-  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
     subresources:
       status: {}
+    additionalPrinterColumns:
+    - name: State
+      type: string
+      description: State of the engine image
+      jsonPath: .status.state
+    - name: Image
+      type: string
+      description: The Longhorn engine image
+      jsonPath: .spec.image
+    - name: RefCount
+      type: integer
+      description: Number of volumes are using the engine image
+      jsonPath: .status.refCount
+    - name: BuildDate
+      type: date
+      description: The build date of the engine image
+      jsonPath: .status.buildDate
+    - name: Age
+      type: date
+      jsonPath: .metadata.creationTimestamp
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   labels: {{- include "longhorn.labels" . | nindent 4 }}
     longhorn-manager: Node
-  annotations:
-    helm.sh/resource-policy: keep
   name: nodes.longhorn.io
 spec:
   group: longhorn.io
@@ -124,17 +258,42 @@ spec:
     - lhn
     singular: node
   scope: Namespaced
-  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
     subresources:
       status: {}
+    additionalPrinterColumns:
+    - name: Ready
+      type: string
+      description: Indicate whether the node is ready
+      jsonPath: .status.conditions['Ready']['status']
+    - name: AllowScheduling
+      type: boolean
+      description: Indicate whether the user disabled/enabled replica scheduling for the node
+      jsonPath: .spec.allowScheduling
+    - name: Schedulable
+      type: string
+      description: Indicate whether Longhorn can schedule replicas on the node
+      jsonPath: .status.conditions['Schedulable']['status']
+    - name: Age
+      type: date
+      jsonPath: .metadata.creationTimestamp
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   labels: {{- include "longhorn.labels" . | nindent 4 }}
     longhorn-manager: InstanceManager
-  annotations:
-    helm.sh/resource-policy: keep
   name: instancemanagers.longhorn.io
 spec:
   group: longhorn.io
@@ -146,6 +305,76 @@ spec:
     - lhim
     singular: instancemanager
   scope: Namespaced
-  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
     subresources:
       status: {}
+    additionalPrinterColumns:
+    - name: State
+      type: string
+      description: The state of the instance manager
+      jsonPath: .status.currentState
+    - name: Type
+      type: string
+      description: The type of the instance manager (engine or replica)
+      jsonPath: .spec.type
+    - name: Node
+      type: string
+      description: The node that the instance manager is running on
+      jsonPath: .spec.nodeID
+    - name: Age
+      type: date
+      jsonPath: .metadata.creationTimestamp
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  labels: {{- include "longhorn.labels" . | nindent 4 }}
+    longhorn-manager: ShareManager
+  name: sharemanagers.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: ShareManager
+    listKind: ShareManagerList
+    plural: sharemanagers
+    shortNames:
+    - lhsm
+    singular: sharemanager
+  scope: Namespaced
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+    subresources:
+      status: {}
+    additionalPrinterColumns:
+    - name: State
+      type: string
+      description: The state of the share manager
+      jsonPath: .status.state
+    - name: Node
+      type: string
+      description: The node that the share manager is owned by
+      jsonPath: .status.ownerID
+    - name: Age
+      type: date
+      jsonPath: .metadata.creationTimestamp

chart/templates/daemonset-sa.yaml

@@ -4,7 +4,7 @@ metadata:
   labels: {{- include "longhorn.labels" . | nindent 4 }}
     app: longhorn-manager
   name: longhorn-manager
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ include "release_namespace" . }}
 spec:
   selector:
     matchLabels:
@@ -28,6 +28,8 @@ spec:
         - "{{ template "registry_url" . }}{{ .Values.image.longhorn.engine.repository }}:{{ .Values.image.longhorn.engine.tag }}"
         - --instance-manager-image
         - "{{ template "registry_url" . }}{{ .Values.image.longhorn.instanceManager.repository }}:{{ .Values.image.longhorn.instanceManager.tag }}"
+        - --share-manager-image
+        - "{{ template "registry_url" . }}{{ .Values.image.longhorn.shareManager.repository }}:{{ .Values.image.longhorn.shareManager.tag }}"
         - --manager-image
         - "{{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }}"
         - --service-account
@@ -82,9 +84,9 @@ spec:
       - name: longhorn-default-setting
         configMap:
           name: longhorn-default-setting
-      {{- if .Values.defaultSettings.registrySecret }}
+      {{- if .Values.privateRegistry.registrySecret }}
       imagePullSecrets:
-      - name: {{ .Values.defaultSettings.registrySecret }}
+      - name: {{ .Values.privateRegistry.registrySecret }}
       {{- end }}
       serviceAccountName: longhorn-service-account
   updateStrategy:
@@ -97,7 +99,7 @@ metadata:
   labels: {{- include "longhorn.labels" . | nindent 4 }}
     app: longhorn-manager
   name: longhorn-backend
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ include "release_namespace" . }}
 spec:
   type: {{ .Values.service.manager.type }}
   sessionAffinity: ClientIP

chart/templates/default-setting.yaml

@@ -2,12 +2,13 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: longhorn-default-setting
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ include "release_namespace" . }}
   labels: {{- include "longhorn.labels" . | nindent 4 }}
 data:
   default-setting.yaml: |-
     backup-target: {{ .Values.defaultSettings.backupTarget }}
     backup-target-credential-secret: {{ .Values.defaultSettings.backupTargetCredentialSecret }}
+    allow-recurring-job-while-volume-detached: {{ .Values.defaultSettings.allowRecurringJobWhileVolumeDetached }}
     create-default-disk-labeled-nodes: {{ .Values.defaultSettings.createDefaultDiskLabeledNodes }}
     default-data-path: {{ .Values.defaultSettings.defaultDataPath }}
     replica-soft-anti-affinity: {{ .Values.defaultSettings.replicaSoftAntiAffinity }}
@@ -15,14 +16,23 @@ data:
     storage-minimal-available-percentage: {{ .Values.defaultSettings.storageMinimalAvailablePercentage }}
     upgrade-checker: {{ .Values.defaultSettings.upgradeChecker }}
     default-replica-count: {{ .Values.defaultSettings.defaultReplicaCount }}
+    default-data-locality: {{ .Values.defaultSettings.defaultDataLocality }}
     guaranteed-engine-cpu: {{ .Values.defaultSettings.guaranteedEngineCPU }}
     default-longhorn-static-storage-class: {{ .Values.defaultSettings.defaultLonghornStaticStorageClass }}
     backupstore-poll-interval: {{ .Values.defaultSettings.backupstorePollInterval }}
     taint-toleration: {{ .Values.defaultSettings.taintToleration }}
     priority-class: {{ .Values.defaultSettings.priorityClass }}
-    registry-secret: {{ .Values.defaultSettings.registrySecret }}
     auto-salvage: {{ .Values.defaultSettings.autoSalvage }}
+    auto-delete-pod-when-volume-detached-unexpectedly: {{ .Values.defaultSettings.autoDeletePodWhenVolumeDetachedUnexpectedly }}
     disable-scheduling-on-cordoned-node: {{ .Values.defaultSettings.disableSchedulingOnCordonedNode }}
     replica-zone-soft-anti-affinity: {{ .Values.defaultSettings.replicaZoneSoftAntiAffinity }}
     volume-attachment-recovery-policy: {{ .Values.defaultSettings.volumeAttachmentRecoveryPolicy }}
+    node-down-pod-deletion-policy: {{ .Values.defaultSettings.nodeDownPodDeletionPolicy }}
+    allow-node-drain-with-last-healthy-replica: {{ .Values.defaultSettings.allowNodeDrainWithLastHealthyReplica }}
     mkfs-ext4-parameters: {{ .Values.defaultSettings.mkfsExt4Parameters }}
+    disable-replica-rebuild: {{ .Values.defaultSettings.disableReplicaRebuild }}
+    replica-replenishment-wait-interval: {{ .Values.defaultSettings.replicaReplenishmentWaitInterval }}
+    disable-revision-counter: {{ .Values.defaultSettings.disableRevisionCounter }}
+    system-managed-pods-image-pull-policy: {{ .Values.defaultSettings.systemManagedPodsImagePullPolicy }}
+    allow-volume-creation-with-degraded-availability: {{ .Values.defaultSettings.allowVolumeCreationWithDegradedAvailability }}
+    auto-cleanup-system-generated-snapshot: {{ .Values.defaultSettings.autoCleanupSystemGeneratedSnapshot }}

chart/templates/deployment-driver.yaml

@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: longhorn-driver-deployer
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ include "release_namespace" . }}
   labels: {{- include "longhorn.labels" . | nindent 4 }}
 spec:
   replicas: 1
@@ -84,9 +84,9 @@ spec:
             value: {{ .Values.csi.snapshotterReplicaCount | quote }}
           {{- end }}
 
-      {{- if .Values.defaultSettings.registrySecret }}
+      {{- if .Values.privateRegistry.registrySecret }}
       imagePullSecrets:
-      - name: {{ .Values.defaultSettings.registrySecret }}
+      - name: {{ .Values.privateRegistry.registrySecret }}
       {{- end }}
       serviceAccountName: longhorn-service-account
       securityContext:

chart/templates/deployment-ui.yaml

@@ -4,7 +4,7 @@ metadata:
   labels: {{- include "longhorn.labels" . | nindent 4 }}
     app: longhorn-ui
   name: longhorn-ui
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ include "release_namespace" . }}
 spec:
   replicas: 1
   selector:
@@ -27,9 +27,9 @@ spec:
         env:
           - name: LONGHORN_MANAGER_IP
             value: "http://longhorn-backend:9500"
-      {{- if .Values.defaultSettings.registrySecret }}
+      {{- if .Values.privateRegistry.registrySecret }}
       imagePullSecrets:
-      - name: {{ .Values.defaultSettings.registrySecret }}
+      - name: {{ .Values.privateRegistry.registrySecret }}
       {{- end }}
 ---
 kind: Service
@@ -41,7 +41,7 @@ metadata:
     kubernetes.io/cluster-service: "true"
     {{- end }}
   name: longhorn-frontend
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ include "release_namespace" . }}
 spec:
   {{- if eq .Values.service.ui.type "Rancher-Proxy" }}
   type: ClusterIP

chart/templates/ingress.yaml

@@ -3,6 +3,7 @@ apiVersion: extensions/v1beta1
 kind: Ingress
 metadata:
   name: longhorn-ingress
+  namespace: {{ include "release_namespace" . }}
   labels: {{- include "longhorn.labels" . | nindent 4 }}
     app: longhorn-ingress
   annotations:

chart/templates/postupgrade-job.yaml

@@ -5,7 +5,7 @@ metadata:
     "helm.sh/hook": post-upgrade
     "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
   name: longhorn-post-upgrade
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ include "release_namespace" . }}
   labels: {{- include "longhorn.labels" . | nindent 4 }}
 spec:
   activeDeadlineSeconds: 900
@@ -18,7 +18,7 @@ spec:
       containers:
       - name: longhorn-post-upgrade
         image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }}
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         command:
         - longhorn-manager
         - post-upgrade
@@ -28,8 +28,8 @@ spec:
             fieldRef:
               fieldPath: metadata.namespace
       restartPolicy: OnFailure
-      {{- if .Values.defaultSettings.registrySecret }}
+      {{- if .Values.privateRegistry.registrySecret }}
       imagePullSecrets:
-      - name: {{ .Values.defaultSettings.registrySecret }}
+      - name: {{ .Values.privateRegistry.registrySecret }}
       {{- end }}
       serviceAccountName: longhorn-service-account

chart/templates/psp.yaml

@@ -35,7 +35,7 @@ kind: Role
 metadata:
   name: longhorn-psp-role
   labels: {{- include "longhorn.labels" . | nindent 4 }}
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ include "release_namespace" . }}
 rules:
 - apiGroups:
   - policy
@@ -51,7 +51,7 @@ kind: RoleBinding
 metadata:
   name: longhorn-psp-binding
   labels: {{- include "longhorn.labels" . | nindent 4 }}
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ include "release_namespace" . }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
@@ -59,8 +59,8 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: longhorn-service-account
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ include "release_namespace" . }}
 - kind: ServiceAccount
   name: default
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ include "release_namespace" . }}
 {{- end }}

chart/templates/registry-secret.yml

@@ -1,8 +1,9 @@
-{{- if .Values.defaultSettings.registrySecret }}
+{{- if .Values.privateRegistry.registrySecret }}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ .Values.defaultSettings.registrySecret }}
+  name: {{ .Values.privateRegistry.registrySecret }}
+  namespace: {{ include "release_namespace" . }}
   labels: {{- include "longhorn.labels" . | nindent 4 }}
 type: kubernetes.io/dockerconfigjson
 data:

chart/templates/serviceaccount.yaml

@@ -2,5 +2,5 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: longhorn-service-account
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ include "release_namespace" . }}
   labels: {{- include "longhorn.labels" . | nindent 4 }}

chart/templates/storageclass.yaml

@@ -1,18 +1,26 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: longhorn-storageclass
+  namespace: {{ include "release_namespace" . }}
+  labels: {{- include "longhorn.labels" . | nindent 4 }}
+data:
+  storageclass.yaml: |
     kind: StorageClass
     apiVersion: storage.k8s.io/v1
     metadata:
       name: longhorn
       annotations:
         storageclass.kubernetes.io/is-default-class: {{ .Values.persistence.defaultClass | quote }}
-  labels: {{- include "longhorn.labels" . | nindent 4 }}
     provisioner: driver.longhorn.io
     allowVolumeExpansion: true
     reclaimPolicy: "{{ .Values.persistence.reclaimPolicy }}"
+    volumeBindingMode: Immediate
     parameters:
       numberOfReplicas: "{{ .Values.persistence.defaultClassReplicaCount }}"
       staleReplicaTimeout: "30"
       fromBackup: ""
       baseImage: ""
-  {{- if .Values.recurringJobs.enable }}
+      {{- if .Values.persistence.recurringJobs.enable }}
-  recurringJobs: '{{ .Values.recurringJobs.jobsList | toPrettyJson | indent 2 | trim }}'
+      recurringJobs: '{{ .Values.persistence.recurringJobs.jobList }}'
       {{- end }}

chart/templates/tls-secrets.yaml

@@ -4,6 +4,7 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: longhorn
+  namespace: {{ include "release_namespace" . }}
   labels: {{- include "longhorn.labels" . | nindent 4 }}
     app: longhorn
 type: kubernetes.io/tls

chart/templates/uninstall-job.yaml

@@ -5,7 +5,7 @@ metadata:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-delete-policy": hook-succeeded
   name: longhorn-uninstall
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ include "release_namespace" . }}
   labels: {{- include "longhorn.labels" . | nindent 4 }}
 spec:
   activeDeadlineSeconds: 900
@@ -18,7 +18,7 @@ spec:
       containers:
       - name: longhorn-uninstall
         image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }}
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         command:
         - longhorn-manager
         - uninstall
@@ -29,8 +29,8 @@ spec:
             fieldRef:
               fieldPath: metadata.namespace
       restartPolicy: OnFailure
-      {{- if .Values.defaultSettings.registrySecret }}
+      {{- if .Values.privateRegistry.registrySecret }}
       imagePullSecrets:
-      - name: {{ .Values.defaultSettings.registrySecret }}
+      - name: {{ .Values.privateRegistry.registrySecret }}
       {{- end }}
       serviceAccountName: longhorn-service-account

chart/values.yaml

@@ -9,32 +9,35 @@ image:
   longhorn:
     engine:
       repository: longhornio/longhorn-engine
-      tag: v1.0.2
+      tag: v1.1.0
     manager:
       repository: longhornio/longhorn-manager
-      tag: v1.0.2
+      tag: v1.1.0
     ui:
       repository: longhornio/longhorn-ui
-      tag: v1.0.2
+      tag: v1.1.0
     instanceManager:
       repository: longhornio/longhorn-instance-manager
-      tag: v1_20200514
+      tag: v1_20201216
+    shareManager:
+      repository: longhornio/longhorn-share-manager
+      tag: v1_20201204
   csi:
     attacher:
       repository: longhornio/csi-attacher
-      tag: v2.0.0
+      tag: v2.2.1-lh1
     provisioner:
       repository: longhornio/csi-provisioner
-      tag: v1.4.0
+      tag: v1.6.0-lh1
     nodeDriverRegistrar:
       repository: longhornio/csi-node-driver-registrar
-      tag: v1.2.0
+      tag: v1.2.0-lh1
     resizer:
       repository: longhornio/csi-resizer
-      tag: v0.3.0
+      tag: v0.5.1-lh1
     snapshotter:
       repository: longhornio/csi-snapshotter
-      tag: v2.1.1
+      tag: v2.1.1-lh1
   pullPolicy: IfNotPresent
 
 service:
@@ -49,14 +52,9 @@ persistence:
   defaultClass: true
   defaultClassReplicaCount: 3
   reclaimPolicy: Delete
-
   recurringJobs:
     enable: false
-  jobsList: []
+    jobList: []
-    # - name: snap
-    #   task: snapshot
-    #   cron: '*/1 * * * *'
-    #   retain: 1
 
 csi:
   kubeletRootDir: ~
@@ -68,8 +66,10 @@ csi:
 defaultSettings:
   backupTarget: ~
   backupTargetCredentialSecret: ~
+  allowRecurringJobWhileVolumeDetached: ~
   createDefaultDiskLabeledNodes: ~
   defaultDataPath: ~
+  defaultDataLocality: ~
   replicaSoftAntiAffinity: ~
   storageOverProvisioningPercentage: ~
   storageMinimalAvailablePercentage: ~
@@ -80,17 +80,26 @@ defaultSettings:
   backupstorePollInterval: ~
   taintToleration: ~
   priorityClass: ~
-  registrySecret: ~
   autoSalvage: ~
+  autoDeletePodWhenVolumeDetachedUnexpectedly: ~
   disableSchedulingOnCordonedNode: ~
   replicaZoneSoftAntiAffinity: ~
   volumeAttachmentRecoveryPolicy: ~
+  nodeDownPodDeletionPolicy: ~
+  allowNodeDrainWithLastHealthyReplica: ~
   mkfsExt4Parameters: ~
+  disableReplicaRebuild: ~
+  replicaReplenishmentWaitInterval: ~
+  disableRevisionCounter: ~
+  systemManagedPodsImagePullPolicy: ~
+  allowVolumeCreationWithDegradedAvailability: ~
+  autoCleanupSystemGeneratedSnapshot: ~
 
 privateRegistry:
   registryUrl: ~
   registryUser: ~
   registryPasswd: ~
+  registrySecret: ~
 
 resources: {}
   # We usually recommend not to specify default resources and to leave this as a conscious
@@ -147,3 +156,7 @@ ingress:
 
 # Configure a pod security policy in the Longhorn namespace to allow privileged pods
 enablePSP: true
+
+## Specify override namespace, specifically this is useful for using longhorn as sub-chart
+## and its release namespace is not the `longhorn-system`
+namespaceOverride: ""

deploy/longhorn-images.txt

@@ -1,8 +1,10 @@
-longhornio/longhorn-engine:v1.0.2
+longhornio/longhorn-engine:v1.1.0
-longhornio/longhorn-instance-manager:v1_20200514
+longhornio/longhorn-instance-manager:v1_20201216
-longhornio/longhorn-manager:v1.0.2
+longhornio/longhorn-share-manager:v1_20201204
-longhornio/longhorn-ui:v1.0.2
+longhornio/longhorn-manager:v1.1.0
-longhornio/csi-attacher:v2.0.0
+longhornio/longhorn-ui:v1.1.0
-longhornio/csi-node-driver-registrar:v1.2.0
+longhornio/csi-attacher:v2.2.1-lh1
-longhornio/csi-provisioner:v1.4.0
+longhornio/csi-provisioner:v1.6.0-lh1
-longhornio/csi-resizer:v0.3.0
+longhornio/csi-resizer:v0.5.1-lh1
+longhornio/csi-snapshotter:v2.1.1-lh1
+longhornio/csi-node-driver-registrar:v1.2.0-lh1

deploy/longhorn.yaml

@@ -32,22 +32,29 @@ rules:
 - apiGroups: ["batch"]
   resources: ["jobs", "cronjobs"]
   verbs: ["*"]
+- apiGroups: ["policy"]
+  resources: ["poddisruptionbudgets"]
+  verbs: ["*"]
 - apiGroups: ["scheduling.k8s.io"]
   resources: ["priorityclasses"]
   verbs: ["watch", "list"]
 - apiGroups: ["storage.k8s.io"]
   resources: ["storageclasses", "volumeattachments", "csinodes", "csidrivers"]
   verbs: ["*"]
-- apiGroups: ["coordination.k8s.io"]
+- apiGroups: ["snapshot.storage.k8s.io"]
-  resources: ["leases"]
+  resources: ["volumesnapshotclasses", "volumesnapshots", "volumesnapshotcontents", "volumesnapshotcontents/status"]
-  verbs: ["get", "watch", "list", "delete", "update", "create"]
+  verbs: ["*"]
 - apiGroups: ["longhorn.io"]
   resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings",
-              "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status"]
+              "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status",
+              "sharemanagers", "sharemanagers/status"]
   verbs: ["*"]
 - apiGroups: ["coordination.k8s.io"]
   resources: ["leases"]
   verbs: ["*"]
+- apiGroups: ["metrics.k8s.io"]
+  resources: ["pods", "nodes"]
+  verbs: ["get", "list"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -62,7 +69,7 @@ subjects:
   name: longhorn-service-account
   namespace: longhorn-system
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   labels:
@@ -78,11 +85,42 @@ spec:
     - lhe
     singular: engine
   scope: Namespaced
-  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
     subresources:
       status: {}
+    additionalPrinterColumns:
+    - name: State
+      type: string
+      description: The current state of the engine
+      jsonPath: .status.currentState
+    - name: Node
+      type: string
+      description: The node that the engine is on
+      jsonPath: .spec.nodeID
+    - name: InstanceManager
+      type: string
+      description: The instance manager of the engine
+      jsonPath: .status.instanceManagerName
+    - name: Image
+      type: string
+      description: The current image of the engine
+      jsonPath: .status.currentImage
+    - name: Age
+      type: date
+      jsonPath: .metadata.creationTimestamp
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   labels:
@@ -98,11 +136,46 @@ spec:
     - lhr
     singular: replica
   scope: Namespaced
-  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
     subresources:
       status: {}
+    additionalPrinterColumns:
+    - name: State
+      type: string
+      description: The current state of the replica
+      jsonPath: .status.currentState
+    - name: Node
+      type: string
+      description: The node that the replica is on
+      jsonPath: .spec.nodeID
+    - name: Disk
+      type: string
+      description: The disk that the replica is on
+      jsonPath: .spec.diskID
+    - name: InstanceManager
+      type: string
+      description: The instance manager of the replica
+      jsonPath: .status.instanceManagerName
+    - name: Image
+      type: string
+      description: The current image of the replica
+      jsonPath: .status.currentImage
+    - name: Age
+      type: date
+      jsonPath: .metadata.creationTimestamp
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   labels:
@@ -118,9 +191,23 @@ spec:
     - lhs
     singular: setting
   scope: Namespaced
-  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        x-kubernetes-preserve-unknown-fields: true
+    additionalPrinterColumns:
+    - name: Value
+      type: string
+      description: The value of the setting
+      jsonPath: .value
+    - name: Age
+      type: date
+      jsonPath: .metadata.creationTimestamp
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   labels:
@@ -136,11 +223,46 @@ spec:
     - lhv
     singular: volume
   scope: Namespaced
-  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
     subresources:
       status: {}
+    additionalPrinterColumns:
+    - name: State
+      type: string
+      description: The state of the volume
+      jsonPath: .status.state
+    - name: Robustness
+      type: string
+      description: The robustness of the volume
+      jsonPath: .status.robustness
+    - name: Scheduled
+      type: string
+      description: The scheduled condition of the volume
+      jsonPath: .status.conditions['scheduled']['status']
+    - name: Size
+      type: string
+      description: The size of the volume
+      jsonPath: .spec.size
+    - name: Node
+      type: string
+      description: The node that the volume is currently attaching to
+      jsonPath: .status.currentNodeID
+    - name: Age
+      type: date
+      jsonPath: .metadata.creationTimestamp
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   labels:
@@ -156,11 +278,42 @@ spec:
     - lhei
     singular: engineimage
   scope: Namespaced
-  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
     subresources:
       status: {}
+    additionalPrinterColumns:
+    - name: State
+      type: string
+      description: State of the engine image
+      jsonPath: .status.state
+    - name: Image
+      type: string
+      description: The Longhorn engine image
+      jsonPath: .spec.image
+    - name: RefCount
+      type: integer
+      description: Number of volumes are using the engine image
+      jsonPath: .status.refCount
+    - name: BuildDate
+      type: date
+      description: The build date of the engine image
+      jsonPath: .status.buildDate
+    - name: Age
+      type: date
+      jsonPath: .metadata.creationTimestamp
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   labels:
@@ -176,11 +329,38 @@ spec:
     - lhn
     singular: node
   scope: Namespaced
-  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
     subresources:
       status: {}
+    additionalPrinterColumns:
+    - name: Ready
+      type: string
+      description: Indicate whether the node is ready
+      jsonPath: .status.conditions['Ready']['status']
+    - name: AllowScheduling
+      type: boolean
+      description: Indicate whether the user disabled/enabled replica scheduling for the node
+      jsonPath: .spec.allowScheduling
+    - name: Schedulable
+      type: string
+      description: Indicate whether Longhorn can schedule replicas on the node
+      jsonPath: .status.conditions['Schedulable']['status']
+    - name: Age
+      type: date
+      jsonPath: .metadata.creationTimestamp
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   labels:
@@ -196,9 +376,79 @@ spec:
     - lhim
     singular: instancemanager
   scope: Namespaced
-  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
     subresources:
       status: {}
+    additionalPrinterColumns:
+    - name: State
+      type: string
+      description: The state of the instance manager
+      jsonPath: .status.currentState
+    - name: Type
+      type: string
+      description: The type of the instance manager (engine or replica)
+      jsonPath: .spec.type
+    - name: Node
+      type: string
+      description: The node that the instance manager is running on
+      jsonPath: .spec.nodeID
+    - name: Age
+      type: date
+      jsonPath: .metadata.creationTimestamp
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  labels:
+    longhorn-manager: ShareManager
+  name: sharemanagers.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: ShareManager
+    listKind: ShareManagerList
+    plural: sharemanagers
+    shortNames:
+      - lhsm
+    singular: sharemanager
+  scope: Namespaced
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+    subresources:
+      status: {}
+    additionalPrinterColumns:
+      - name: State
+        type: string
+        description: The state of the share manager
+        jsonPath: .status.state
+      - name: Node
+        type: string
+        description: The node that the share manager is owned by
+        jsonPath: .status.ownerID
+      - name: Age
+        type: date
+        jsonPath: .metadata.creationTimestamp
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -209,6 +459,7 @@ data:
   default-setting.yaml: |-
     backup-target:
     backup-target-credential-secret:
+    allow-recurring-job-while-volume-detached:
     create-default-disk-labeled-nodes:
     default-data-path:
     replica-soft-anti-affinity:
@@ -216,17 +467,26 @@ data:
     storage-minimal-available-percentage:
     upgrade-checker:
     default-replica-count:
+    default-data-locality:
     guaranteed-engine-cpu:
     default-longhorn-static-storage-class:
     backupstore-poll-interval:
     taint-toleration:
     priority-class:
-    registry-secret:
     auto-salvage:
+    auto-delete-pod-when-volume-detached-unexpectedly:
     disable-scheduling-on-cordoned-node:
     replica-zone-soft-anti-affinity:
     volume-attachment-recovery-policy:
+    node-down-pod-deletion-policy:
+    allow-node-drain-with-last-healthy-replica:
     mkfs-ext4-parameters:
+    disable-replica-rebuild:
+    replica-replenishment-wait-interval:
+    disable-revision-counter:
+    system-managed-pods-image-pull-policy:
+    allow-volume-creation-with-degraded-availability:
+    auto-cleanup-system-generated-snapshot:
 ---
 apiVersion: policy/v1beta1
 kind: PodSecurityPolicy
@@ -290,6 +550,31 @@ subjects:
     name: default
     namespace: longhorn-system
 ---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: longhorn-storageclass
+  namespace: longhorn-system
+data:
+  storageclass.yaml: |
+    kind: StorageClass
+    apiVersion: storage.k8s.io/v1
+    metadata:
+      name: longhorn
+    provisioner: driver.longhorn.io
+    allowVolumeExpansion: true
+    reclaimPolicy: Delete
+    volumeBindingMode: Immediate
+    parameters:
+      numberOfReplicas: "3"
+      staleReplicaTimeout: "2880"
+      fromBackup: ""
+    #  diskSelector: "ssd,fast"
+    #  nodeSelector: "storage,fast"
+    #  recurringJobs: '[{"name":"snap", "task":"snapshot", "cron":"*/1 * * * *", "retain":1},
+    #                   {"name":"backup", "task":"backup", "cron":"*/2 * * * *", "retain":1,
+    #                    "labels": {"interval":"2m"}}]'
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -308,7 +593,7 @@ spec:
     spec:
       containers:
       - name: longhorn-manager
-        image: longhornio/longhorn-manager:v1.0.2
+        image: longhornio/longhorn-manager:v1.1.0
         imagePullPolicy: IfNotPresent
         securityContext:
           privileged: true
@@ -317,11 +602,13 @@ spec:
         - -d
         - daemon
         - --engine-image
-        - longhornio/longhorn-engine:v1.0.2
+        - longhornio/longhorn-engine:v1.1.0
         - --instance-manager-image
-        - longhornio/longhorn-instance-manager:v1_20200514
+        - longhornio/longhorn-instance-manager:v1_20201216
+        - --share-manager-image
+        - longhornio/longhorn-share-manager:v1_20201204
         - --manager-image
-        - longhornio/longhorn-manager:v1.0.2
+        - longhornio/longhorn-manager:v1.1.0
         - --service-account
         - longhorn-service-account
         ports:
@@ -335,9 +622,6 @@ spec:
           mountPath: /host/dev/
         - name: proc
           mountPath: /host/proc/
-        - name: varrun
-          mountPath: /var/run/
-          mountPropagation: Bidirectional
         - name: longhorn
           mountPath: /var/lib/longhorn/
           mountPropagation: Bidirectional
@@ -366,9 +650,6 @@ spec:
       - name: proc
         hostPath:
           path: /proc/
-      - name: varrun
-        hostPath:
-          path: /var/run/
       - name: longhorn
         hostPath:
           path: /var/lib/longhorn/
@@ -418,7 +699,7 @@ spec:
     spec:
       containers:
       - name: longhorn-ui
-        image: longhornio/longhorn-ui:v1.0.2
+        image: longhornio/longhorn-ui:v1.1.0
         imagePullPolicy: IfNotPresent
         securityContext:
           runAsUser: 0
@@ -465,18 +746,18 @@ spec:
     spec:
       initContainers:
         - name: wait-longhorn-manager
-          image: longhornio/longhorn-manager:v1.0.2
+          image: longhornio/longhorn-manager:v1.1.0
           command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done']
       containers:
         - name: longhorn-driver-deployer
-          image: longhornio/longhorn-manager:v1.0.2
+          image: longhornio/longhorn-manager:v1.1.0
           imagePullPolicy: IfNotPresent
           command:
           - longhorn-manager
           - -d
           - deploy-driver
           - --manager-image
-          - longhornio/longhorn-manager:v1.0.2
+          - longhornio/longhorn-manager:v1.1.0
           - --manager-url
           - http://longhorn-backend:9500/v1
           env:
@@ -498,13 +779,15 @@ spec:
           # For AirGap Installation
           # Replace PREFIX with your private registery
           #- name: CSI_ATTACHER_IMAGE
-          #  value: PREFIX/csi-attacher:v2.0.0
+          #  value: PREFIX/csi-attacher:v2.2.1-lh1
           #- name: CSI_PROVISIONER_IMAGE
-          #  value: PREFIX/csi-provisioner:v1.4.0
+          #  value: PREFIX/csi-provisioner:v1.6.0-lh1
           #- name: CSI_NODE_DRIVER_REGISTRAR_IMAGE
-          #  value: PREFIX/csi-node-driver-registrar:v1.2.0
+          #  value: PREFIX/csi-node-driver-registrar:v1.2.0-lh1
           #- name: CSI_RESIZER_IMAGE
-          #  value: PREFIX/csi-resizer:v0.3.0
+          #  value: PREFIX/csi-resizer:v0.5.1-lh1
+          #- name: CSI_SNAPSHOTTER_IMAGE
+          #  value: PREFIX/csi-snapshotter:v2.1.1-lh1
           # Manually specify number of CSI attacher replicas
           #- name: CSI_ATTACHER_REPLICA_COUNT
           #  value: "3"
@@ -513,25 +796,11 @@ spec:
           #  value: "3"
           #- name: CSI_RESIZER_REPLICA_COUNT
           #  value: "3"
+          #- name: CSI_SNAPSHOTTER_REPLICA_COUNT
+          #  value: "3"
       #imagePullSecrets:
       #- name:
       serviceAccountName: longhorn-service-account
       securityContext:
         runAsUser: 0
 ---
-kind: StorageClass
-apiVersion: storage.k8s.io/v1
-metadata:
-  name: longhorn
-provisioner: driver.longhorn.io
-allowVolumeExpansion: true
-parameters:
-  numberOfReplicas: "3"
-  staleReplicaTimeout: "2880"
-  fromBackup: ""
-#  diskSelector: "ssd,fast"
-#  nodeSelector: "storage,fast"
-#  recurringJobs: '[{"name":"snap", "task":"snapshot", "cron":"*/1 * * * *", "retain":1},
-#                   {"name":"backup", "task":"backup", "cron":"*/2 * * * *", "retain":1,
-#                    "labels": {"interval":"2m"}}]'
----

deploy/release-images.txt

@@ -1,8 +1,10 @@
-longhornio/longhorn-engine:v1.0.2
+longhornio/longhorn-engine:v1.1.0
-longhornio/longhorn-instance-manager:v1_20200514
+longhornio/longhorn-instance-manager:v1_20201216
-longhornio/longhorn-manager:v1.0.2
+longhornio/longhorn-share-manager:v1_20201204
-longhornio/longhorn-ui:v1.0.2
+longhornio/longhorn-manager:v1.1.0
-quay.io/k8scsi/csi-attacher:v2.0.0
+longhornio/longhorn-ui:v1.1.0
-quay.io/k8scsi/csi-node-driver-registrar:v1.2.0
+longhornio/csi-attacher:v2.2.1-lh1
-quay.io/k8scsi/csi-provisioner:v1.4.0
+longhornio/csi-provisioner:v1.6.0-lh1
-quay.io/k8scsi/csi-resizer:v0.3.0
+longhornio/csi-resizer:v0.5.1-lh1
+longhornio/csi-snapshotter:v2.1.1-lh1
+longhornio/csi-node-driver-registrar:v1.2.0-lh1

examples/block_volume.yaml

@@ -2,6 +2,7 @@ apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: longhorn-block-vol
+  namespace: default
 spec:
   accessModes:
     - ReadWriteOnce

examples/csi/example_pv.yaml

@@ -22,6 +22,7 @@ apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: longhorn-vol-pvc
+  namespace: default
 spec:
   accessModes:
     - ReadWriteOnce

examples/data_migration.yaml

@@ -0,0 +1,34 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  namespace: default  # namespace where the pvc's exist
+  name: volume-migration
+spec:
+  completions: 1
+  parallelism: 1
+  backoffLimit: 3
+  template:
+    metadata:
+      name: volume-migration
+      labels:
+        name: volume-migration
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: volume-migration
+          image: ubuntu:xenial
+          tty: true
+          command: [ "/bin/sh" ]
+          args: [ "-c", "cp -r -v /mnt/old /mnt/new" ]
+          volumeMounts:
+            - name: old-vol
+              mountPath: /mnt/old
+            - name: new-vol
+              mountPath: /mnt/new
+      volumes:
+        - name: old-vol
+          persistentVolumeClaim:
+            claimName: data-source-pvc # change to data source pvc
+        - name: new-vol
+          persistentVolumeClaim:
+            claimName: data-target-pvc # change to data target pvc

examples/deployment.yaml

@@ -15,6 +15,7 @@ apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: mysql-pvc
+  namespace: default
 spec:
   accessModes:
     - ReadWriteOnce
@@ -29,6 +30,7 @@ metadata:
   name: mysql
   labels:
     app: mysql
+  namespace: default
 spec:
   selector:
     matchLabels:
@@ -51,9 +53,6 @@ spec:
               - /var/lib/mysql/lost+found
           initialDelaySeconds: 5
           periodSeconds: 5
-        env:
-        - name: MYSQL_ROOT_PASSWORD
-          value: changeme
         ports:
         - containerPort: 3306
           name: mysql

examples/pod_with_pvc.yaml

@@ -2,6 +2,7 @@ apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: longhorn-volv-pvc
+  namespace: default
 spec:
   accessModes:
     - ReadWriteOnce

examples/rwx/01-security.yaml

@@ -1,85 +0,0 @@
-apiVersion: policy/v1beta1
-kind: PodSecurityPolicy
-metadata:
-  name: longhorn-nfs-provisioner
-spec:
-  fsGroup:
-    rule: RunAsAny
-  allowedCapabilities:
-    - DAC_READ_SEARCH
-    - SYS_RESOURCE
-  runAsUser:
-    rule: RunAsAny
-  seLinux:
-    rule: RunAsAny
-  supplementalGroups:
-    rule: RunAsAny
-  volumes:
-    - configMap
-    - downwardAPI
-    - emptyDir
-    - persistentVolumeClaim
-    - secret
-    - hostPath
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: longhorn-nfs-provisioner
-rules:
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "create", "delete"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims"]
-    verbs: ["get", "list", "watch", "update"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["events"]
-    verbs: ["create", "update", "patch"]
-  - apiGroups: [""]
-    resources: ["services", "endpoints"]
-    verbs: ["get"]
-  - apiGroups: ["extensions"]
-    resources: ["podsecuritypolicies"]
-    resourceNames: ["nfs-provisioner"]
-    verbs: ["use"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: longhorn-nfs-provisioner
-subjects:
-  - kind: ServiceAccount
-    name: longhorn-nfs-provisioner
-    namespace: longhorn-system
-roleRef:
-  kind: ClusterRole
-  name: longhorn-nfs-provisioner
-  apiGroup: rbac.authorization.k8s.io
----
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: leader-locking-longhorn-nfs-provisioner
-  namespace: longhorn-system
-rules:
-  - apiGroups: [""]
-    resources: ["endpoints"]
-    verbs: ["get", "list", "watch", "create", "update", "patch"]
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: leader-locking-longhorn-nfs-provisioner
-  namespace: longhorn-system
-subjects:
-  - kind: ServiceAccount
-    name: longhorn-nfs-provisioner
-    namespace: longhorn-system
-roleRef:
-  kind: Role
-  name: leader-locking-longhorn-nfs-provisioner
-  apiGroup: rbac.authorization.k8s.io

examples/rwx/02-longhorn-nfs-provisioner.yaml

@@ -1,178 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: longhorn-nfs-provisioner
-  namespace: longhorn-system
----
-kind: Service
-apiVersion: v1
-metadata:
-  name: longhorn-nfs-provisioner
-  namespace: longhorn-system
-  labels:
-    app: longhorn-nfs-provisioner
-spec:
-  # hardcode a cluster ip for the service
-  # so that on delete & recreate of the service the previous pv's still point
-  # to this nfs-provisioner, pick a new ip for each new nfs provisioner
-  clusterIP: 10.43.111.111
-  ports:
-    - name: nfs
-      port: 2049
-    - name: nfs-udp
-      port: 2049
-      protocol: UDP
-    - name: nlockmgr
-      port: 32803
-    - name: nlockmgr-udp
-      port: 32803
-      protocol: UDP
-    - name: mountd
-      port: 20048
-    - name: mountd-udp
-      port: 20048
-      protocol: UDP
-    - name: rquotad
-      port: 875
-    - name: rquotad-udp
-      port: 875
-      protocol: UDP
-    - name: rpcbind
-      port: 111
-    - name: rpcbind-udp
-      port: 111
-      protocol: UDP
-    - name: statd
-      port: 662
-    - name: statd-udp
-      port: 662
-      protocol: UDP
-  selector:
-    app: longhorn-nfs-provisioner
----
-kind: Deployment
-apiVersion: apps/v1
-metadata:
-  name: longhorn-nfs-provisioner
-  namespace: longhorn-system
-spec:
-  selector:
-    matchLabels:
-      app: longhorn-nfs-provisioner
-  replicas: 1
-  strategy:
-    type: Recreate
-  template:
-    metadata:
-      labels:
-        app: longhorn-nfs-provisioner
-    spec:
-      serviceAccount: longhorn-nfs-provisioner
-      containers:
-        - name: longhorn-nfs-provisioner
-          image: quay.io/kubernetes_incubator/nfs-provisioner:latest
-          ports:
-            - name: nfs
-              containerPort: 2049
-            - name: nfs-udp
-              containerPort: 2049
-              protocol: UDP
-            - name: nlockmgr
-              containerPort: 32803
-            - name: nlockmgr-udp
-              containerPort: 32803
-              protocol: UDP
-            - name: mountd
-              containerPort: 20048
-            - name: mountd-udp
-              containerPort: 20048
-              protocol: UDP
-            - name: rquotad
-              containerPort: 875
-            - name: rquotad-udp
-              containerPort: 875
-              protocol: UDP
-            - name: rpcbind
-              containerPort: 111
-            - name: rpcbind-udp
-              containerPort: 111
-              protocol: UDP
-            - name: statd
-              containerPort: 662
-            - name: statd-udp
-              containerPort: 662
-              protocol: UDP
-          securityContext:
-            capabilities:
-              add:
-                - DAC_READ_SEARCH
-                - SYS_RESOURCE
-          args:
-            - "-provisioner=nfs.longhorn.io"
-            - "-device-based-fsids=false"
-          env:
-            - name: POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-            - name: SERVICE_NAME
-              value: longhorn-nfs-provisioner
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-          imagePullPolicy: "IfNotPresent"
-          readinessProbe:
-            exec:
-              command:
-                - ls
-                - /export
-            initialDelaySeconds: 5
-            periodSeconds: 5
-          livenessProbe:
-            exec:
-              command:
-                - ls
-                - /export
-            initialDelaySeconds: 5
-            periodSeconds: 5
-          volumeMounts:
-            - name: export-volume
-              mountPath: /export
-      volumes:
-        - name: export-volume
-          persistentVolumeClaim:
-            claimName: longhorn-nfs-provisioner
-      # we want really quick failover
-      terminationGracePeriodSeconds: 30
-      tolerations:
-        - effect: NoExecute
-          key: node.kubernetes.io/not-ready
-          operator: Exists
-          tolerationSeconds: 60
-        - effect: NoExecute
-          key: node.kubernetes.io/unreachable
-          operator: Exists
-          tolerationSeconds: 60
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: longhorn-nfs-provisioner # longhorn backing pvc
-  namespace: longhorn-system
-spec:
-  storageClassName: longhorn
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: "20G" # make this 10% bigger then the workload pvc
----
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: longhorn-nfs # workload storage class
-provisioner: nfs.longhorn.io
-mountOptions:
-  - "vers=4.1"
-  - "noresvport"

examples/rwx/03-rwx-test.yaml

@@ -1,57 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: nfs-test
-spec:
-  accessModes:
-    - ReadWriteMany
-  storageClassName: longhorn-nfs
-  resources:
-    requests:
-      storage: 1Gi
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: nfs-test
-  labels:
-    app: nfs-test
-spec:
-  replicas: 4
-  selector:
-    matchLabels:
-      app: nfs-test
-  strategy:
-    type: Recreate
-  template:
-    metadata:
-      labels:
-        app: nfs-test
-    spec:
-      containers:
-        - image: ubuntu:xenial
-          imagePullPolicy: Always
-          command: ["/bin/sh", "-c"]
-          args:
-            - sleep 30; touch /mnt/nfs-test/test.log; while true; do date >> /mnt/nfs-test/test.log; sleep 1; done;
-          name: nfs-test
-          stdin: true
-          tty: true
-          livenessProbe:
-            exec:
-              command:
-                - timeout
-                - "10"
-                - ls
-                - /mnt/nfs-test
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            timeoutSeconds: 10
-          volumeMounts:
-            - mountPath: /mnt/nfs-test
-              name: nfs-test
-      restartPolicy: Always
-      volumes:
-        - name: nfs-test
-          persistentVolumeClaim:
-            claimName: nfs-test

examples/simple_pvc.yaml

@@ -2,6 +2,7 @@ apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: longhorn-simple-pvc
+  namespace: default
 spec:
   accessModes:
     - ReadWriteOnce

examples/snapshot/existing_backup.yaml

@@ -0,0 +1,14 @@
+apiVersion: snapshot.storage.k8s.io/v1beta1
+kind: VolumeSnapshotContent
+metadata:
+  name: test-existing-backup
+spec:
+  volumeSnapshotClassName: longhorn
+  driver: driver.longhorn.io
+  deletionPolicy: Delete
+  source:
+    # NOTE: change this to point to an existing backup on the backupstore
+    snapshotHandle: bs://test-vol/backup-625159fb469e492e
+  volumeSnapshotRef:
+    name: test-snapshot-existing-backup
+    namespace: default

examples/snapshot/restore_existing_backup.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: test-restore-existing-backup
+  namespace: default
+spec:
+  storageClassName: longhorn
+  dataSource:
+    name: test-snapshot-existing-backup
+    kind: VolumeSnapshot
+    apiGroup: snapshot.storage.k8s.io
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi

examples/snapshot/restore_pvc_snapshot.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: test-restore-snapshot-pvc
+  namespace: default
+spec:
+  storageClassName: longhorn
+  dataSource:
+    name: test-snapshot-pvc
+    kind: VolumeSnapshot
+    apiGroup: snapshot.storage.k8s.io
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi

examples/snapshot/snapshot_existing.yaml

@@ -0,0 +1,8 @@
+apiVersion: snapshot.storage.k8s.io/v1beta1
+kind: VolumeSnapshot
+metadata:
+  name: test-snapshot-existing-backup
+spec:
+  volumeSnapshotClassName: longhorn
+  source:
+    volumeSnapshotContentName: test-existing-backup

examples/snapshot/snapshot_pvc.yaml

@@ -0,0 +1,8 @@
+apiVersion: snapshot.storage.k8s.io/v1beta1
+kind: VolumeSnapshot
+metadata:
+  name: test-snapshot-pvc
+spec:
+  volumeSnapshotClassName: longhorn
+  source:
+    persistentVolumeClaimName: test-vol

examples/snapshot/snapshotclass.yaml

@@ -0,0 +1,9 @@
+kind: VolumeSnapshotClass
+apiVersion: snapshot.storage.k8s.io/v1beta1
+metadata:
+  name: longhorn
+driver: driver.longhorn.io
+deletionPolicy: Delete
+#parameters:
+#  csi.storage.k8s.io/snapshotter-secret-name: mysecret
+#  csi.storage.k8s.io/snapshotter-secret-namespace: mysecretnamespace

examples/statefulset.yaml

@@ -16,6 +16,7 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: web
+  namespace: default
 spec:
   selector:
     matchLabels:

uninstall/uninstall.yaml

@@ -16,7 +16,7 @@ rules:
     verbs:
       - "*"
   - apiGroups: [""]
-    resources: ["pods", "persistentvolumes", "persistentvolumeclaims", "nodes"]
+    resources: ["pods", "persistentvolumes", "persistentvolumeclaims", "nodes", "configmaps", "secrets", "services", "endpoints"]
     verbs: ["*"]
   - apiGroups: ["apps"]
     resources: ["daemonsets", "statefulsets", "deployments"]
@@ -24,14 +24,20 @@ rules:
   - apiGroups: ["batch"]
     resources: ["jobs", "cronjobs"]
     verbs: ["*"]
+  - apiGroups: ["policy"]
+    resources: ["poddisruptionbudgets"]
+    verbs: ["*"]
   - apiGroups: ["scheduling.k8s.io"]
     resources: ["priorityclasses"]
     verbs: ["watch", "list"]
   - apiGroups: ["storage.k8s.io"]
-    resources: ["csidrivers"]
+    resources: ["csidrivers", "storageclasses"]
     verbs: ["*"]
   - apiGroups: ["longhorn.io"]
-    resources: ["volumes", "engines", "replicas", "settings", "engineimages", "nodes", "instancemanagers"]
+    resources: ["volumes", "engines", "replicas", "settings", "engineimages", "nodes", "instancemanagers", "sharemanagers"]
+    verbs: ["*"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
     verbs: ["*"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -61,7 +67,7 @@ spec:
     spec:
       containers:
       - name: longhorn-uninstall
-        image: longhornio/longhorn-manager:v1.0.2
+        image: longhornio/longhorn-manager:v1.1.0
         imagePullPolicy: Always
         command:
         - longhorn-manager