From da9d0cac327a787d85812f091bfa7b7774d1d3dc Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Thu, 21 Mar 2019 18:50:30 -0700 Subject: [PATCH 1/7] Sync with manager commit 62a6ff4a9dfbfc2081917184815a8f71e549492b Author: Sheng Yang Date: Thu Mar 21 18:41:45 2019 -0700 Longhorn v0.4.1-rc1 release --- deploy/longhorn.yaml | 17 ++++++++++------- uninstall/uninstall.yaml | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/deploy/longhorn.yaml b/deploy/longhorn.yaml index 9f47f30..2474e61 100644 --- a/deploy/longhorn.yaml +++ b/deploy/longhorn.yaml @@ -181,7 +181,7 @@ spec: spec: containers: - name: longhorn-manager - image: rancher/longhorn-manager:v0.4.0 + image: rancher/longhorn-manager:v0.4.1-rc1 imagePullPolicy: Always securityContext: privileged: true @@ -190,9 +190,9 @@ spec: - -d - daemon - --engine-image - - rancher/longhorn-engine:v0.4.0 + - rancher/longhorn-engine:v0.4.1-rc1 - --manager-image - - rancher/longhorn-manager:v0.4.0 + - rancher/longhorn-manager:v0.4.1-rc1 - --service-account - longhorn-service-account ports: @@ -269,7 +269,7 @@ spec: spec: containers: - name: longhorn-ui - image: rancher/longhorn-ui:v0.4.0 + image: rancher/longhorn-ui:v0.4.1-rc1 ports: - containerPort: 8000 env: @@ -308,23 +308,26 @@ spec: spec: initContainers: - name: wait-longhorn-manager - image: rancher/longhorn-manager:v0.4.0 + image: rancher/longhorn-manager:v0.4.1-rc1 command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done'] containers: - name: longhorn-driver-deployer - image: rancher/longhorn-manager:v0.4.0 + image: rancher/longhorn-manager:v0.4.1-rc1 imagePullPolicy: Always command: - longhorn-manager - -d - deploy-driver - --manager-image - - rancher/longhorn-manager:v0.4.0 + - rancher/longhorn-manager:v0.4.1-rc1 - --manager-url - http://longhorn-backend:9500/v1 # manually choose "flexvolume" or "csi" #- --driver #- flexvolume + # manually set root directory for csi + #- --kubelet-root-dir + #- /var/lib/rancher/k3s/agent/kubelet env: - name: POD_NAMESPACE valueFrom: diff --git a/uninstall/uninstall.yaml b/uninstall/uninstall.yaml index e13d1a2..2d15334 100644 --- a/uninstall/uninstall.yaml +++ b/uninstall/uninstall.yaml @@ -12,7 +12,7 @@ spec: spec: containers: - name: longhorn-uninstall - image: rancher/longhorn-manager:v0.4.0 + image: rancher/longhorn-manager:v0.4.1-rc1 imagePullPolicy: Always command: - longhorn-manager From 14b194d8265b5b4e8414a1274a6b6f014ebf5011 Mon Sep 17 00:00:00 2001 From: Shuo Wu Date: Thu, 21 Mar 2019 23:44:25 +0000 Subject: [PATCH 2/7] Add rancheros.md --- README.md | 1 + docs/rancheros.md | 64 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 docs/rancheros.md diff --git a/README.md b/README.md index 2fd3092..cdf5a34 100644 --- a/README.md +++ b/README.md @@ -264,6 +264,7 @@ Longhorn will always try to maintain at least given number of healthy replicas f ### [Google Kubernetes Engine](./docs/gke.md) ### [Upgrade](./docs/upgrade.md) ### [Deal with Kubernetes node failure](./docs/node-failure.md) +### [Use CSI driver on RancherOS + RKE](./docs/rancheros.md) ## Troubleshooting You can click `Generate Support Bundle` link at the bottom of the UI to download a zip file contains Longhorn related configuration and logs. diff --git a/docs/rancheros.md b/docs/rancheros.md new file mode 100644 index 0000000..3ce64f7 --- /dev/null +++ b/docs/rancheros.md @@ -0,0 +1,64 @@ +# Longhorn CSI on RancherOS + RKE + +## Requirements + 1. Kubernetes v1.11 or higher. + 2. Longhorn v0.4.1 or higher. + 3. RancherOS Ubuntu console. + + +## Instruction +### For Kubernetes v1.11 only + The following step is not needed for Kubernetes v1.12+. + + Add extra_binds for kubelet in RKE `cluster.yml`: + ``` + services: + kubelet: + extra_binds: + - "/opt/rke/var/lib/kubelet/plugins:/var/lib/kubelet/plugins" + ``` + +### For each node: + #### 1. Switch to ubuntu console + + `sudo ros console switch ubuntu`, then type `y` + + #### 2. Install open-iscsi for each node. + ``` + sudo su + apt update + apt install -y open-iscsi + ``` + #### 3. Modify configuration for iscsi. + + 1. Open config file `/etc/iscsi/iscsid.conf` + 2. Comment `iscsid.startup = /bin/systemctl start iscsid.socket` + 3. Uncomment `iscsid.startup = /sbin/iscsid` + + + +## Background +CSI doesn't work with RancherOS + RKE before Longhorn v0.4.1. The reason is: + +1. RancherOS sets argument `root-dir=/opt/rke/var/lib/kubelet` for kubelet, , which is different from the default value `/var/lib/kubelet`. + +2. **For k8s v1.12+** + + Kubelet will detect the `csi.sock` according to argument `<--kubelet-registration-path>` passed in by Kubernetes CSI driver-registrar, and `-reg.sock` (for Longhorn, it's `io.rancher.longhorn-reg.sock`) on kubelet path `/plugins`. + + **For k8s v1.11** + + Kubelet will find both sockets on kubelet path `/var/lib/kubelet/plugins`. + +3. By default, Longhorn CSI driver create and expose these 2 sock files on host path `/var/lib/kubelet/plugins`. + +4. Then kubelet cannot find `-reg.sock`, so CSI driver doesn't work. + +5. Furthermore, kubelet will instruct CSI plugin to mount Longhorn volume on `/pods//volumes/kubernetes.io~csi//mount`. + + But this path inside CSI plugin container won't be binded mount on host path. And the mount operation for Longhorn volume is meaningless. + + Hence Kubernetes cannot connect to Longhorn using CSI driver. + +## Reference +https://github.com/kubernetes-csi/driver-registrar \ No newline at end of file From ef96541f0dddcdc36a11c142c4ad376ce31002a8 Mon Sep 17 00:00:00 2001 From: Shuo Wu Date: Sat, 23 Mar 2019 01:31:52 +0000 Subject: [PATCH 3/7] Add doc restore-to-file.md --- README.md | 1 + docs/restore-to-file.md | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 docs/restore-to-file.md diff --git a/README.md b/README.md index cdf5a34..330ff11 100644 --- a/README.md +++ b/README.md @@ -265,6 +265,7 @@ Longhorn will always try to maintain at least given number of healthy replicas f ### [Upgrade](./docs/upgrade.md) ### [Deal with Kubernetes node failure](./docs/node-failure.md) ### [Use CSI driver on RancherOS + RKE](./docs/rancheros.md) +### [Restore a backup to an image file](./docs/restore-to-file.md) ## Troubleshooting You can click `Generate Support Bundle` link at the bottom of the UI to download a zip file contains Longhorn related configuration and logs. diff --git a/docs/restore-to-file.md b/docs/restore-to-file.md new file mode 100644 index 0000000..bbdf782 --- /dev/null +++ b/docs/restore-to-file.md @@ -0,0 +1,23 @@ +# Use command restore-to-file +This command gives users the ability to restore a backup to a `raw` image or a `qcow2` image. If the backup is based on a backing file, users should provide the backing file as a `qcow2` image with `--backing file` parameter. + +## Instruction +1. Copy the yaml template + + 1.1 Without backing file: Make a copy of `examples/restore_to_file.yaml.template` as e.g. `restore.yaml`. + + 1.2 With backing file: Make a copy of `examples/restore_to_file_with_backing_file.yaml.template` as e.g. `restore.yaml`. + + 1.2.1 Set argument `backing-file` by replacing `` with your base image, e.g. `rancher/longhorn-test:baseimage-ext4`. + +1. Set host node on which the output file should be placed by replacing ``. + +2. Specify the host path of output file by modifying field `hostpath` of volume `disk-directory`. By default the directory is `/tmp/restore/`. + +3. Set the first argument (backup url) by replacing ``, e.g. `s3://backupbucket@us-east-1/backupstore?backup=backup-bd326da2c4414b02&volume=volumeexamplename`. Do not delete `''`. + +4. Set argument `output-file` by replacing ``, e.g. `volume.raw` or `volume.qcow2`. + +5. Set argument `output-format` by replacing ``. Now support `raw` or `qcow2` only. + +6. Set S3 Credential Secret by replacing ``, e.g. `minio-secret`. From 6f2d7d0170b5f29aade6b764e220ec3a8aa27cc7 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Fri, 22 Mar 2019 19:23:16 -0700 Subject: [PATCH 4/7] Update restore-to-file.md --- docs/restore-to-file.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/restore-to-file.md b/docs/restore-to-file.md index bbdf782..e030f08 100644 --- a/docs/restore-to-file.md +++ b/docs/restore-to-file.md @@ -4,20 +4,24 @@ This command gives users the ability to restore a backup to a `raw` image or a ` ## Instruction 1. Copy the yaml template - 1.1 Without backing file: Make a copy of `examples/restore_to_file.yaml.template` as e.g. `restore.yaml`. - - 1.2 With backing file: Make a copy of `examples/restore_to_file_with_backing_file.yaml.template` as e.g. `restore.yaml`. + 1.1 Volume has no base image: Make a copy of `examples/restore_to_file.yaml.template` as e.g. `restore.yaml`. - 1.2.1 Set argument `backing-file` by replacing `` with your base image, e.g. `rancher/longhorn-test:baseimage-ext4`. + 1.2 Volume has a base image: Make a copy of `examples/restore_to_file_with_backing_file.yaml.template` as e.g. `restore.yaml`, and set argument `backing-file` by replacing `` with your base image, e.g. `rancher/longhorn-test:baseimage-ext4`. -1. Set host node on which the output file should be placed by replacing ``. +2. Set the node which the output file should be placed on by replacing ``, e.g. `node1`. -2. Specify the host path of output file by modifying field `hostpath` of volume `disk-directory`. By default the directory is `/tmp/restore/`. +3. Specify the host path of output file by modifying field `hostpath` of volume `disk-directory`. By default the directory is `/tmp/restore/`. -3. Set the first argument (backup url) by replacing ``, e.g. `s3://backupbucket@us-east-1/backupstore?backup=backup-bd326da2c4414b02&volume=volumeexamplename`. Do not delete `''`. +4. Set the first argument (backup url) by replacing ``, e.g. `s3://backupbucket@us-east-1/backupstore?backup=backup-bd326da2c4414b02&volume=volumeexamplename`. Do not delete `''`. -4. Set argument `output-file` by replacing ``, e.g. `volume.raw` or `volume.qcow2`. +5. Set argument `output-file` by replacing ``, e.g. `volume.raw` or `volume.qcow2`. -5. Set argument `output-format` by replacing ``. Now support `raw` or `qcow2` only. +6. Set argument `output-format` by replacing ``. Now support `raw` or `qcow2` only. -6. Set S3 Credential Secret by replacing ``, e.g. `minio-secret`. +7. Set S3 Credential Secret by replacing ``, e.g. `minio-secret`. + +8. Execute the yaml using e.g. `kubectl create -f restore.yaml`. + +9. Watching the result using `kubectl -n longhorn-system get pod restore-to-file -w` + +After the pod status changed to `Completed`, you should able to find `` at e.g. `/tmp/restore` on the ``. From 0ac92d5bacb78fc73ac548f364bc10b9c32353b6 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Fri, 22 Mar 2019 19:25:35 -0700 Subject: [PATCH 5/7] Update restore-to-file.md --- docs/restore-to-file.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/restore-to-file.md b/docs/restore-to-file.md index e030f08..9338d2a 100644 --- a/docs/restore-to-file.md +++ b/docs/restore-to-file.md @@ -6,7 +6,7 @@ This command gives users the ability to restore a backup to a `raw` image or a ` 1.1 Volume has no base image: Make a copy of `examples/restore_to_file.yaml.template` as e.g. `restore.yaml`. - 1.2 Volume has a base image: Make a copy of `examples/restore_to_file_with_backing_file.yaml.template` as e.g. `restore.yaml`, and set argument `backing-file` by replacing `` with your base image, e.g. `rancher/longhorn-test:baseimage-ext4`. + 1.2 Volume has a base image: Make a copy of `examples/restore_to_file_with_base_image.yaml.template` as e.g. `restore.yaml`, and set argument `backing-file` by replacing `` with your base image, e.g. `rancher/longhorn-test:baseimage-ext4`. 2. Set the node which the output file should be placed on by replacing ``, e.g. `node1`. From 69e6a333f9352f4fe8ff7bc341ca7f72ceb2ca99 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Fri, 22 Mar 2019 20:04:26 -0700 Subject: [PATCH 6/7] Sync with manager commit de3962d5a0b99b70f440dc4cef2c43773097026f Author: Sheng Yang Date: Fri Mar 22 19:51:13 2019 -0700 Longhorn v0.4.1 release --- deploy/longhorn.yaml | 14 +-- examples/restore_to_file.yaml.template | 48 ++++++++++ ...tore_to_file_with_base_image.yaml.template | 94 +++++++++++++++++++ uninstall/uninstall.yaml | 2 +- 4 files changed, 150 insertions(+), 8 deletions(-) create mode 100644 examples/restore_to_file.yaml.template create mode 100644 examples/restore_to_file_with_base_image.yaml.template diff --git a/deploy/longhorn.yaml b/deploy/longhorn.yaml index 2474e61..62c4884 100644 --- a/deploy/longhorn.yaml +++ b/deploy/longhorn.yaml @@ -181,7 +181,7 @@ spec: spec: containers: - name: longhorn-manager - image: rancher/longhorn-manager:v0.4.1-rc1 + image: rancher/longhorn-manager:v0.4.1 imagePullPolicy: Always securityContext: privileged: true @@ -190,9 +190,9 @@ spec: - -d - daemon - --engine-image - - rancher/longhorn-engine:v0.4.1-rc1 + - rancher/longhorn-engine:v0.4.1 - --manager-image - - rancher/longhorn-manager:v0.4.1-rc1 + - rancher/longhorn-manager:v0.4.1 - --service-account - longhorn-service-account ports: @@ -269,7 +269,7 @@ spec: spec: containers: - name: longhorn-ui - image: rancher/longhorn-ui:v0.4.1-rc1 + image: rancher/longhorn-ui:v0.4.1 ports: - containerPort: 8000 env: @@ -308,18 +308,18 @@ spec: spec: initContainers: - name: wait-longhorn-manager - image: rancher/longhorn-manager:v0.4.1-rc1 + image: rancher/longhorn-manager:v0.4.1 command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done'] containers: - name: longhorn-driver-deployer - image: rancher/longhorn-manager:v0.4.1-rc1 + image: rancher/longhorn-manager:v0.4.1 imagePullPolicy: Always command: - longhorn-manager - -d - deploy-driver - --manager-image - - rancher/longhorn-manager:v0.4.1-rc1 + - rancher/longhorn-manager:v0.4.1 - --manager-url - http://longhorn-backend:9500/v1 # manually choose "flexvolume" or "csi" diff --git a/examples/restore_to_file.yaml.template b/examples/restore_to_file.yaml.template new file mode 100644 index 0000000..7bfb6e3 --- /dev/null +++ b/examples/restore_to_file.yaml.template @@ -0,0 +1,48 @@ +apiVersion: v1 +kind: Pod +metadata: + name: restore-to-file + namespace: longhorn-system +spec: + nodeName: + containers: + - name: restore-to-file + command: + # set restore-to-file arguments here + - /bin/sh + - -c + - longhorn backup restore-to-file + '' + --output-file '/tmp/restore/' + --output-format + # the version of longhorn engine should be v0.4.1 or higher + image: rancher/longhorn-engine:v0.4.1 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + volumeMounts: + - name: disk-directory + mountPath: /tmp/restore # the argument should be in this directory + env: + # set Backup Target Credential Secret here. + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: + key: AWS_SECRET_ACCESS_KEY + - name: AWS_ENDPOINTS + valueFrom: + secretKeyRef: + name: + key: AWS_ENDPOINTS + volumes: + # the output file can be found on this host path + - name: disk-directory + hostPath: + path: /tmp/restore + restartPolicy: Never diff --git a/examples/restore_to_file_with_base_image.yaml.template b/examples/restore_to_file_with_base_image.yaml.template new file mode 100644 index 0000000..9a7e154 --- /dev/null +++ b/examples/restore_to_file_with_base_image.yaml.template @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: Pod +metadata: + name: restore-to-file + namespace: longhorn-system +spec: + nodeName: + initContainers: + - name: prime-base-image + # set base image here + command: + - /bin/sh + - -c + - echo primed-base-image + # set base image here + image: + imagePullPolicy: Always + containers: + - name: base-image + command: + - /bin/sh + - -c + - mkdir -p /share/base_image && + mount --bind /base_image/ /share/base_image && + echo base image mounted at /share/base_image && + trap 'umount /share/base_image && echo unmounted' TERM && + while true; do $(ls /talk/done 2>&1); if [ $? -eq 0 ]; then break; + fi; echo waiting; sleep 1; done; + umount /share/base_image && echo unmounted + # set base image here + image: + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + volumeMounts: + - name: share + mountPath: /share + mountPropagation: Bidirectional + - name: talk + mountPath: /talk + - name: restore-to-file + command: + # set restore-to-file arguments here + - /bin/sh + - -c + - while true; do list=$(ls /share/base_image/* 2>&1); if [ $? -eq 0 ]; then break; + fi; echo waiting; sleep 1; done; echo Directory found $list; + longhorn backup restore-to-file + '' + --backing-file $list + --output-file '/tmp/restore/' + --output-format + && touch /talk/done && chmod 777 /talk/done && echo created /share/done + # the version of longhorn engine should be v0.4.1 or higher + image: rancher/longhorn-engine:v0.4.1 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + volumeMounts: + - name: share + mountPath: /share + mountPropagation: HostToContainer + readOnly: true + - name: talk + mountPath: /talk + - name: disk-directory + mountPath: /tmp/restore # the argument should be in this directory + env: + # set Backup Target Credential Secret here. + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: + key: AWS_SECRET_ACCESS_KEY + - name: AWS_ENDPOINTS + valueFrom: + secretKeyRef: + name: + key: AWS_ENDPOINTS + volumes: + - name: share + emptyDir: {} + - name: talk + emptyDir: {} + # the output file can be found on this host path + - name: disk-directory + hostPath: + path: /tmp/restore + restartPolicy: Never diff --git a/uninstall/uninstall.yaml b/uninstall/uninstall.yaml index 2d15334..64c9209 100644 --- a/uninstall/uninstall.yaml +++ b/uninstall/uninstall.yaml @@ -12,7 +12,7 @@ spec: spec: containers: - name: longhorn-uninstall - image: rancher/longhorn-manager:v0.4.1-rc1 + image: rancher/longhorn-manager:v0.4.1 imagePullPolicy: Always command: - longhorn-manager From 2c07481b75ff2f8cadd59c227f8bd4166ad8aa17 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Fri, 22 Mar 2019 20:06:04 -0700 Subject: [PATCH 7/7] Longhorn v0.4.1 release Longhorn v0.4.1 has added support for RancherOS and K3S. Highlights: 1. Support for RancherOS and K3S. See https://github.com/rancher/longhorn/blob/v0.4.1/docs/rancheros.md 2. Support for restore a backup into an image file. See https://github.com/rancher/longhorn/blob/v0.4.1/docs/restore-to-file.md 3. Improve logging mechanism across the board. See all the issues resolved in v0.4.1 at: https://github.com/rancher/longhorn/milestone/10?closed=1 The volume engines would need to upgrade to v0.4.1 as well. Please follow the instruction to upgrade engine. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 330ff11..47caaa5 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ You can read more details of Longhorn and its design [here](http://rancher.com/m Longhorn is a work in progress. It's an alpha quality software at the moment. We appreciate your comments as we continue to work on it. -The latest release of Longhorn is **v0.4.0**, shipped with Longhorn Engine **v0.4.0** as the default engine image. +The latest release of Longhorn is **v0.4.1**, shipped with Longhorn Engine **v0.4.1** as the default engine image. ## Source code Longhorn is 100% open source software. Project source code is spread across a number of repos: