From e8ea27f8590b0f2f92b57c19ce4c8587430756f1 Mon Sep 17 00:00:00 2001 From: Michal Berger Date: Tue, 3 Aug 2021 09:16:49 +0200 Subject: [PATCH] docker: Add docker-compose for building basic SPDK containers This suite can be used to deploy containers with the following functionality (more details in README.md): - storage-target - proxy-container - traffic-generator This will run simple fio test as per fio.conf against nvmf controller provided by initiator-container. Similar task can be performed directly from initiator-container as well. Each container includes SPDK installation with most common tools, e.g. rpc.py, available under $PATH. This allows for something like: docker-compose exec storage-target rpc.py nvmf_get_subsystems Note that SPDK environment heavily depends on a running kernel hence all the containers need to be privileged. That said, to make sure containers are not affecting the host too much, some tasks must be done prior running them. This includes: - loading proper kernel modules (like nvme-fabrics, etc.) - allocating hugepages and having at least one hugetlbfs mount available under /dev/hugepages base_build is created as docker multi-stage build. This is done in order to decrease the size of the final image. The SPDK RPMs are built inside a base image and then copied over to the main image (+ fio binary) - this leaves all the dependencies inside the intermediate image instead of the final one. The resulted difference in size may look similar to the following (it may differ depending on the docker version etc.): no multi-stage build: spdk_base == 1.04GB multi-stage build: spdk_base == 261MB Signed-off-by: Michal Berger Signed-off-by: Tomasz Zawadzki Change-Id: I825bd0d0bb4071bd9d44b6a0749c033894899ae0 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9055 Reviewed-by: Jim Harris Reviewed-by: Monica Kenguva Reviewed-by: Xiaodong Liu Reviewed-by: Changpeng Liu Tested-by: SPDK CI Jenkins Community-CI: Broadcom CI --- docker/README.md | 94 ++++++++++++++++++++++++++++ docker/build_base/Dockerfile | 40 ++++++++++++ docker/build_base/post-install | 20 ++++++ docker/build_base/pre-install | 43 +++++++++++++ docker/docker-compose.yaml | 62 ++++++++++++++++++ docker/spdk-app/Dockerfile | 17 +++++ docker/spdk-app/init | 32 ++++++++++ docker/spdk-app/proxy-container.conf | 68 ++++++++++++++++++++ docker/spdk-app/storage-target.conf | 63 +++++++++++++++++++ docker/traffic-generator/Dockerfile | 17 +++++ docker/traffic-generator/conf | 22 +++++++ docker/traffic-generator/fio.conf | 16 +++++ docker/traffic-generator/init | 14 +++++ rpmbuild/spdk.spec | 1 + 14 files changed, 509 insertions(+) create mode 100644 docker/README.md create mode 100644 docker/build_base/Dockerfile create mode 100755 docker/build_base/post-install create mode 100755 docker/build_base/pre-install create mode 100644 docker/docker-compose.yaml create mode 100644 docker/spdk-app/Dockerfile create mode 100755 docker/spdk-app/init create mode 100644 docker/spdk-app/proxy-container.conf create mode 100644 docker/spdk-app/storage-target.conf create mode 100644 docker/traffic-generator/Dockerfile create mode 100644 docker/traffic-generator/conf create mode 100644 docker/traffic-generator/fio.conf create mode 100755 docker/traffic-generator/init diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..9b79b8f42 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,94 @@ +# SPDK Docker suite + +This suite is meant to serve as an example of how SPDK can be encapsulated +into docker container images. The example containers consist of SPDK NVMe-oF +target sharing devices to another SPDK NVMe-oF application. Which serves +as both initiator and target. Finally a traffic generator based on FIO +issues I/O to the connected devices. + +## Prerequisites + +docker: We recommend version 20.10 and above because it supports cgroups v2 for +customization of host resources like CPUs, memory, and block I/O. + +docker-compose: We recommend using 1.29.2 version or newer. + +kernel: Hugepages must be allocated prior running the containers and hugetlbfs +mount must be available under /dev/hugepages. Also, tmpfs should be mounted +under /dev/shm. Depending on the use-case, some kernel modules should be also +loaded into the kernel prior running the containers. + +proxy: If you are working behind firewall make sure dockerd is aware of the +proxy. Please refer to: +[docker-proxy](https://docs.docker.com/config/daemon/systemd/#httphttps-proxy) + +To pass `$http_proxy` to docker-compose build use: +~~~{.sh} +docker-compose build --build-arg PROXY=$http_proxy +~~~ + +## How-To + +`docker-compose.yaml` shows an example deployment of the storage containers based on SPDK. +Running `docker-compose build` creates 4 docker images: + +- build_base +- storage-target +- proxy-container +- traffic-generator + +The `build_base` image provides the core components required to containerize SPDK +applications. The fedora:33 image from the Fedora Container Registry is used and then SPDK is installed. SPDK is installed out of `build_base/spdk.tar.gz` provided. +See `build_base` folder for details on what's included in the final image. + +Running `docker-compose up` creates 3 docker containers: + +-- storage-target: Contains SPDK NVMe-oF target exposing single subsystem to +`proxy-container` based on malloc bdev. +-- proxy-container: Contains SPDK NVMe-oF target connecting to `storage-target` +and then exposing the same subsystem to `traffic-generator`. +-- traffic-generator: Contains FIO using SPDK plugin to connect to `proxy-container` +and runs a sample workload. + +Each container is connected to a separate "spdk" network which is created before +deploying the containers. See `docker-compose.yaml` for the network's detailed setup and ip assignment. + +All the above boils down to: + +~~~{.sh} +cd docker +tar -czf build_base/spdk.tar.gz --exclude='docker/*' -C .. . +docker-compose build +docker-compose up +~~~ + +The `storage-target` and `proxy-container` can be started as services. +Allowing for multiple `traffic-generator` containers to connect. + +~~~{.sh} +docker-compose up -d proxy-container +docker-compose run traffic-generator +~~~ + +Enviroment variables to containers can be passed as shown in +[docs](https://docs.docker.com/compose/environment-variables/). +For example extra arguments to fio can be passed as so: + +~~~{.sh} +docker-compose run -e FIO_ARGS="--minimal" traffic-generator +~~~ + +As each container includes SPDK installation it is possible to use rpc.py to +examine the final setup. E.g.: + +~~~{.sh} +docker-compose exec storage-target rpc.py bdev_get_bdevs +docker-compose exec proxy-container rpc.py nvmf_get_subsystems +~~~ + +## Caveats + +- If you run docker < 20.10 under distro which switched fully to cgroups2 + (e.g. f33) make sure that /sys/fs/cgroup/systemd exists otherwise docker/build + will simply fail. +- Each SPDK app inside the containers is limited to single, separate CPU. diff --git a/docker/build_base/Dockerfile b/docker/build_base/Dockerfile new file mode 100644 index 000000000..1be076ec6 --- /dev/null +++ b/docker/build_base/Dockerfile @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) Intel Corporation + +FROM fedora:33 AS base + +# Generic args +ARG PROXY +ARG NO_PROXY + +ENV http_proxy=$PROXY +ENV https_proxy=$PROXY +ENV no_proxy=$NO_PROXY + + +COPY spdk.tar.gz /spdk.tar.gz +COPY pre-install /install +RUN /install + +# We are doing a multi-stage build here. This means that previous image, +# base, is going to end up as an intermediate one, untagged, - this +# image can be then manually removed (--force-rm doesn't work here. Go +# figure). +FROM fedora:33 AS spdk + +LABEL maintainer=spdk.io + +# Proxy configuration must be set for each build separately... +ARG PROXY +ARG NO_PROXY + +ENV http_proxy=$PROXY +ENV https_proxy=$PROXY +ENV no_proxy=$NO_PROXY + +# Copy SPDK's RPMs built during pre-install step. +COPY --from=base /tmp/*.rpm /tmp/ +COPY --from=base /tmp/fio /tmp/ +# Wrap up the image +COPY post-install /install +RUN /install diff --git a/docker/build_base/post-install b/docker/build_base/post-install new file mode 100755 index 000000000..6b843bacc --- /dev/null +++ b/docker/build_base/post-install @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -e + +dnf install -y /tmp/*.rpm + +# Be nice for docker exec and link SPDK scripts|binaries under common PATH +# location like /usr/bin. +ln -sf $(ls -1dpA /usr/local/bin/* | grep -v "/$") /usr/bin +ln -sf $(ls -1dpA /usr/local/bin/fio/* | grep -v "/$") /usr/bin +ln -s /usr/libexec/spdk/scripts/rpc.py /usr/bin +ln -s /usr/libexec/spdk/scripts/rpc_http_proxy.py /usr/bin +ln -s /usr/libexec/spdk/scripts/setup.sh /usr/bin +ln -s /usr/libexec/spdk/include/spdk /usr/include +ln -s /usr/libexec/spdk/scripts/ /usr + +mkdir -p /usr/src/fio +mv /tmp/fio /usr/src/fio + +dnf clean all +rm -f /tmp/*.rpm diff --git a/docker/build_base/pre-install b/docker/build_base/pre-install new file mode 100755 index 000000000..368c6fcb4 --- /dev/null +++ b/docker/build_base/pre-install @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -e + +spdk_repo=$(mktemp -dt "spdk.XXXXXX") +spdk_tar=/spdk.tar.gz + +cleanup() { + + rm -f "$HOME/rpmbuild/rpm/x86_64/"*.rpm + rm -f "$spdk_tar" + rm -rf "$spdk_repo" +} + +trap 'cleanup' EXIT + +if [[ ! -e $spdk_tar ]]; then + printf 'Missing %s\n' "$spdk_tar" >&2 + exit 1 +fi + +tar -C "$spdk_repo" -xf "$spdk_tar" + +# Required for building RPM +dnf install -y rpm-build + +# Spice it a bit with supported sources +"$spdk_repo/scripts/pkgdep.sh" -d +"$spdk_repo/test/common/config/vm_setup.sh" --test-conf=fio + +# HACK: In case we received a .tar with built SPDK we need to overwrite the +# configuration to update all the paths make would need to lookup - this is +# needed since we execute inside a different mount namespace so we won't be +# able to find any absoulte paths that were used prior creating the .tar. +"$spdk_repo/configure" > /dev/null + +# Deploy SPDK inside the container +DEPS="no" "$spdk_repo/rpmbuild/rpm.sh" \ + --with-shared \ + --with-fio + +mv "$HOME/rpmbuild/rpm/x86_64/"*.rpm /tmp +mv "/usr/src/fio/fio" /tmp +dnf clean all diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 000000000..3b86cede0 --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) Intel Corporation +version: "3.8" +services: + build_base: + image: spdk + build: + context: build_base + container_name: build_base + storage-target: + image: spdk-app + build: + context: spdk-app + container_name: storage-target + depends_on: + - build_base + networks: + spdk: + ipv4_address: 192.168.42.2 + volumes: + - /dev/hugepages:/dev/hugepages + - ./spdk-app/storage-target.conf:/config + environment: + - SPDK_ARGS=-m 0x2 + privileged: true + proxy-container: + image: spdk-app + build: + context: spdk-app + container_name: proxy-container + depends_on: + - storage-target + networks: + spdk: + ipv4_address: 192.168.42.3 + volumes: + - /dev/hugepages:/dev/hugepages + - ./spdk-app/proxy-container.conf:/config + environment: + - SPDK_ARGS=-m 0x4 + privileged: true + traffic-generator: + image: traffic-generator + build: + context: traffic-generator + container_name: traffic-generator + depends_on: + - proxy-container + networks: + spdk: + volumes: + - /dev/hugepages:/dev/hugepages + - ./traffic-generator/conf:/config + - ./traffic-generator/fio.conf:/fio.conf + privileged: true +networks: + spdk: + name: "spdk" + ipam: + config: + - subnet: 192.168.42.0/29 + gateway: 192.168.42.1 diff --git a/docker/spdk-app/Dockerfile b/docker/spdk-app/Dockerfile new file mode 100644 index 000000000..ac54cbd50 --- /dev/null +++ b/docker/spdk-app/Dockerfile @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) Intel Corporation + +FROM spdk + +# Generic args +ARG PROXY +ARG NO_PROXY + +ENV http_proxy=$PROXY +ENV https_proxy=$PROXY +ENV no_proxy=$NO_PROXY + + +COPY init /init + +ENTRYPOINT ["/init"] diff --git a/docker/spdk-app/init b/docker/spdk-app/init new file mode 100755 index 000000000..a9d933fa9 --- /dev/null +++ b/docker/spdk-app/init @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +app=spdk_tgt args=() limit_args=() + +# Override default app +if [[ -n $SPDK_APP ]]; then + app=$SPDK_APP +fi + +# Define extra arguments to the app +if [[ -n $SPDK_ARGS ]]; then + args=($SPDK_ARGS) +fi + +# Limit the app with to following options, +# to allow for minimal impact on the host. +limit_args+=("--no-pci") +limit_args+=("--num-trace-entries" 0) + +# if set, don't include limit_args[] on the cmdline +if [[ ! -v SPDK_NO_LIMIT ]]; then + args+=("${limit_args[@]}") +fi + +if [[ -e /config ]]; then + args+=("--json" "/config") +fi + +# Wait a bit to make sure ip is in place +sleep 2s + +exec "$app" "${args[@]}" diff --git a/docker/spdk-app/proxy-container.conf b/docker/spdk-app/proxy-container.conf new file mode 100644 index 000000000..416d8d92b --- /dev/null +++ b/docker/spdk-app/proxy-container.conf @@ -0,0 +1,68 @@ +{ + "subsystems": [ + { + "subsystem": "bdev", + "config": [ + { + "method": "bdev_nvme_attach_controller", + "params": { + "name": "Nvme0", + "trtype": "TCP", + "adrfam": "IPv4", + "traddr": "192.168.42.2", + "trsvcid": "4420", + "subnqn": "nqn.2016-06.io.spdk:cnode1", + "prchk_reftag": false, + "prchk_guard": false + } + } + ] + }, + { + "subsystem": "nvmf", + "config": [ + { + "method": "nvmf_create_transport", + "params": { + "trtype": "TCP", + "io_unit_size": 8192 + } + }, + { + "method": "nvmf_create_subsystem", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "allow_any_host": true, + "serial_number": "SPDK00000000000001", + "model_number": "SPDK bdev Controller", + "max_namespaces": 32, + "min_cntlid": 1, + "max_cntlid": 65519 + } + }, + { + "method": "nvmf_subsystem_add_ns", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "namespace": { + "nsid": 1, + "bdev_name": "Nvme0n1" + } + } + }, + { + "method": "nvmf_subsystem_add_listener", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "listen_address": { + "trtype": "TCP", + "adrfam": "IPv4", + "traddr": "192.168.42.3", + "trsvcid": "4420" + } + } + } + ] + } + ] +} diff --git a/docker/spdk-app/storage-target.conf b/docker/spdk-app/storage-target.conf new file mode 100644 index 000000000..1916e3974 --- /dev/null +++ b/docker/spdk-app/storage-target.conf @@ -0,0 +1,63 @@ +{ + "subsystems": [ + { + "subsystem": "bdev", + "config": [ + { + "method": "bdev_malloc_create", + "params": { + "name": "Malloc0", + "num_blocks": 131072, + "block_size": 512 + } + } + ] + }, + { + "subsystem": "nvmf", + "config": [ + { + "method": "nvmf_create_transport", + "params": { + "trtype": "TCP", + "io_unit_size": 8192 + } + }, + { + "method": "nvmf_create_subsystem", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "allow_any_host": true, + "serial_number": "SPDK00000000000001", + "model_number": "SPDK bdev Controller", + "max_namespaces": 32, + "min_cntlid": 1, + "max_cntlid": 65519 + } + }, + { + "method": "nvmf_subsystem_add_ns", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "namespace": { + "nsid": 1, + "bdev_name": "Malloc0" + } + } + }, + { + "method": "nvmf_subsystem_add_listener", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "listen_address": { + "trtype": "TCP", + "adrfam": "IPv4", + "traddr": "192.168.42.2", + "trsvcid": "4420" + } + } + } + ] + } + ] +} diff --git a/docker/traffic-generator/Dockerfile b/docker/traffic-generator/Dockerfile new file mode 100644 index 000000000..ac54cbd50 --- /dev/null +++ b/docker/traffic-generator/Dockerfile @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) Intel Corporation + +FROM spdk + +# Generic args +ARG PROXY +ARG NO_PROXY + +ENV http_proxy=$PROXY +ENV https_proxy=$PROXY +ENV no_proxy=$NO_PROXY + + +COPY init /init + +ENTRYPOINT ["/init"] diff --git a/docker/traffic-generator/conf b/docker/traffic-generator/conf new file mode 100644 index 000000000..2637169c4 --- /dev/null +++ b/docker/traffic-generator/conf @@ -0,0 +1,22 @@ +{ + "subsystems": [ + { + "subsystem": "bdev", + "config": [ + { + "method": "bdev_nvme_attach_controller", + "params": { + "name": "Nvme0", + "trtype": "TCP", + "adrfam": "IPv4", + "traddr": "192.168.42.3", + "trsvcid": "4420", + "subnqn": "nqn.2016-06.io.spdk:cnode1", + "prchk_reftag": false, + "prchk_guard": false + } + } + ] + } + ] +} diff --git a/docker/traffic-generator/fio.conf b/docker/traffic-generator/fio.conf new file mode 100644 index 000000000..7da647919 --- /dev/null +++ b/docker/traffic-generator/fio.conf @@ -0,0 +1,16 @@ +[global] +ioengine=spdk_bdev +spdk_json_conf=/config +thread=1 +direct=1 +rw=randread +ramp_time=0 +norandommap=1 +time_based=1 +bs=4k +numjobs=1 +runtime=10 + +[filename0] +filename=Nvme0n1 +iodepth=128 diff --git a/docker/traffic-generator/init b/docker/traffic-generator/init new file mode 100755 index 000000000..eb8850598 --- /dev/null +++ b/docker/traffic-generator/init @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +args=() + +# Define extra arguments to the app +if [[ -n $FIO_ARGS ]]; then + args+=($FIO_ARGS) +fi + +# Wait a bit to make sure ip is in place +sleep 2s + +export LD_PRELOAD=/usr/local/bin/fio/spdk_bdev +exec /usr/src/fio/fio "${args[@]}" /fio.conf diff --git a/rpmbuild/spdk.spec b/rpmbuild/spdk.spec index 1429914e6..508ef58c7 100644 --- a/rpmbuild/spdk.spec +++ b/rpmbuild/spdk.spec @@ -75,6 +75,7 @@ cp -a %{dpdk_build_path}/lib/* %{buildroot}/usr/local/lib/dpdk/ # Try to include all the binaries that were potentially built [[ -e build/examples ]] && cp -a build/examples/* %{buildroot}/usr/local/bin/ [[ -e build/bin ]] && cp -a build/bin/* %{buildroot}/usr/local/bin/ +[[ -e build/fio ]] && cp -a build/fio %{buildroot}/usr/local/bin/fio # And some useful setup scripts SPDK uses mkdir -p %{buildroot}/usr/libexec/spdk