diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..9b79b8f42 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,94 @@ +# SPDK Docker suite + +This suite is meant to serve as an example of how SPDK can be encapsulated +into docker container images. The example containers consist of SPDK NVMe-oF +target sharing devices to another SPDK NVMe-oF application. Which serves +as both initiator and target. Finally a traffic generator based on FIO +issues I/O to the connected devices. + +## Prerequisites + +docker: We recommend version 20.10 and above because it supports cgroups v2 for +customization of host resources like CPUs, memory, and block I/O. + +docker-compose: We recommend using 1.29.2 version or newer. + +kernel: Hugepages must be allocated prior running the containers and hugetlbfs +mount must be available under /dev/hugepages. Also, tmpfs should be mounted +under /dev/shm. Depending on the use-case, some kernel modules should be also +loaded into the kernel prior running the containers. + +proxy: If you are working behind firewall make sure dockerd is aware of the +proxy. Please refer to: +[docker-proxy](https://docs.docker.com/config/daemon/systemd/#httphttps-proxy) + +To pass `$http_proxy` to docker-compose build use: +~~~{.sh} +docker-compose build --build-arg PROXY=$http_proxy +~~~ + +## How-To + +`docker-compose.yaml` shows an example deployment of the storage containers based on SPDK. +Running `docker-compose build` creates 4 docker images: + +- build_base +- storage-target +- proxy-container +- traffic-generator + +The `build_base` image provides the core components required to containerize SPDK +applications. The fedora:33 image from the Fedora Container Registry is used and then SPDK is installed. SPDK is installed out of `build_base/spdk.tar.gz` provided. +See `build_base` folder for details on what's included in the final image. + +Running `docker-compose up` creates 3 docker containers: + +-- storage-target: Contains SPDK NVMe-oF target exposing single subsystem to +`proxy-container` based on malloc bdev. +-- proxy-container: Contains SPDK NVMe-oF target connecting to `storage-target` +and then exposing the same subsystem to `traffic-generator`. +-- traffic-generator: Contains FIO using SPDK plugin to connect to `proxy-container` +and runs a sample workload. + +Each container is connected to a separate "spdk" network which is created before +deploying the containers. See `docker-compose.yaml` for the network's detailed setup and ip assignment. + +All the above boils down to: + +~~~{.sh} +cd docker +tar -czf build_base/spdk.tar.gz --exclude='docker/*' -C .. . +docker-compose build +docker-compose up +~~~ + +The `storage-target` and `proxy-container` can be started as services. +Allowing for multiple `traffic-generator` containers to connect. + +~~~{.sh} +docker-compose up -d proxy-container +docker-compose run traffic-generator +~~~ + +Enviroment variables to containers can be passed as shown in +[docs](https://docs.docker.com/compose/environment-variables/). +For example extra arguments to fio can be passed as so: + +~~~{.sh} +docker-compose run -e FIO_ARGS="--minimal" traffic-generator +~~~ + +As each container includes SPDK installation it is possible to use rpc.py to +examine the final setup. E.g.: + +~~~{.sh} +docker-compose exec storage-target rpc.py bdev_get_bdevs +docker-compose exec proxy-container rpc.py nvmf_get_subsystems +~~~ + +## Caveats + +- If you run docker < 20.10 under distro which switched fully to cgroups2 + (e.g. f33) make sure that /sys/fs/cgroup/systemd exists otherwise docker/build + will simply fail. +- Each SPDK app inside the containers is limited to single, separate CPU. diff --git a/docker/build_base/Dockerfile b/docker/build_base/Dockerfile new file mode 100644 index 000000000..1be076ec6 --- /dev/null +++ b/docker/build_base/Dockerfile @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) Intel Corporation + +FROM fedora:33 AS base + +# Generic args +ARG PROXY +ARG NO_PROXY + +ENV http_proxy=$PROXY +ENV https_proxy=$PROXY +ENV no_proxy=$NO_PROXY + + +COPY spdk.tar.gz /spdk.tar.gz +COPY pre-install /install +RUN /install + +# We are doing a multi-stage build here. This means that previous image, +# base, is going to end up as an intermediate one, untagged, - this +# image can be then manually removed (--force-rm doesn't work here. Go +# figure). +FROM fedora:33 AS spdk + +LABEL maintainer=spdk.io + +# Proxy configuration must be set for each build separately... +ARG PROXY +ARG NO_PROXY + +ENV http_proxy=$PROXY +ENV https_proxy=$PROXY +ENV no_proxy=$NO_PROXY + +# Copy SPDK's RPMs built during pre-install step. +COPY --from=base /tmp/*.rpm /tmp/ +COPY --from=base /tmp/fio /tmp/ +# Wrap up the image +COPY post-install /install +RUN /install diff --git a/docker/build_base/post-install b/docker/build_base/post-install new file mode 100755 index 000000000..6b843bacc --- /dev/null +++ b/docker/build_base/post-install @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -e + +dnf install -y /tmp/*.rpm + +# Be nice for docker exec and link SPDK scripts|binaries under common PATH +# location like /usr/bin. +ln -sf $(ls -1dpA /usr/local/bin/* | grep -v "/$") /usr/bin +ln -sf $(ls -1dpA /usr/local/bin/fio/* | grep -v "/$") /usr/bin +ln -s /usr/libexec/spdk/scripts/rpc.py /usr/bin +ln -s /usr/libexec/spdk/scripts/rpc_http_proxy.py /usr/bin +ln -s /usr/libexec/spdk/scripts/setup.sh /usr/bin +ln -s /usr/libexec/spdk/include/spdk /usr/include +ln -s /usr/libexec/spdk/scripts/ /usr + +mkdir -p /usr/src/fio +mv /tmp/fio /usr/src/fio + +dnf clean all +rm -f /tmp/*.rpm diff --git a/docker/build_base/pre-install b/docker/build_base/pre-install new file mode 100755 index 000000000..368c6fcb4 --- /dev/null +++ b/docker/build_base/pre-install @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -e + +spdk_repo=$(mktemp -dt "spdk.XXXXXX") +spdk_tar=/spdk.tar.gz + +cleanup() { + + rm -f "$HOME/rpmbuild/rpm/x86_64/"*.rpm + rm -f "$spdk_tar" + rm -rf "$spdk_repo" +} + +trap 'cleanup' EXIT + +if [[ ! -e $spdk_tar ]]; then + printf 'Missing %s\n' "$spdk_tar" >&2 + exit 1 +fi + +tar -C "$spdk_repo" -xf "$spdk_tar" + +# Required for building RPM +dnf install -y rpm-build + +# Spice it a bit with supported sources +"$spdk_repo/scripts/pkgdep.sh" -d +"$spdk_repo/test/common/config/vm_setup.sh" --test-conf=fio + +# HACK: In case we received a .tar with built SPDK we need to overwrite the +# configuration to update all the paths make would need to lookup - this is +# needed since we execute inside a different mount namespace so we won't be +# able to find any absoulte paths that were used prior creating the .tar. +"$spdk_repo/configure" > /dev/null + +# Deploy SPDK inside the container +DEPS="no" "$spdk_repo/rpmbuild/rpm.sh" \ + --with-shared \ + --with-fio + +mv "$HOME/rpmbuild/rpm/x86_64/"*.rpm /tmp +mv "/usr/src/fio/fio" /tmp +dnf clean all diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 000000000..3b86cede0 --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) Intel Corporation +version: "3.8" +services: + build_base: + image: spdk + build: + context: build_base + container_name: build_base + storage-target: + image: spdk-app + build: + context: spdk-app + container_name: storage-target + depends_on: + - build_base + networks: + spdk: + ipv4_address: 192.168.42.2 + volumes: + - /dev/hugepages:/dev/hugepages + - ./spdk-app/storage-target.conf:/config + environment: + - SPDK_ARGS=-m 0x2 + privileged: true + proxy-container: + image: spdk-app + build: + context: spdk-app + container_name: proxy-container + depends_on: + - storage-target + networks: + spdk: + ipv4_address: 192.168.42.3 + volumes: + - /dev/hugepages:/dev/hugepages + - ./spdk-app/proxy-container.conf:/config + environment: + - SPDK_ARGS=-m 0x4 + privileged: true + traffic-generator: + image: traffic-generator + build: + context: traffic-generator + container_name: traffic-generator + depends_on: + - proxy-container + networks: + spdk: + volumes: + - /dev/hugepages:/dev/hugepages + - ./traffic-generator/conf:/config + - ./traffic-generator/fio.conf:/fio.conf + privileged: true +networks: + spdk: + name: "spdk" + ipam: + config: + - subnet: 192.168.42.0/29 + gateway: 192.168.42.1 diff --git a/docker/spdk-app/Dockerfile b/docker/spdk-app/Dockerfile new file mode 100644 index 000000000..ac54cbd50 --- /dev/null +++ b/docker/spdk-app/Dockerfile @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) Intel Corporation + +FROM spdk + +# Generic args +ARG PROXY +ARG NO_PROXY + +ENV http_proxy=$PROXY +ENV https_proxy=$PROXY +ENV no_proxy=$NO_PROXY + + +COPY init /init + +ENTRYPOINT ["/init"] diff --git a/docker/spdk-app/init b/docker/spdk-app/init new file mode 100755 index 000000000..a9d933fa9 --- /dev/null +++ b/docker/spdk-app/init @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +app=spdk_tgt args=() limit_args=() + +# Override default app +if [[ -n $SPDK_APP ]]; then + app=$SPDK_APP +fi + +# Define extra arguments to the app +if [[ -n $SPDK_ARGS ]]; then + args=($SPDK_ARGS) +fi + +# Limit the app with to following options, +# to allow for minimal impact on the host. +limit_args+=("--no-pci") +limit_args+=("--num-trace-entries" 0) + +# if set, don't include limit_args[] on the cmdline +if [[ ! -v SPDK_NO_LIMIT ]]; then + args+=("${limit_args[@]}") +fi + +if [[ -e /config ]]; then + args+=("--json" "/config") +fi + +# Wait a bit to make sure ip is in place +sleep 2s + +exec "$app" "${args[@]}" diff --git a/docker/spdk-app/proxy-container.conf b/docker/spdk-app/proxy-container.conf new file mode 100644 index 000000000..416d8d92b --- /dev/null +++ b/docker/spdk-app/proxy-container.conf @@ -0,0 +1,68 @@ +{ + "subsystems": [ + { + "subsystem": "bdev", + "config": [ + { + "method": "bdev_nvme_attach_controller", + "params": { + "name": "Nvme0", + "trtype": "TCP", + "adrfam": "IPv4", + "traddr": "192.168.42.2", + "trsvcid": "4420", + "subnqn": "nqn.2016-06.io.spdk:cnode1", + "prchk_reftag": false, + "prchk_guard": false + } + } + ] + }, + { + "subsystem": "nvmf", + "config": [ + { + "method": "nvmf_create_transport", + "params": { + "trtype": "TCP", + "io_unit_size": 8192 + } + }, + { + "method": "nvmf_create_subsystem", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "allow_any_host": true, + "serial_number": "SPDK00000000000001", + "model_number": "SPDK bdev Controller", + "max_namespaces": 32, + "min_cntlid": 1, + "max_cntlid": 65519 + } + }, + { + "method": "nvmf_subsystem_add_ns", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "namespace": { + "nsid": 1, + "bdev_name": "Nvme0n1" + } + } + }, + { + "method": "nvmf_subsystem_add_listener", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "listen_address": { + "trtype": "TCP", + "adrfam": "IPv4", + "traddr": "192.168.42.3", + "trsvcid": "4420" + } + } + } + ] + } + ] +} diff --git a/docker/spdk-app/storage-target.conf b/docker/spdk-app/storage-target.conf new file mode 100644 index 000000000..1916e3974 --- /dev/null +++ b/docker/spdk-app/storage-target.conf @@ -0,0 +1,63 @@ +{ + "subsystems": [ + { + "subsystem": "bdev", + "config": [ + { + "method": "bdev_malloc_create", + "params": { + "name": "Malloc0", + "num_blocks": 131072, + "block_size": 512 + } + } + ] + }, + { + "subsystem": "nvmf", + "config": [ + { + "method": "nvmf_create_transport", + "params": { + "trtype": "TCP", + "io_unit_size": 8192 + } + }, + { + "method": "nvmf_create_subsystem", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "allow_any_host": true, + "serial_number": "SPDK00000000000001", + "model_number": "SPDK bdev Controller", + "max_namespaces": 32, + "min_cntlid": 1, + "max_cntlid": 65519 + } + }, + { + "method": "nvmf_subsystem_add_ns", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "namespace": { + "nsid": 1, + "bdev_name": "Malloc0" + } + } + }, + { + "method": "nvmf_subsystem_add_listener", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "listen_address": { + "trtype": "TCP", + "adrfam": "IPv4", + "traddr": "192.168.42.2", + "trsvcid": "4420" + } + } + } + ] + } + ] +} diff --git a/docker/traffic-generator/Dockerfile b/docker/traffic-generator/Dockerfile new file mode 100644 index 000000000..ac54cbd50 --- /dev/null +++ b/docker/traffic-generator/Dockerfile @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) Intel Corporation + +FROM spdk + +# Generic args +ARG PROXY +ARG NO_PROXY + +ENV http_proxy=$PROXY +ENV https_proxy=$PROXY +ENV no_proxy=$NO_PROXY + + +COPY init /init + +ENTRYPOINT ["/init"] diff --git a/docker/traffic-generator/conf b/docker/traffic-generator/conf new file mode 100644 index 000000000..2637169c4 --- /dev/null +++ b/docker/traffic-generator/conf @@ -0,0 +1,22 @@ +{ + "subsystems": [ + { + "subsystem": "bdev", + "config": [ + { + "method": "bdev_nvme_attach_controller", + "params": { + "name": "Nvme0", + "trtype": "TCP", + "adrfam": "IPv4", + "traddr": "192.168.42.3", + "trsvcid": "4420", + "subnqn": "nqn.2016-06.io.spdk:cnode1", + "prchk_reftag": false, + "prchk_guard": false + } + } + ] + } + ] +} diff --git a/docker/traffic-generator/fio.conf b/docker/traffic-generator/fio.conf new file mode 100644 index 000000000..7da647919 --- /dev/null +++ b/docker/traffic-generator/fio.conf @@ -0,0 +1,16 @@ +[global] +ioengine=spdk_bdev +spdk_json_conf=/config +thread=1 +direct=1 +rw=randread +ramp_time=0 +norandommap=1 +time_based=1 +bs=4k +numjobs=1 +runtime=10 + +[filename0] +filename=Nvme0n1 +iodepth=128 diff --git a/docker/traffic-generator/init b/docker/traffic-generator/init new file mode 100755 index 000000000..eb8850598 --- /dev/null +++ b/docker/traffic-generator/init @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +args=() + +# Define extra arguments to the app +if [[ -n $FIO_ARGS ]]; then + args+=($FIO_ARGS) +fi + +# Wait a bit to make sure ip is in place +sleep 2s + +export LD_PRELOAD=/usr/local/bin/fio/spdk_bdev +exec /usr/src/fio/fio "${args[@]}" /fio.conf diff --git a/rpmbuild/spdk.spec b/rpmbuild/spdk.spec index 1429914e6..508ef58c7 100644 --- a/rpmbuild/spdk.spec +++ b/rpmbuild/spdk.spec @@ -75,6 +75,7 @@ cp -a %{dpdk_build_path}/lib/* %{buildroot}/usr/local/lib/dpdk/ # Try to include all the binaries that were potentially built [[ -e build/examples ]] && cp -a build/examples/* %{buildroot}/usr/local/bin/ [[ -e build/bin ]] && cp -a build/bin/* %{buildroot}/usr/local/bin/ +[[ -e build/fio ]] && cp -a build/fio %{buildroot}/usr/local/bin/fio # And some useful setup scripts SPDK uses mkdir -p %{buildroot}/usr/libexec/spdk