diff --git a/docker/README.md b/docker/README.md index 9b79b8f42..664414dd8 100644 --- a/docker/README.md +++ b/docker/README.md @@ -30,12 +30,13 @@ docker-compose build --build-arg PROXY=$http_proxy ## How-To `docker-compose.yaml` shows an example deployment of the storage containers based on SPDK. -Running `docker-compose build` creates 4 docker images: +Running `docker-compose build` creates 5 docker images: - build_base - storage-target - proxy-container -- traffic-generator +- traffic-generator-nvme +- traffic-generator-virtio The `build_base` image provides the core components required to containerize SPDK applications. The fedora:33 image from the Fedora Container Registry is used and then SPDK is installed. SPDK is installed out of `build_base/spdk.tar.gz` provided. @@ -46,8 +47,11 @@ Running `docker-compose up` creates 3 docker containers: -- storage-target: Contains SPDK NVMe-oF target exposing single subsystem to `proxy-container` based on malloc bdev. -- proxy-container: Contains SPDK NVMe-oF target connecting to `storage-target` -and then exposing the same subsystem to `traffic-generator`. --- traffic-generator: Contains FIO using SPDK plugin to connect to `proxy-container` +and then exposing the same devices to `traffic-generator-nvme` using NVMe-oF and +to `traffic-generator-virtio` using Virtio. +-- traffic-generator-nvme: Contains FIO using SPDK plugin to connect to `proxy-container` +and runs a sample workload. +-- traffic-generator-virtio: Contains FIO using SPDK plugin to connect to `proxy-container` and runs a sample workload. Each container is connected to a separate "spdk" network which is created before @@ -63,11 +67,12 @@ docker-compose up ~~~ The `storage-target` and `proxy-container` can be started as services. -Allowing for multiple `traffic-generator` containers to connect. +Allowing for multiple traffic generator containers to connect. ~~~{.sh} docker-compose up -d proxy-container -docker-compose run traffic-generator +docker-compose run traffic-generator-nvme +docker-compose run traffic-generator-virtio ~~~ Enviroment variables to containers can be passed as shown in @@ -75,7 +80,7 @@ Enviroment variables to containers can be passed as shown in For example extra arguments to fio can be passed as so: ~~~{.sh} -docker-compose run -e FIO_ARGS="--minimal" traffic-generator +docker-compose run -e FIO_ARGS="--minimal" traffic-generator-nvme ~~~ As each container includes SPDK installation it is possible to use rpc.py to diff --git a/docker/build_base/pre-install b/docker/build_base/pre-install index 368c6fcb4..78e07f521 100755 --- a/docker/build_base/pre-install +++ b/docker/build_base/pre-install @@ -36,6 +36,7 @@ dnf install -y rpm-build # Deploy SPDK inside the container DEPS="no" "$spdk_repo/rpmbuild/rpm.sh" \ --with-shared \ + --with-virtio \ --with-fio mv "$HOME/rpmbuild/rpm/x86_64/"*.rpm /tmp diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 3b86cede0..f3206229e 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -36,22 +36,38 @@ services: volumes: - /dev/hugepages:/dev/hugepages - ./spdk-app/proxy-container.conf:/config + - vhost-user:/vhost-user environment: - - SPDK_ARGS=-m 0x4 + - SPDK_ARGS=-m 0x4 -S /vhost-user privileged: true - traffic-generator: + traffic-generator-virtio: image: traffic-generator build: context: traffic-generator - container_name: traffic-generator + container_name: traffic-generator-virtio depends_on: - proxy-container networks: spdk: volumes: - /dev/hugepages:/dev/hugepages - - ./traffic-generator/conf:/config - - ./traffic-generator/fio.conf:/fio.conf + - ./traffic-generator/conf-virtio:/config + - vhost-user:/vhost-user + - ./traffic-generator/fio-virtio.conf:/fio.conf + privileged: true + traffic-generator-nvme: + image: traffic-generator + build: + context: traffic-generator + container_name: traffic-generator-nvme + depends_on: + - proxy-container + networks: + spdk: + volumes: + - /dev/hugepages:/dev/hugepages + - ./traffic-generator/conf-nvme:/config + - ./traffic-generator/fio-nvme.conf:/fio.conf privileged: true networks: spdk: @@ -60,3 +76,5 @@ networks: config: - subnet: 192.168.42.0/29 gateway: 192.168.42.1 +volumes: + vhost-user: diff --git a/docker/spdk-app/proxy-container.conf b/docker/spdk-app/proxy-container.conf index 416d8d92b..d0368a205 100644 --- a/docker/spdk-app/proxy-container.conf +++ b/docker/spdk-app/proxy-container.conf @@ -63,6 +63,18 @@ } } ] + }, + { + "subsystem": "vhost", + "config": [ + { + "method": "vhost_create_blk_controller", + "params": { + "ctrlr": "VirtioBlk0", + "dev_name": "Nvme0n2" + } + } + ] } ] } diff --git a/docker/spdk-app/storage-target.conf b/docker/spdk-app/storage-target.conf index 1916e3974..7d4269c51 100644 --- a/docker/spdk-app/storage-target.conf +++ b/docker/spdk-app/storage-target.conf @@ -10,6 +10,14 @@ "num_blocks": 131072, "block_size": 512 } + }, + { + "method": "bdev_malloc_create", + "params": { + "name": "Malloc1", + "num_blocks": 131072, + "block_size": 512 + } } ] }, @@ -45,6 +53,16 @@ } } }, + { + "method": "nvmf_subsystem_add_ns", + "params": { + "nqn": "nqn.2016-06.io.spdk:cnode1", + "namespace": { + "nsid": 2, + "bdev_name": "Malloc1" + } + } + }, { "method": "nvmf_subsystem_add_listener", "params": { diff --git a/docker/traffic-generator/conf b/docker/traffic-generator/conf-nvme similarity index 100% rename from docker/traffic-generator/conf rename to docker/traffic-generator/conf-nvme diff --git a/docker/traffic-generator/conf-virtio b/docker/traffic-generator/conf-virtio new file mode 100644 index 000000000..575649586 --- /dev/null +++ b/docker/traffic-generator/conf-virtio @@ -0,0 +1,18 @@ +{ + "subsystems": [ + { + "subsystem": "bdev", + "config": [ + { + "method": "bdev_virtio_attach_controller", + "params": { + "name": "VirtioBlk0", + "trtype": "user", + "traddr": "/vhost-user/VirtioBlk0", + "dev_type": "blk" + } + } + ] + } + ] +} diff --git a/docker/traffic-generator/fio.conf b/docker/traffic-generator/fio-nvme.conf similarity index 93% rename from docker/traffic-generator/fio.conf rename to docker/traffic-generator/fio-nvme.conf index 7da647919..06bde5f58 100644 --- a/docker/traffic-generator/fio.conf +++ b/docker/traffic-generator/fio-nvme.conf @@ -1,6 +1,7 @@ [global] ioengine=spdk_bdev spdk_json_conf=/config +spdk_mem=1024 thread=1 direct=1 rw=randread diff --git a/docker/traffic-generator/fio-virtio.conf b/docker/traffic-generator/fio-virtio.conf new file mode 100644 index 000000000..5292a8dfa --- /dev/null +++ b/docker/traffic-generator/fio-virtio.conf @@ -0,0 +1,18 @@ +[global] +ioengine=spdk_bdev +spdk_json_conf=/config +spdk_single_seg=1 +spdk_mem=1024 +thread=1 +direct=1 +rw=randread +ramp_time=0 +norandommap=1 +time_based=1 +bs=4k +numjobs=1 +runtime=10 + +[filename0] +filename=VirtioBlk0 +iodepth=128