From 8000cedbe310faea476a70ce9edab027a0b20939 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Sat, 24 Jul 2021 00:23:26 +0200 Subject: [PATCH] bdev/ocf: Add runtime cache bdev flush support Introduce two RPC calls for starting flush and getting flush status of OCF cache bdev: - bdev_ocf_flush_start - bdev_ocf_flush_status Signed-off-by: Robert Baldyga Signed-off-by: Rafal Stefanowski Change-Id: I1d659da6fc51396e0d070af35372ee130c40ae8b Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/8961 Tested-by: SPDK CI Jenkins Community-CI: Mellanox Build Bot Reviewed-by: Jim Harris Reviewed-by: Tomasz Zawadzki --- doc/jsonrpc.md | 95 ++++++++++++++++++++++++ module/bdev/ocf/vbdev_ocf.h | 9 ++- module/bdev/ocf/vbdev_ocf_rpc.c | 126 ++++++++++++++++++++++++++++++++ python/spdk/rpc/bdev.py | 29 ++++++++ scripts/rpc.py | 14 ++++ test/ocf/integrity/flush.sh | 81 ++++++++++++++++++++ test/ocf/ocf.sh | 1 + 7 files changed, 354 insertions(+), 1 deletion(-) create mode 100755 test/ocf/integrity/flush.sh diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index 4aab0bef9..4afc56cae 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -2859,6 +2859,101 @@ Example response: } ~~~ +### bdev_ocf_flush_start {#rpc_bdev_ocf_flush_start} + +Start flushing OCF cache device. + +Automatic flushes of dirty data are managed by OCF cleaning policy settings. +In addition to that, all dirty data is flushed to core device when there is +an attempt to stop caching. +On the other hand, this RPC call gives a possibility to flush dirty data manually +when there is a need for it, e.g. to speed up the shutdown process when data +hasn't been flushed for a long time. +This RPC returns immediately, and flush is then being performed in the +background. To see the status of flushing operation use bdev_ocf_flush_status. + +#### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name + +#### Example + +Example request: + +~~~json +{ + "params": { + "name": "ocf0" + }, + "jsonrpc": "2.0", + "method": "bdev_ocf_flush_start", + "id": 1 +} +~~~ + +Example response: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +### bdev_ocf_flush_status {#rpc_bdev_ocf_flush_status} + +Get flush status of OCF cache device. + +Automatic flushes of dirty data are managed by OCF cleaning policy settings. +In addition to that, all dirty data is flushed to core device when there is +an attempt to stop caching. +On the other hand, there is a possibility to flush dirty data manually +when there is a need for it, e.g. to speed up the shutdown process when data +hasn't been flushed for a long time. +This RPC reports if such manual flush is still in progress and if the operation +was successful. To start manual flush use bdev_ocf_flush_start. + +#### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Bdev name + +#### Response + +Status of OCF cache device flush. + +#### Example + +Example request: + +~~~json +{ + "params": { + "name": "ocf0" + }, + "jsonrpc": "2.0", + "method": "bdev_ocf_flush_status", + "id": 1 +} +~~~ + +Example response: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "in_progress": false, + "status": 0 + } +} +~~~ + ### bdev_malloc_create {#rpc_bdev_malloc_create} Construct @ref bdev_config_malloc diff --git a/module/bdev/ocf/vbdev_ocf.h b/module/bdev/ocf/vbdev_ocf.h index dc9998ba5..2e5429145 100644 --- a/module/bdev/ocf/vbdev_ocf.h +++ b/module/bdev/ocf/vbdev_ocf.h @@ -141,9 +141,16 @@ struct vbdev_ocf { /* Management context */ struct vbdev_ocf_mngt_ctx mngt_ctx; - /* Cache conext */ + + /* Cache context */ struct vbdev_ocf_cache_ctx *cache_ctx; + /* Status of flushing operation */ + struct { + bool in_progress; + int status; + } flush; + /* Exposed SPDK bdev. Registered in bdev layer */ struct spdk_bdev exp_bdev; diff --git a/module/bdev/ocf/vbdev_ocf_rpc.c b/module/bdev/ocf/vbdev_ocf_rpc.c index 0fea4dd3c..b31cb2424 100644 --- a/module/bdev/ocf/vbdev_ocf_rpc.c +++ b/module/bdev/ocf/vbdev_ocf_rpc.c @@ -438,3 +438,129 @@ end: free_rpc_bdev_ocf_set_seqcutoff(&req); } SPDK_RPC_REGISTER("bdev_ocf_set_seqcutoff", rpc_bdev_ocf_set_seqcutoff, SPDK_RPC_RUNTIME) + +struct get_ocf_flush_start_ctx { + struct spdk_jsonrpc_request *request; + struct vbdev_ocf *vbdev; +}; + +static void +rpc_bdev_ocf_flush_start_cmpl(ocf_cache_t cache, void *priv, int error) +{ + struct get_ocf_flush_start_ctx *ctx = priv; + + ctx->vbdev->flush.in_progress = false; + ctx->vbdev->flush.status = error; + + ocf_mngt_cache_read_unlock(cache); + + free(ctx); +} + +static void +rpc_bdev_ocf_flush_start_lock_cmpl(ocf_cache_t cache, void *priv, int error) +{ + struct get_ocf_flush_start_ctx *ctx = priv; + + if (error) { + spdk_jsonrpc_send_error_response_fmt(ctx->request, + SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Could not lock cache: %d", error); + free(ctx); + return; + } + + ctx->vbdev->flush.in_progress = true; + ocf_mngt_cache_flush(cache, rpc_bdev_ocf_flush_start_cmpl, ctx); + + spdk_jsonrpc_send_bool_response(ctx->request, true); +} + +static void +rpc_bdev_ocf_flush_start(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_bdev_ocf_name req = {NULL}; + struct get_ocf_flush_start_ctx *ctx; + int status; + + ctx = calloc(1, sizeof(*ctx)); + if (!ctx) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Not enough memory to process request"); + goto end; + } + + status = spdk_json_decode_object(params, rpc_bdev_ocf_name_decoders, + SPDK_COUNTOF(rpc_bdev_ocf_name_decoders), + &req); + if (status) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parameters"); + free(ctx); + goto end; + } + + ctx->vbdev = vbdev_ocf_get_by_name(req.name); + if (ctx->vbdev == NULL) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + spdk_strerror(ENODEV)); + free(ctx); + goto end; + } + + if (!ctx->vbdev->ocf_cache) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Couldn't flush cache: device not attached"); + free(ctx); + goto end; + } + + ctx->request = request; + ocf_mngt_cache_read_lock(ctx->vbdev->ocf_cache, rpc_bdev_ocf_flush_start_lock_cmpl, ctx); + +end: + free_rpc_bdev_ocf_name(&req); +} +SPDK_RPC_REGISTER("bdev_ocf_flush_start", rpc_bdev_ocf_flush_start, SPDK_RPC_RUNTIME) + +static void +rpc_bdev_ocf_flush_status(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_bdev_ocf_name req = {NULL}; + struct spdk_json_write_ctx *w; + struct vbdev_ocf *vbdev; + int status; + + status = spdk_json_decode_object(params, rpc_bdev_ocf_name_decoders, + SPDK_COUNTOF(rpc_bdev_ocf_name_decoders), + &req); + if (status) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parameters"); + goto end; + } + + vbdev = vbdev_ocf_get_by_name(req.name); + if (vbdev == NULL) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + spdk_strerror(ENODEV)); + goto end; + } + + w = spdk_jsonrpc_begin_result(request); + + spdk_json_write_object_begin(w); + spdk_json_write_named_bool(w, "in_progress", vbdev->flush.in_progress); + if (!vbdev->flush.in_progress) { + spdk_json_write_named_int32(w, "status", vbdev->flush.status); + } + spdk_json_write_object_end(w); + + spdk_jsonrpc_end_result(request, w); + +end: + free_rpc_bdev_ocf_name(&req); +} +SPDK_RPC_REGISTER("bdev_ocf_flush_status", rpc_bdev_ocf_flush_status, SPDK_RPC_RUNTIME) diff --git a/python/spdk/rpc/bdev.py b/python/spdk/rpc/bdev.py index c7555231e..94bf47c79 100644 --- a/python/spdk/rpc/bdev.py +++ b/python/spdk/rpc/bdev.py @@ -234,6 +234,35 @@ def bdev_ocf_set_seqcutoff(client, name, policy, threshold, promotion_count): return client.call('bdev_ocf_set_seqcutoff', params) +def bdev_ocf_flush_start(client, name): + """Start flushing OCF cache device + + Args: + name: name of OCF bdev + """ + params = { + 'name': name, + } + + return client.call('bdev_ocf_flush_start', params) + + +def bdev_ocf_flush_status(client, name): + """Get flush status of OCF cache device + + Args: + name: name of OCF bdev + + Returns: + Flush status + """ + params = { + 'name': name, + } + + return client.call('bdev_ocf_flush_status', params) + + def bdev_malloc_create(client, num_blocks, block_size, name=None, uuid=None, optimal_io_boundary=None, md_size=None, md_interleave=None, dif_type=None, dif_is_head_of_md=None): """Construct a malloc block device. diff --git a/scripts/rpc.py b/scripts/rpc.py index 95df3b404..fb21b7475 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -364,6 +364,20 @@ if __name__ == "__main__": help='Sequential cutoff policy') p.set_defaults(func=bdev_ocf_set_seqcutoff) + def bdev_ocf_flush_start(args): + rpc.bdev.bdev_ocf_flush_start(args.client, name=args.name) + p = subparsers.add_parser('bdev_ocf_flush_start', + help='Start flushing OCF cache device') + p.add_argument('name', help='Name of OCF bdev') + p.set_defaults(func=bdev_ocf_flush_start) + + def bdev_ocf_flush_status(args): + print_json(rpc.bdev.bdev_ocf_flush_status(args.client, name=args.name)) + p = subparsers.add_parser('bdev_ocf_flush_status', + help='Get flush status of OCF cache device') + p.add_argument('name', help='Name of OCF bdev') + p.set_defaults(func=bdev_ocf_flush_status) + def bdev_malloc_create(args): num_blocks = (args.total_size * 1024 * 1024) // args.block_size print_json(rpc.bdev.bdev_malloc_create(args.client, diff --git a/test/ocf/integrity/flush.sh b/test/ocf/integrity/flush.sh new file mode 100755 index 000000000..e69abd470 --- /dev/null +++ b/test/ocf/integrity/flush.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +rootdir=$(readlink -f $curdir/../../..) +source $rootdir/test/common/autotest_common.sh + +bdevperf=$rootdir/test/bdev/bdevperf/bdevperf +rpc_py="$rootdir/scripts/rpc.py -s /var/tmp/spdk.sock" + +check_flush_in_progress() { + $rpc_py bdev_ocf_flush_status MalCache0 \ + | jq -e '.in_progress' > /dev/null +} + +bdevperf_config() { + local config + + config="$( + cat <<- JSON + { + "method": "bdev_malloc_create", + "params": { + "name": "Malloc0", + "num_blocks": 102400, + "block_size": 512 + } + }, + { + "method": "bdev_malloc_create", + "params": { + "name": "Malloc1", + "num_blocks": 1024000, + "block_size": 512 + } + }, + { + "method": "bdev_ocf_create", + "params": { + "name": "MalCache0", + "mode": "wb", + "cache_line_size": 4, + "cache_bdev_name": "Malloc0", + "core_bdev_name": "Malloc1" + } + } + JSON + )" + + jq . <<- JSON + { + "subsystems": [ + { + "subsystem": "bdev", + "config": [ + $( + IFS="," + printf '%s\n' "$config" + ), + { + "method": "bdev_wait_for_examine" + } + ] + } + ] + } + JSON +} + +$bdevperf --json <(bdevperf_config) -q 128 -o 4096 -w write -t 120 -r /var/tmp/spdk.sock & +bdevperf_pid=$! +trap 'killprocess $bdevperf_pid' SIGINT SIGTERM EXIT +waitforlisten $bdevperf_pid +sleep 5 + +$rpc_py bdev_ocf_flush_start MalCache0 +sleep 1 + +while check_flush_in_progress; do + sleep 1 +done +$rpc_py bdev_ocf_flush_status MalCache0 | jq -e '.status == 0' diff --git a/test/ocf/ocf.sh b/test/ocf/ocf.sh index dd81371f3..e04577778 100755 --- a/test/ocf/ocf.sh +++ b/test/ocf/ocf.sh @@ -8,6 +8,7 @@ source $rootdir/test/common/autotest_common.sh run_test "ocf_fio_modes" "$testdir/integrity/fio-modes.sh" run_test "ocf_bdevperf_iotypes" "$testdir/integrity/bdevperf-iotypes.sh" run_test "ocf_stats" "$testdir/integrity/stats.sh" +run_test "ocf_flush" "$testdir/integrity/flush.sh" run_test "ocf_create_destruct" "$testdir/management/create-destruct.sh" run_test "ocf_multicore" "$testdir/management/multicore.sh" run_test "ocf_persistent_metadata" "$testdir/management/persistent-metadata.sh"