blobfs: add the file with new cache buffer to cache list in cache thread

Also poll thread in the UT to cover the thread context switch.

Change-Id: I3dc765b66aa707c36eb3913b3e0d2c1c3986e282
Signed-off-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1793
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>

lib/blobfs/blobfs.c

@@ -2148,11 +2148,22 @@ _blobfs_cache_pool_reclaim(void *arg)
 	return 1;
 }
 
+static void
+_add_file_to_cache_pool(void *ctx)
+{
+	struct spdk_file *file = ctx;
+
+	pthread_spin_lock(&g_caches_lock);
+	TAILQ_INSERT_TAIL(&g_caches, file, cache_tailq);
+	pthread_spin_unlock(&g_caches_lock);
+}
+
 static struct cache_buffer *
 cache_insert_buffer(struct spdk_file *file, uint64_t offset)
 {
 	struct cache_buffer *buf;
 	int count = 0;
+	bool need_update = false;
 
 	buf = calloc(1, sizeof(*buf));
 	if (buf == NULL) {
@@ -2177,12 +2188,14 @@ cache_insert_buffer(struct spdk_file *file, uint64_t offset)
 	buf->buf_size = CACHE_BUFFER_SIZE;
 	buf->offset = offset;
 
-	pthread_spin_lock(&g_caches_lock);
 	if (file->tree->present_mask == 0) {
-		TAILQ_INSERT_TAIL(&g_caches, file, cache_tailq);
+		need_update = true;
 	}
 	file->tree = tree_insert_buffer(file->tree, buf);
-	pthread_spin_unlock(&g_caches_lock);
+
+	if (need_update) {
+		spdk_thread_send_msg(g_cache_pool_thread, _add_file_to_cache_pool, file);
+	}
 
 	return buf;
 }

test/unit/lib/blobfs/blobfs_sync_ut/blobfs_sync_ut.c

@@ -123,18 +123,26 @@ fs_op_with_handle_complete(void *ctx, struct spdk_filesystem *fs, int fserrno)
 }
 
 static void
-_fs_init(void *arg)
+fs_thread_poll(void)
 {
 	struct spdk_thread *thread;
+
+	thread = spdk_get_thread();
+	while (spdk_thread_poll(thread, 0, 0) > 0) {}
+	while (spdk_thread_poll(g_cache_pool_thread, 0, 0) > 0) {}
+}
+
+static void
+_fs_init(void *arg)
+{
 	struct spdk_bs_dev *dev;
 
 	g_fs = NULL;
 	g_fserrno = -1;
 	dev = init_dev();
 	spdk_fs_init(dev, NULL, send_request, fs_op_with_handle_complete, NULL);
-	thread = spdk_get_thread();
+
-	while (spdk_thread_poll(thread, 0, 0) > 0) {}
+	fs_thread_poll();
-	while (spdk_thread_poll(g_cache_pool_thread, 0, 0) > 0) {}
 
 	SPDK_CU_ASSERT_FATAL(g_fs != NULL);
 	SPDK_CU_ASSERT_FATAL(g_fs->bdev == dev);
@@ -144,16 +152,14 @@ _fs_init(void *arg)
 static void
 _fs_load(void *arg)
 {
-	struct spdk_thread *thread;
 	struct spdk_bs_dev *dev;
 
 	g_fs = NULL;
 	g_fserrno = -1;
 	dev = init_dev();
 	spdk_fs_load(dev, send_request, fs_op_with_handle_complete, NULL);
-	thread = spdk_get_thread();
+
-	while (spdk_thread_poll(thread, 0, 0) > 0) {}
+	fs_thread_poll();
-	while (spdk_thread_poll(g_cache_pool_thread, 0, 0) > 0) {}
 
 	SPDK_CU_ASSERT_FATAL(g_fs != NULL);
 	SPDK_CU_ASSERT_FATAL(g_fs->bdev == dev);
@@ -163,13 +169,11 @@ _fs_load(void *arg)
 static void
 _fs_unload(void *arg)
 {
-	struct spdk_thread *thread;
-
 	g_fserrno = -1;
 	spdk_fs_unload(g_fs, fs_op_complete, NULL);
-	thread = spdk_get_thread();
+
-	while (spdk_thread_poll(thread, 0, 0) > 0) {}
+	fs_thread_poll();
-	while (spdk_thread_poll(g_cache_pool_thread, 0, 0) > 0) {}
+
 	CU_ASSERT(g_fserrno == 0);
 	g_fs = NULL;
 }
@@ -210,6 +214,8 @@ cache_read_after_write(void)
 
 	spdk_file_close(g_file, channel);
 
+	fs_thread_poll();
+
 	rc = spdk_fs_file_stat(g_fs, channel, "testfile", &stat);
 	CU_ASSERT(rc == 0);
 	CU_ASSERT(sizeof(w_buf) == stat.size);
@@ -223,6 +229,9 @@ cache_read_after_write(void)
 	CU_ASSERT(memcmp(w_buf, r_buf, sizeof(r_buf)) == 0);
 
 	spdk_file_close(g_file, channel);
+
+	fs_thread_poll();
+
 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
 	CU_ASSERT(rc == 0);
 
@@ -276,6 +285,8 @@ file_length(void)
 	 */
 	spdk_file_close(g_file, channel);
 
+	fs_thread_poll();
+
 	rc = spdk_fs_file_stat(g_fs, channel, "testfile", &stat);
 	CU_ASSERT(rc == 0);
 	CU_ASSERT(buf_length == stat.size);
@@ -304,6 +315,8 @@ file_length(void)
 
 	spdk_file_close(g_file, channel);
 
+	fs_thread_poll();
+
 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
 	CU_ASSERT(rc == 0);
 
@@ -338,6 +351,7 @@ append_write_to_extend_blob(void)
 	free(buf);
 
 	spdk_file_close(g_file, channel);
+	fs_thread_poll();
 	spdk_fs_free_thread_ctx(channel);
 	ut_send_request(_fs_unload, NULL);
 
@@ -358,6 +372,7 @@ append_write_to_extend_blob(void)
 	CU_ASSERT(rc == 0);
 	CU_ASSERT(2 * blob_size == __file_get_blob_size(g_file));
 	spdk_file_close(g_file, channel);
+	fs_thread_poll();
 	CU_ASSERT(g_file->length == buf_length + 2);
 
 	spdk_fs_free_thread_ctx(channel);
@@ -413,6 +428,8 @@ partial_buffer(void)
 	 */
 	spdk_file_close(g_file, channel);
 
+	fs_thread_poll();
+
 	rc = spdk_fs_file_stat(g_fs, channel, "testfile", &stat);
 	CU_ASSERT(rc == 0);
 	CU_ASSERT(buf_length == stat.size);
@@ -449,6 +466,9 @@ cache_write_null_buffer(void)
 	CU_ASSERT(rc == 0);
 
 	spdk_file_close(g_file, channel);
+
+	fs_thread_poll();
+
 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
 	CU_ASSERT(rc == 0);
 
@@ -465,7 +485,6 @@ fs_create_sync(void)
 {
 	int rc;
 	struct spdk_fs_thread_ctx *channel;
-	struct spdk_thread *thread;
 
 	ut_send_request(_fs_init, NULL);
 
@@ -484,8 +503,7 @@ fs_create_sync(void)
 
 	spdk_fs_free_thread_ctx(channel);
 
-	thread = spdk_get_thread();
+	fs_thread_poll();
-	while (spdk_thread_poll(thread, 0, 0) > 0) {}
 
 	ut_send_request(_fs_unload, NULL);
 }
@@ -495,7 +513,6 @@ fs_rename_sync(void)
 {
 	int rc;
 	struct spdk_fs_thread_ctx *channel;
-	struct spdk_thread *thread;
 
 	ut_send_request(_fs_init, NULL);
 
@@ -513,10 +530,10 @@ fs_rename_sync(void)
 	CU_ASSERT(strcmp(spdk_file_get_name(g_file), "newtestfile") == 0);
 
 	spdk_file_close(g_file, channel);
-	spdk_fs_free_thread_ctx(channel);
 
-	thread = spdk_get_thread();
+	fs_thread_poll();
-	while (spdk_thread_poll(thread, 0, 0) > 0) {}
+
+	spdk_fs_free_thread_ctx(channel);
 
 	ut_send_request(_fs_unload, NULL);
 }
@@ -527,7 +544,6 @@ cache_append_no_cache(void)
 	int rc;
 	char buf[100];
 	struct spdk_fs_thread_ctx *channel;
-	struct spdk_thread *thread;
 
 	ut_send_request(_fs_init, NULL);
 
@@ -542,6 +558,9 @@ cache_append_no_cache(void)
 	spdk_file_write(g_file, channel, buf, 1 * sizeof(buf), sizeof(buf));
 	CU_ASSERT(spdk_file_get_length(g_file) == 2 * sizeof(buf));
 	spdk_file_sync(g_file, channel);
+
+	fs_thread_poll();
+
 	cache_free_buffers(g_file);
 	spdk_file_write(g_file, channel, buf, 2 * sizeof(buf), sizeof(buf));
 	CU_ASSERT(spdk_file_get_length(g_file) == 3 * sizeof(buf));
@@ -551,14 +570,14 @@ cache_append_no_cache(void)
 	CU_ASSERT(spdk_file_get_length(g_file) == 5 * sizeof(buf));
 
 	spdk_file_close(g_file, channel);
+
+	fs_thread_poll();
+
 	rc = spdk_fs_delete_file(g_fs, channel, "testfile");
 	CU_ASSERT(rc == 0);
 
 	spdk_fs_free_thread_ctx(channel);
 
-	thread = spdk_get_thread();
-	while (spdk_thread_poll(thread, 0, 0) > 0) {}
-
 	ut_send_request(_fs_unload, NULL);
 }
 
@@ -568,7 +587,6 @@ fs_delete_file_without_close(void)
 	int rc;
 	struct spdk_fs_thread_ctx *channel;
 	struct spdk_file *file;
-	struct spdk_thread *thread;
 
 	ut_send_request(_fs_init, NULL);
 	channel = spdk_fs_alloc_thread_ctx(g_fs);
@@ -588,14 +606,13 @@ fs_delete_file_without_close(void)
 
 	spdk_file_close(g_file, channel);
 
+	fs_thread_poll();
+
 	rc = spdk_fs_open_file(g_fs, channel, "testfile", 0, &file);
 	CU_ASSERT(rc != 0);
 
 	spdk_fs_free_thread_ctx(channel);
 
-	thread = spdk_get_thread();
-	while (spdk_thread_poll(thread, 0, 0) > 0) {}
-
 	ut_send_request(_fs_unload, NULL);
 
 }