zfs: replace tpool with taskq

They're basically the same thing; lets just carry one.

Sponsored-by: https://despairlabs.com/sponsor/
Reviewed-by: Alexander Motin <alexander.motin@TrueNAS.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Rob Norris <robn@despairlabs.com>
Closes #17948

Author: robn

cmd/zed/agents/zfs_mod.c

@@ -82,7 +82,7 @@
 #include <sys/sunddi.h>
 #include <sys/sysevent/eventdefs.h>
 #include <sys/sysevent/dev.h>
-#include <thread_pool.h>
+#include <sys/taskq.h>
 #include <pthread.h>
 #include <unistd.h>
 #include <errno.h>
@@ -98,7 +98,7 @@ typedef void (*zfs_process_func_t)(zpool_handle_t *, nvlist_t *, boolean_t);
 libzfs_handle_t *g_zfshdl;
 list_t g_pool_list;	/* list of unavailable pools at initialization */
 list_t g_device_list;	/* list of disks with asynchronous label request */
-tpool_t *g_tpool;
+taskq_t *g_taskq;
 boolean_t g_enumeration_done;
 pthread_t g_zfs_tid;	/* zfs_enum_pools() thread */
 
@@ -749,8 +749,8 @@ zfs_iter_pool(zpool_handle_t *zhp, void *data)
 				continue;
 			if (zfs_toplevel_state(zhp) >= VDEV_STATE_DEGRADED) {
 				list_remove(&g_pool_list, pool);
-				(void) tpool_dispatch(g_tpool, zfs_enable_ds,
-				    pool);
+				(void) taskq_dispatch(g_taskq, zfs_enable_ds,
+				    pool, TQ_SLEEP);
 				break;
 			}
 		}
@@ -1347,9 +1347,9 @@ zfs_slm_fini(void)
 	/* wait for zfs_enum_pools thread to complete */
 	(void) pthread_join(g_zfs_tid, NULL);
 	/* destroy the thread pool */
-	if (g_tpool != NULL) {
-		tpool_wait(g_tpool);
-		tpool_destroy(g_tpool);
+	if (g_taskq != NULL) {
+		taskq_wait(g_taskq);
+		taskq_destroy(g_taskq);
 	}
 
 	while ((pool = list_remove_head(&g_pool_list)) != NULL) {

cmd/zpool/zpool_iter.c

@@ -34,7 +34,6 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <thread_pool.h>
 
 #include <libzfs.h>
 #include <libzutil.h>
@@ -653,21 +652,21 @@ all_pools_for_each_vdev_gather_cb(zpool_handle_t *zhp, void *cb_vcdl)
 static void
 all_pools_for_each_vdev_run_vcdl(vdev_cmd_data_list_t *vcdl)
 {
-	tpool_t *t;
-
-	t = tpool_create(1, 5 * sysconf(_SC_NPROCESSORS_ONLN), 0, NULL);
-	if (t == NULL)
+	taskq_t *tq = taskq_create("vdev_run_cmd",
+	    5 * sysconf(_SC_NPROCESSORS_ONLN), minclsyspri, 1, INT_MAX,
+	    TASKQ_DYNAMIC);
+	if (tq == NULL)
 		return;
 
 	/* Spawn off the command for each vdev */
 	for (int i = 0; i < vcdl->count; i++) {
-		(void) tpool_dispatch(t, vdev_run_cmd_thread,
-		    (void *) &vcdl->data[i]);
+		(void) taskq_dispatch(tq, vdev_run_cmd_thread,
+		    (void *) &vcdl->data[i], TQ_SLEEP);
 	}
 
 	/* Wait for threads to finish */
-	tpool_wait(t);
-	tpool_destroy(t);
+	taskq_wait(tq);
+	taskq_destroy(tq);
 }
 
 /*

cmd/zpool/zpool_main.c

@@ -52,7 +52,6 @@
 #include <stdlib.h>
 #include <string.h>
 #include <termios.h>
-#include <thread_pool.h>
 #include <time.h>
 #include <unistd.h>
 #include <pwd.h>
@@ -2389,7 +2388,7 @@ zpool_do_destroy(int argc, char **argv)
 }
 
 typedef struct export_cbdata {
-	tpool_t *tpool;
+	taskq_t *taskq;
 	pthread_mutex_t mnttab_lock;
 	boolean_t force;
 	boolean_t hardforce;
@@ -2414,12 +2413,12 @@ zpool_export_one(zpool_handle_t *zhp, void *data)
 	 * zpool_disable_datasets() is not thread-safe for mnttab access.
 	 * So we serialize access here for 'zpool export -a' parallel case.
 	 */
-	if (cb->tpool != NULL)
+	if (cb->taskq != NULL)
 		(void) pthread_mutex_lock(&cb->mnttab_lock);
 
 	int retval = zpool_disable_datasets(zhp, cb->force);
 
-	if (cb->tpool != NULL)
+	if (cb->taskq != NULL)
 		(void) pthread_mutex_unlock(&cb->mnttab_lock);
 
 	if (retval)
@@ -2463,15 +2462,16 @@ zpool_export_task(void *arg)
 static int
 zpool_export_one_async(zpool_handle_t *zhp, void *data)
 {
-	tpool_t *tpool = ((export_cbdata_t *)data)->tpool;
+	taskq_t *tq = ((export_cbdata_t *)data)->taskq;
 	async_export_args_t *aea = safe_malloc(sizeof (async_export_args_t));
 
 	/* save pool name since zhp will go out of scope */
 	aea->aea_poolname = strdup(zpool_get_name(zhp));
 	aea->aea_cbdata = data;
 
 	/* ship off actual export to another thread */
-	if (tpool_dispatch(tpool, zpool_export_task, (void *)aea) != 0)
+	if (taskq_dispatch(tq, zpool_export_task, (void *)aea,
+	    TQ_SLEEP) == TASKQID_INVALID)
 		return (errno);	/* unlikely */
 	else
 		return (0);
@@ -2517,7 +2517,7 @@ zpool_do_export(int argc, char **argv)
 
 	cb.force = force;
 	cb.hardforce = hardforce;
-	cb.tpool = NULL;
+	cb.taskq = NULL;
 	cb.retval = 0;
 	argc -= optind;
 	argv += optind;
@@ -2531,16 +2531,17 @@ zpool_do_export(int argc, char **argv)
 			usage(B_FALSE);
 		}
 
-		cb.tpool = tpool_create(1, 5 * sysconf(_SC_NPROCESSORS_ONLN),
-		    0, NULL);
+		cb.taskq = taskq_create("zpool_export",
+		    5 * sysconf(_SC_NPROCESSORS_ONLN), minclsyspri, 1, INT_MAX,
+		    TASKQ_DYNAMIC);
 		(void) pthread_mutex_init(&cb.mnttab_lock, NULL);
 
 		/* Asynchronously call zpool_export_one using thread pool */
 		ret = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
 		    B_FALSE, zpool_export_one_async, &cb);
 
-		tpool_wait(cb.tpool);
-		tpool_destroy(cb.tpool);
+		taskq_wait(cb.taskq);
+		taskq_destroy(cb.taskq);
 		(void) pthread_mutex_destroy(&cb.mnttab_lock);
 
 		return (ret | cb.retval);
@@ -3945,10 +3946,11 @@ import_pools(nvlist_t *pools, nvlist_t *props, char *mntopts, int flags,
 	uint_t npools = 0;
 
 
-	tpool_t *tp = NULL;
+	taskq_t *tq = NULL;
 	if (import->do_all) {
-		tp = tpool_create(1, 5 * sysconf(_SC_NPROCESSORS_ONLN),
-		    0, NULL);
+		tq = taskq_create("zpool_import_all",
+		    5 * sysconf(_SC_NPROCESSORS_ONLN), minclsyspri, 1, INT_MAX,
+		    TASKQ_DYNAMIC);
 	}
 
 	/*
@@ -3997,8 +3999,8 @@ import_pools(nvlist_t *pools, nvlist_t *props, char *mntopts, int flags,
 				ip->ip_mntthreads = mount_tp_nthr / npools;
 				ip->ip_err = &err;
 
-				(void) tpool_dispatch(tp, do_import_task,
-				    (void *)ip);
+				(void) taskq_dispatch(tq, do_import_task,
+				    (void *)ip, TQ_SLEEP);
 			} else {
 				/*
 				 * If we're importing from cachefile, then
@@ -4047,8 +4049,8 @@ import_pools(nvlist_t *pools, nvlist_t *props, char *mntopts, int flags,
 		}
 	}
 	if (import->do_all) {
-		tpool_wait(tp);
-		tpool_destroy(tp);
+		taskq_wait(tq);
+		taskq_destroy(tq);
 	}
 
 	/*

include/Makefile.am

@@ -189,8 +189,7 @@ USER_H = \
 	libzfs_core.h \
 	libzfsbootenv.h \
 	libzpool.h \
-	libzutil.h \
-	thread_pool.h
+	libzutil.h
 
 
 if CONFIG_USER

include/thread_pool.h

@@ -24,10 +24,10 @@
 #    libunicode --/       \        /       \    \-------\ \
 #                          \      /         \            \ |
 #                          libzutil        libzfs_core*  | |
-#                          | | | | \          |     |    | |
-#                          | | | | |          |     |    | |
-#                          | | | | |          |     |    | |
-#    libtpool -------------/ | | | \---- libnvpair* |    | |
+#                            | | | \          |     |    | |
+#                            | | | |          |     |    | |
+#                            | | | |          |     |    | |
+#                            | | | \---- libnvpair* |    | |
 #                            | | |                  |    | |
 #    libefi -----------------/ | \------ libavl* --------/ |
 #                              |                    |      |
@@ -58,7 +58,6 @@ include $(srcdir)/%D%/libicp/Makefile.am
 include $(srcdir)/%D%/libnvpair/Makefile.am
 include $(srcdir)/%D%/libshare/Makefile.am
 include $(srcdir)/%D%/libspl/Makefile.am
-include $(srcdir)/%D%/libtpool/Makefile.am
 include $(srcdir)/%D%/libunicode/Makefile.am
 include $(srcdir)/%D%/libzdb/Makefile.am
 include $(srcdir)/%D%/libzfs_core/Makefile.am

lib/Makefile.am

@@ -31,6 +31,8 @@
 
 #include <pthread.h>
 #include <stdint.h>
+#include <sys/kmem.h>
+#include <sys/thread.h>
 #include <sys/mutex.h>
 #include <sys/rwlock.h>
 #include <sys/condvar.h>