From 7b91fb69c2e805645de3e89da28532932908f699 Mon Sep 17 00:00:00 2001 From: Khairul Azhar Kasmiran Date: Mon, 27 May 2024 21:50:58 +0800 Subject: [PATCH] Revert #4506 (#4519) --- .github/workflows/ci.yml | 1 - binrz/rz-test/run.c | 2 +- binrz/rz-test/rz-test.c | 612 +++++++++++++++++---------------- binrz/rz-test/rz_test.h | 2 +- librz/arch/similarity.c | 2 +- librz/bin/bfile_string.c | 2 +- librz/core/cbin.c | 2 +- librz/core/cconfig.c | 10 +- librz/include/rz_basefind.h | 2 +- librz/include/rz_bin.h | 3 +- librz/include/rz_th.h | 21 +- librz/include/rz_userconf.h.in | 1 - librz/util/subprocess.c | 126 ++++--- librz/util/thread_iterators.c | 6 +- librz/util/thread_pool.c | 35 +- librz/util/thread_queue.c | 4 +- meson.build | 1 - meson_options.txt | 1 - test/unit/test_rz_test.c | 2 +- test/unit/test_threads.c | 33 +- 20 files changed, 419 insertions(+), 449 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e148dc68bc..7c4fee037a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -239,7 +239,6 @@ jobs: if: matrix.os == 'macos-12' && matrix.enabled run: | sudo security authorizationdb write system.privilege.taskport allow - sudo /usr/sbin/DevToolsSecurity --enable - name: Run unit tests (meson) continue-on-error: ${{ matrix.allow_failure }} if: matrix.build_system == 'meson' && matrix.enabled diff --git a/binrz/rz-test/run.c b/binrz/rz-test/run.c index a92cff146e..635cfbe723 100644 --- a/binrz/rz-test/run.c +++ b/binrz/rz-test/run.c @@ -558,7 +558,7 @@ RZ_API RzTestResultInfo *rz_test_run_test(RzTestRunConfig *config, RzTest *test) return ret; } -RZ_API void rz_test_result_info_free(RZ_NULLABLE RzTestResultInfo *result) { +RZ_API void rz_test_test_result_info_free(RzTestResultInfo *result) { if (!result) { return; } diff --git a/binrz/rz-test/rz-test.c b/binrz/rz-test/rz-test.c index c96558df59..25a9ae1938 100644 --- a/binrz/rz-test/rz-test.c +++ b/binrz/rz-test/rz-test.c @@ -15,6 +15,7 @@ #define Color_HLINSERT Color_BGINSERT Color_INSERT #define Color_HLDELETE Color_BGDELETE Color_DELETE +#define WORKERS_DEFAULT 8 #define RIZIN_CMD_DEFAULT "rizin" #define RZ_ASM_CMD_DEFAULT "rz-asm" #define JSON_TEST_FILE_DEFAULT "bins/elf/crackme0x00b" @@ -25,40 +26,36 @@ #define WORKERS_DEFAULT_STR STR(WORKERS_DEFAULT) #define TIMEOUT_DEFAULT_STR STR(TIMEOUT_DEFAULT) -typedef struct test_counter_t { - size_t n_tests; ///< Total number of tests - size_t n_handled; ///< Tests that has been handled - size_t ok; ///< Tests that succeeded - size_t xx; ///< Tests that failed - size_t fx; ///< Tests that succeeded but also marked as broken - size_t br; ///< Tests that failed but also marked as broken -} TestCounter; - -typedef struct test_return_t { - size_t max_path; - ut64 time_start; ///< Start time of the tests - bool interactive; ///< When true allows to review the tests interactively - st64 expected_succ; ///< Changes the return code if this value mismatches the actual total OK number - st64 expected_fail; ///< Changes the return code if this value mismatches the actual total XX number - int return_val; ///< Main return value - HtSP *path_counter; ///< Hashmap which is initialized when log_mode is true - RzAtomicBool *running; ///< Atomic boolean to stop result_thread -} ResultThreadData; +typedef struct rz_testfile_counts_t { + ut64 tests_left; // count of remaining tests + ut64 ok; + ut64 xx; + ut64 br; + ut64 fx; +} RzTestFileCounts; typedef struct rz_test_state_t { RzTestRunConfig run_config; bool verbose; RzTestDatabase *db; - PJ *pj; - ResultThreadData data; ///< Thread context for result_thread - RzThreadQueue *results; ///< Results of the executed tests. + PJ *test_results; + + RzThreadCond *cond; // signaled from workers to main thread to update status + RzThreadLock *lock; // protects everything below + HtSP *path_left; // char * (path to test file) => RzTestFileCounts * + RzPVector /**/ completed_paths; + ut64 ok_count; + ut64 xx_count; + ut64 br_count; + ut64 fx_count; + RzPVector /**/ queue; + RzPVector /**/ results; } RzTestState; -static void worker_thread(void *element, void *user); -static void *result_thread(void *user); -static void handle_result(RzTestState *state, RzTestResultInfo *result, TestCounter *counter); -static void print_counter(RzTestState *state, TestCounter *cnt); -static void interact(RzTestState *state, RzPVector /**/ *failed_results); +static void *worker_th(RzTestState *state); +static void print_state(RzTestState *state, ut64 prev_completed); +static void print_log(RzTestState *state, ut64 prev_completed, ut64 prev_paths_completed); +static void interact(RzTestState *state); static bool interact_fix(RzTestResultInfo *result, RzPVector /**/ *fixup_results); static void interact_break(RzTestResultInfo *result, RzPVector /**/ *fixup_results); static void interact_commands(RzTestResultInfo *result, RzPVector /**/ *fixup_results); @@ -198,29 +195,32 @@ static bool rz_test_chdir_fromtest(const char *test_path) { return found; } +static bool log_mode = false; + int rz_test_main(int argc, const char **argv) { - size_t n_threads = RZ_THREAD_N_CORES_ALL_AVAILABLE; + int workers_count = WORKERS_DEFAULT; bool verbose = false; bool nothing = false; bool quiet = false; + bool interactive = false; char *rizin_cmd = NULL; char *rz_asm_cmd = NULL; char *json_test_file = NULL; char *output_file = NULL; char *fuzz_dir = NULL; - bool log_mode = false; - RzTestState state = { 0 }; - ut64 timeout_sec = TIMEOUT_DEFAULT; - const char *rz_test_dir = NULL; RzPVector *except_dir = rz_pvector_new(free); + const char *rz_test_dir = NULL; + ut64 timeout_sec = TIMEOUT_DEFAULT; + st64 expect_succ = -1; + st64 expect_fail = -1; + int ret = 0; + if (!except_dir) { RZ_LOG_ERROR("Fail to create RzPVector\n"); - return -1; + ret = -1; + goto beach; } - state.data.expected_succ = -1; - state.data.expected_fail = -1; - #if __WINDOWS__ UINT old_cp = GetConsoleOutputCP(); { @@ -242,7 +242,7 @@ int rz_test_main(int argc, const char **argv) { while ((c = rz_getopt_next(&opt)) != -1) { switch (c) { case 'h': - state.data.return_val = help(true); + ret = help(true); goto beach; case 'q': quiet = true; @@ -255,13 +255,13 @@ int rz_test_main(int argc, const char **argv) { printf("%s\n", s); free(s); } - state.data.return_val = 0; + ret = 0; goto beach; case 'V': verbose = true; break; case 'i': - state.data.interactive = true; + interactive = true; break; case 'L': log_mode = true; @@ -271,10 +271,10 @@ int rz_test_main(int argc, const char **argv) { fuzz_dir = strdup(opt.arg); break; case 'j': - n_threads = atoi(opt.arg); - if (n_threads <= 0) { - eprintf("Invalid thread count (must be positive)\n"); - state.data.return_val = help(false); + workers_count = atoi(opt.arg); + if (workers_count <= 0) { + eprintf("Invalid thread count\n"); + ret = help(false); goto beach; } break; @@ -311,21 +311,21 @@ int rz_test_main(int argc, const char **argv) { break; case 's': // rz_num_math returns 0 for both '0' and invalid str - state.data.expected_succ = rz_num_math(NULL, opt.arg); - if (!rz_num_is_valid_input(NULL, opt.arg) || state.data.expected_succ < 0) { + expect_succ = rz_num_math(NULL, opt.arg); + if (!rz_num_is_valid_input(NULL, opt.arg) || expect_succ < 0) { RZ_LOG_ERROR("Number of expected successful tests is invalid\n"); goto beach; } break; case 'x': - state.data.expected_fail = rz_num_math(NULL, opt.arg); - if (!rz_num_is_valid_input(NULL, opt.arg) || state.data.expected_fail < 0) { + expect_fail = rz_num_math(NULL, opt.arg); + if (!rz_num_is_valid_input(NULL, opt.arg) || expect_fail < 0) { RZ_LOG_ERROR("Number of expected failed tests is invalid\n"); goto beach; } break; default: - state.data.return_val = help(false); + ret = help(false); goto beach; } } @@ -334,7 +334,7 @@ int rz_test_main(int argc, const char **argv) { if (rz_test_dir) { if (chdir(rz_test_dir) == -1) { eprintf("Cannot find %s directory.\n", rz_test_dir); - state.data.return_val = -1; + ret = -1; goto beach; } } else { @@ -343,7 +343,7 @@ int rz_test_main(int argc, const char **argv) { : rz_test_chdir(argv[0]); if (!dir_found) { eprintf("Cannot find db/ directory related to the given test.\n"); - state.data.return_val = -1; + ret = -1; goto beach; } } @@ -356,12 +356,14 @@ int rz_test_main(int argc, const char **argv) { if (!rz_subprocess_init()) { eprintf("Subprocess init failed\n"); - state.data.return_val = -1; + ret = -1; goto beach; } atexit(rz_subprocess_fini); rz_sys_setenv("TZ", "UTC"); + ut64 time_start = rz_time_now_mono(); + RzTestState state = { 0 }; // Avoid PATH search for each process launched if (!rizin_cmd) { rizin_cmd = rz_file_path(RIZIN_CMD_DEFAULT); @@ -376,13 +378,25 @@ int rz_test_main(int argc, const char **argv) { state.verbose = verbose; state.db = rz_test_test_database_new(); if (!state.db) { - state.data.return_val = -1; + ret = -1; goto beach; } - + rz_pvector_init(&state.queue, NULL); + rz_pvector_init(&state.results, (RzPVectorFree)rz_test_test_result_info_free); + rz_pvector_init(&state.completed_paths, NULL); if (output_file) { - state.pj = pj_new(); - pj_a(state.pj); + state.test_results = pj_new(); + pj_a(state.test_results); + } + state.lock = rz_th_lock_new(false); + if (!state.lock) { + ret = -1; + goto beach; + } + state.cond = rz_th_cond_new(); + if (!state.cond) { + ret = -1; + goto beach; } if (opt.ind < argc) { @@ -396,14 +410,14 @@ int rz_test_main(int argc, const char **argv) { eprintf("Category: %s\n", arg); if (!strcmp(arg, "unit")) { if (!rz_test_test_run_unit()) { - state.data.return_val = -1; + ret = -1; goto beach; } continue; } else if (!strcmp(arg, "fuzz")) { if (!fuzz_dir) { eprintf("No fuzz dir given. Use -F [dir]\n"); - state.data.return_val = -1; + ret = -1; goto beach; } if (!rz_test_test_database_load_fuzz(state.db, fuzz_dir)) { @@ -426,7 +440,7 @@ int rz_test_main(int argc, const char **argv) { rz_test_test_database_free(state.db); free(tf); free(alloc_arg); - state.data.return_val = -1; + ret = -1; goto beach; } RZ_FREE(alloc_arg); @@ -437,7 +451,7 @@ int rz_test_main(int argc, const char **argv) { if (!rz_test_test_database_load(state.db, "db")) { eprintf("Failed to load tests from ./db\n"); rz_test_test_database_free(state.db); - state.data.return_val = -1; + ret = -1; goto beach; } if (fuzz_dir && !rz_test_test_database_load_fuzz(state.db, fuzz_dir)) { @@ -467,30 +481,6 @@ int rz_test_main(int argc, const char **argv) { } } - if (log_mode) { - // Log mode prints the state after every completed file. - // The count of tests left per file is stored in a ht. - HtSP *path_counter = ht_sp_new(HT_STR_DUP, NULL, free); - state.data.max_path = 0; - if (path_counter) { - void **it; - rz_pvector_foreach (&state.db->tests, it) { - RzTest *test = *it; - TestCounter *cnt = ht_sp_find(path_counter, test->path, NULL); - if (!cnt) { - size_t path_len = strlen(test->path); - if (path_len > state.data.max_path) { - state.data.max_path = path_len; - } - cnt = RZ_NEW0(TestCounter); - ht_sp_insert(path_counter, test->path, cnt); - } - cnt->n_tests++; - } - } - state.data.path_counter = path_counter; - } - RZ_FREE(cwd); uint32_t loaded_tests = rz_pvector_len(&state.db->tests); printf("Loaded %u tests.\n", loaded_tests); @@ -513,52 +503,114 @@ int rz_test_main(int argc, const char **argv) { } } - if (rz_pvector_len(&state.db->tests) < 1) { + if (rz_pvector_len(&state.db->tests) != 0) { + rz_pvector_insert_range(&state.queue, 0, state.db->tests.v.a, rz_pvector_len(&state.db->tests)); + } else { eprintf("No tests discovered\n"); - goto coast; } - state.results = rz_th_queue_new(RZ_THREAD_QUEUE_UNLIMITED, (RzListFree)rz_test_result_info_free); - if (!state.results) { - eprintf("Failed to create result queue.\n"); - goto coast; + if (log_mode) { + // Log mode prints the state after every completed file. + // The count of tests left per file is stored in a ht. + state.path_left = ht_sp_new(HT_STR_DUP, NULL, free); + if (state.path_left) { + void **it; + rz_pvector_foreach (&state.queue, it) { + RzTest *test = *it; + RzTestFileCounts *counts = ht_sp_find(state.path_left, test->path, NULL); + if (!counts) { + counts = calloc(1, sizeof(RzTestFileCounts)); + ht_sp_insert(state.path_left, test->path, counts); + } + counts->tests_left++; + } + } } - state.data.running = rz_atomic_bool_new(true); - if (!state.data.running) { - eprintf("Failed to create atomic boolean.\n"); - goto coast; + rz_th_lock_enter(state.lock); + + RzPVector workers; + rz_pvector_init(&workers, NULL); + int i; + for (i = 0; i < workers_count; i++) { + RzThread *th = rz_th_new((RzThreadFunction)worker_th, &state); + if (!th) { + eprintf("Failed to start thread.\n"); + rz_th_lock_leave(state.lock); + exit(-1); + } + rz_pvector_push(&workers, th); } - RzThread *rth = rz_th_new(result_thread, &state); - if (!rth) { - eprintf("Failed to start the result thread.\n"); - return -1; + ut64 prev_completed = UT64_MAX; + ut64 prev_paths_completed = 0; + while (true) { + ut64 completed = (ut64)rz_pvector_len(&state.results); + if (log_mode) { + print_log(&state, prev_completed, prev_paths_completed); + } else if (completed != prev_completed) { + print_state(&state, prev_completed); + } + prev_completed = completed; + prev_paths_completed = (ut64)rz_pvector_len(&state.completed_paths); + if (completed == rz_pvector_len(&state.db->tests)) { + break; + } + rz_th_cond_wait(state.cond, state.lock); } - eprintf("Using %d threads\n", rz_th_max_threads(n_threads)); + rz_th_lock_leave(state.lock); - state.data.time_start = rz_time_now_mono(); - rz_th_iterate_pvector(&state.db->tests, worker_thread, n_threads, &state); + printf("\n"); - // notify the result thread that we finished running all tests. - rz_atomic_bool_set(state.data.running, false); + void **it; + rz_pvector_foreach (&workers, it) { + RzThread *th = *it; + rz_th_wait(th); + rz_th_free(th); + } + rz_pvector_clear(&workers); - // wait for the result thread to empty the queue. - rz_th_wait(rth); - rz_th_free(rth); + ut64 seconds = (rz_time_now_mono() - time_start) / 1000000; + printf("Finished in"); + if (seconds > 60) { + ut64 minutes = seconds / 60; + printf(" %" PFMT64u " minutes and", minutes); + seconds -= (minutes * 60); + } + printf(" %" PFMT64u " seconds.\n", seconds % 60); if (output_file) { - pj_end(state.pj); - char *results = pj_drain(state.pj); + pj_end(state.test_results); + char *results = pj_drain(state.test_results); rz_file_dump(output_file, (ut8 *)results, strlen(results), false); free(results); } + if (interactive) { + interact(&state); + } + + if (expect_succ > 0 && expect_succ != state.ok_count) { + ret = 1; + } + + if (expect_fail > 0 && expect_fail != state.xx_count) { + ret = 1; + } + + if (expect_fail < 0 && expect_succ < 0 && state.xx_count) { + ret = 1; + } + coast: - ht_sp_free(state.data.path_counter); - rz_atomic_bool_free(state.data.running); - rz_th_queue_free(state.results); + rz_pvector_clear(&state.queue); + rz_pvector_clear(&state.results); + rz_pvector_clear(&state.completed_paths); + rz_test_test_database_free(state.db); + rz_th_lock_free(state.lock); + rz_th_cond_free(state.cond); + ht_sp_free(state.path_left); beach: free(output_file); free(rizin_cmd); @@ -573,13 +625,11 @@ beach: (void)rz_sys_cmdf("chcp %u > NUL", old_cp); } #endif - return state.data.return_val; + return ret; } -static void result_to_json(RzTestState *state, RzTestResultInfo *result) { - rz_return_if_fail(result); - - PJ *pj = state->pj; +static void test_result_to_json(PJ *pj, RzTestResultInfo *result) { + rz_return_if_fail(pj && result); pj_o(pj); pj_k(pj, "type"); RzTest *test = result->test; @@ -624,68 +674,61 @@ static void result_to_json(RzTestState *state, RzTestResultInfo *result) { pj_end(pj); } -static void worker_thread(void *element, void *user) { - RzTest *test = (RzTest *)element; - RzTestState *state = (RzTestState *)user; - RzTestResultInfo *result = rz_test_run_test(&state->run_config, test); - rz_th_queue_push(state->results, result, true); -} - -static void *result_thread(void *user) { - TestCounter counter = { 0 }; - RzTestState *state = (RzTestState *)user; - RzTestResultInfo *result = NULL; - RzPVector failed_results = { 0 }; - - counter.n_tests = rz_pvector_len(&state->db->tests); - rz_pvector_init(&failed_results, (RzPVectorFree)rz_test_result_info_free); - +static void *worker_th(RzTestState *state) { + rz_th_lock_enter(state->lock); while (true) { - if (!(result = rz_th_queue_pop(state->results, false))) { - if (rz_atomic_bool_get(state->data.running)) { - rz_sys_usleep(250); - continue; - } - // queue is empty and there is nothing - // else to do. we terminate the loop + if (rz_pvector_empty(&state->queue)) { break; } + RzTest *test = rz_pvector_pop(&state->queue); + rz_th_lock_leave(state->lock); - handle_result(state, result, &counter); - if (state->data.interactive && result->result == RZ_TEST_RESULT_FAILED) { - rz_pvector_push(&failed_results, result); - } else { - rz_test_result_info_free(result); + RzTestResultInfo *result = rz_test_run_test(&state->run_config, test); + + rz_th_lock_enter(state->lock); + rz_pvector_push(&state->results, result); + if (!log_mode) { + switch (result->result) { + case RZ_TEST_RESULT_OK: + state->ok_count++; + break; + case RZ_TEST_RESULT_FAILED: + state->xx_count++; + break; + case RZ_TEST_RESULT_BROKEN: + state->br_count++; + break; + case RZ_TEST_RESULT_FIXED: + state->fx_count++; + break; + } } + if (state->path_left) { + RzTestFileCounts *counts = ht_sp_find(state->path_left, test->path, NULL); + if (counts) { + switch (result->result) { + case RZ_TEST_RESULT_OK: + counts->ok++; + break; + case RZ_TEST_RESULT_FAILED: + counts->xx++; + break; + case RZ_TEST_RESULT_BROKEN: + counts->br++; + break; + case RZ_TEST_RESULT_FIXED: + counts->fx++; + break; + } + counts->tests_left--; + if (!counts->tests_left) { + rz_pvector_push(&state->completed_paths, (void *)test->path); + } + } + } + rz_th_cond_signal(state->cond); } - ut64 seconds = (rz_time_now_mono() - state->data.time_start) / 1000000; - - print_counter(state, &counter); - printf("\nFinished in"); - if (seconds > 60) { - ut64 minutes = seconds / 60; - printf(" %" PFMT64u " minutes and", minutes); - seconds -= (minutes * 60); - } - printf(" %" PFMT64u " seconds.\n", seconds % 60); - - if (state->data.interactive) { - interact(state, &failed_results); - rz_pvector_clear(&failed_results); - } - - if (state->data.expected_succ > 0 && state->data.expected_succ != counter.ok) { - state->data.return_val = 1; - } - - if (state->data.expected_fail > 0 && state->data.expected_fail != counter.xx) { - state->data.return_val = 1; - } - - if (state->data.expected_fail < 0 && state->data.expected_succ < 0 && counter.xx) { - state->data.return_val = 1; - } - + rz_th_lock_leave(state->lock); return NULL; } @@ -827,143 +870,110 @@ static void print_result_diff(RzTestRunConfig *config, RzTestResultInfo *result) } } -static void print_path_completion(RzTestState *state, RzTestResultInfo *result) { - if (!state->data.path_counter) { - return; +static void print_new_results(RzTestState *state, ut64 prev_completed) { + // Detailed test result (with diff if necessary) + ut64 completed = (ut64)rz_pvector_len(&state->results); + ut64 i; + for (i = prev_completed; i < completed; i++) { + RzTestResultInfo *result = rz_pvector_at(&state->results, (size_t)i); + if (state->test_results) { + test_result_to_json(state->test_results, result); + } + if (!state->verbose && (result->result == RZ_TEST_RESULT_OK || result->result == RZ_TEST_RESULT_FIXED || result->result == RZ_TEST_RESULT_BROKEN)) { + continue; + } + char *name = rz_test_test_name(result->test); + if (!name) { + continue; + } + printf("\n" RZ_CONS_CURSOR_UP RZ_CONS_CLEAR_LINE); + switch (result->result) { + case RZ_TEST_RESULT_OK: + printf(Color_GREEN "[OK]" Color_RESET); + break; + case RZ_TEST_RESULT_FAILED: + printf(Color_RED "[XX]" Color_RESET); + break; + case RZ_TEST_RESULT_BROKEN: + printf(Color_BLUE "[BR]" Color_RESET); + break; + case RZ_TEST_RESULT_FIXED: + printf(Color_CYAN "[FX]" Color_RESET); + break; + } + if (result->timeout) { + printf(Color_CYAN " TIMEOUT" Color_RESET); + } + printf(" %s " Color_YELLOW "%s" Color_RESET "\n", result->test->path, name); + if (result->result == RZ_TEST_RESULT_FAILED || (state->verbose && result->result == RZ_TEST_RESULT_BROKEN)) { + print_result_diff(&state->run_config, result); + } + free(name); } - - const char *path = result->test->path; - if (!path) { - rz_warn_if_reached(); - return; - } - - TestCounter *cnt = ht_sp_find(state->data.path_counter, path, NULL); - if (!cnt) { - rz_warn_if_reached(); - return; - } - - cnt->n_handled++; - switch (result->result) { - case RZ_TEST_RESULT_OK: - cnt->ok++; - break; - case RZ_TEST_RESULT_FAILED: - cnt->xx++; - break; - case RZ_TEST_RESULT_BROKEN: - cnt->br++; - break; - case RZ_TEST_RESULT_FIXED: - cnt->fx++; - break; - default: - rz_warn_if_reached(); - break; - } - - if (cnt->n_handled < cnt->n_tests) { - return; - } - - const int max_path = state->data.max_path; - - if (cnt->xx > 0) { - printf(Color_RED "[XX]" Color_RESET " %-*s %8" PFMTSZu " OK %8" PFMTSZu " BR %8" PFMTSZu " XX %8" PFMTSZu " FX\n", - max_path, path, cnt->ok, cnt->br, cnt->xx, cnt->fx); - } else { - printf(Color_GREEN "[OK]" Color_RESET " %-*s %8" PFMTSZu " OK %8" PFMTSZu " BR %8" PFMTSZu " XX %8" PFMTSZu " FX\n", - max_path, path, cnt->ok, cnt->br, cnt->xx, cnt->fx); - } - - // free the counter. - ht_sp_delete(state->data.path_counter, path); } -static void print_counter(RzTestState *state, TestCounter *cnt) { - printf(RZ_CONS_CLEAR_LINE "[%" PFMTSZu "/%" PFMTSZu "]", cnt->n_tests, cnt->n_handled); - printf(" %8" PFMTSZu " OK %8" PFMTSZu " BR %8" PFMTSZu " XX %8" PFMTSZu " FX", cnt->ok, cnt->br, cnt->xx, cnt->fx); +static void print_state_counts(RzTestState *state) { + printf("%8" PFMT64u " OK %8" PFMT64u " BR %8" PFMT64u " XX %8" PFMT64u " FX", + state->ok_count, state->br_count, state->xx_count, state->fx_count); } -static void print_result(RzTestState *state, RzTestResultInfo *result) { - if (!state->verbose && result->result != RZ_TEST_RESULT_FAILED) { - return; - } - char *name = rz_test_test_name(result->test); - if (!name) { - return; - } - printf("\n" RZ_CONS_CURSOR_UP RZ_CONS_CLEAR_LINE); - switch (result->result) { - case RZ_TEST_RESULT_OK: - printf(Color_GREEN "[OK]" Color_RESET); - break; - case RZ_TEST_RESULT_FAILED: - printf(Color_RED "[XX]" Color_RESET); - break; - case RZ_TEST_RESULT_BROKEN: - printf(Color_BLUE "[BR]" Color_RESET); - break; - case RZ_TEST_RESULT_FIXED: - printf(Color_CYAN "[FX]" Color_RESET); - break; - } - if (result->timeout) { - printf(Color_CYAN " TIMEOUT" Color_RESET); - } - printf(" %s " Color_YELLOW "%s" Color_RESET "\n", result->test->path, name); - if (result->result == RZ_TEST_RESULT_FAILED || (state->verbose && result->result == RZ_TEST_RESULT_BROKEN)) { - print_result_diff(&state->run_config, result); - } - free(name); -} - -static void handle_result(RzTestState *state, RzTestResultInfo *result, TestCounter *counter) { +static void print_state(RzTestState *state, ut64 prev_completed) { #if __WINDOWS__ setvbuf(stdout, NULL, _IOFBF, 8192); #endif + print_new_results(state, prev_completed); - counter->n_handled++; - switch (result->result) { - case RZ_TEST_RESULT_OK: - counter->ok++; - break; - case RZ_TEST_RESULT_FAILED: - counter->xx++; - break; - case RZ_TEST_RESULT_BROKEN: - counter->br++; - break; - case RZ_TEST_RESULT_FIXED: - counter->fx++; - break; - default: - rz_warn_if_reached(); - break; + // [x/x] OK 42 BR 0 ... + printf(RZ_CONS_CLEAR_LINE); + int w = printf("[%" PFMT64u "/%" PFMT64u "]", (ut64)rz_pvector_len(&state->results), (ut64)rz_pvector_len(&state->db->tests)); + while (w >= 0 && w < 20) { + printf(" "); + w++; } - - if (state->pj) { - result_to_json(state, result); - } - - print_result(state, result); - print_path_completion(state, result); - if (!state->data.path_counter) { - print_counter(state, counter); - } - - // ensure to flush stdout + printf(" "); + print_state_counts(state); fflush(stdout); - #if __WINDOWS__ setvbuf(stdout, NULL, _IONBF, 0); #endif } -static void interact(RzTestState *state, RzPVector /**/ *failed_results) { - if (rz_pvector_empty(failed_results)) { - return; +static void print_log(RzTestState *state, ut64 prev_completed, ut64 prev_paths_completed) { + print_new_results(state, prev_completed); + ut64 paths_completed = rz_pvector_len(&state->completed_paths); + for (; prev_paths_completed < paths_completed; prev_paths_completed++) { + const char *name = (const char *)rz_pvector_at(&state->completed_paths, prev_paths_completed); + if (!name) { + name = "unknown path. something is very wrong."; + } + printf("[**] %50s ", name); + if (state->path_left) { + RzTestFileCounts *counts = ht_sp_find(state->path_left, name, NULL); + if (counts) { + state->ok_count += counts->ok; + state->xx_count += counts->xx; + state->br_count += counts->br; + state->fx_count += counts->fx; + } + } + print_state_counts(state); + printf("\n"); + fflush(stdout); + } +} + +static void interact(RzTestState *state) { + void **it; + RzPVector failed_results; + rz_pvector_init(&failed_results, NULL); + rz_pvector_foreach (&state->results, it) { + RzTestResultInfo *result = *it; + if (result->result == RZ_TEST_RESULT_FAILED) { + rz_pvector_push(&failed_results, result); + } + } + if (rz_pvector_empty(&failed_results)) { + goto beach; } #if __WINDOWS__ @@ -972,11 +982,10 @@ static void interact(RzTestState *state, RzPVector /**/ *fai printf("\n"); printf("#####################\n"); printf(" %" PFMT64u " failed test(s) " UTF8_POLICE_CARS_REVOLVING_LIGHT "\n", - (ut64)rz_pvector_len(failed_results)); + (ut64)rz_pvector_len(&failed_results)); - void **it; ut32 cnt = 0; - rz_pvector_foreach (failed_results, it) { + rz_pvector_foreach (&failed_results, it) { cnt++; RzTestResultInfo *result = *it; if (result->test->type != RZ_TEST_TYPE_CMD && result->test->type != RZ_TEST_TYPE_ASM) { @@ -986,7 +995,7 @@ static void interact(RzTestState *state, RzPVector /**/ *fai printf("#####################\n\n"); char *name = rz_test_test_name(result->test); if (name) { - printf(Color_RED "[XX]" Color_RESET " %s " Color_YELLOW "%s" Color_RESET " (%d/%zu)\n", result->test->path, name, cnt, rz_pvector_len(failed_results)); + printf(Color_RED "[XX]" Color_RESET " %s " Color_YELLOW "%s" Color_RESET " (%d/%zu)\n", result->test->path, name, cnt, rz_pvector_len(&failed_results)); free(name); } print_result_diff(&state->run_config, result); @@ -1009,7 +1018,7 @@ static void interact(RzTestState *state, RzPVector /**/ *fai } switch (buf[0]) { case 'f': - if (!interact_fix(result, failed_results)) { + if (!interact_fix(result, &failed_results)) { printf("This test has failed too hard to be fixed.\n"); goto menu; } @@ -1017,20 +1026,23 @@ static void interact(RzTestState *state, RzPVector /**/ *fai case 'i': break; case 'b': - interact_break(result, failed_results); + interact_break(result, &failed_results); break; case 'c': if (have_commands) { - interact_commands(result, failed_results); + interact_commands(result, &failed_results); break; } goto menu; case 'q': - return; + goto beach; default: goto menu; } } + +beach: + rz_pvector_clear(&failed_results); } static char *format_cmd_kv(const char *key, const char *val) { diff --git a/binrz/rz-test/rz_test.h b/binrz/rz-test/rz_test.h index 88860942be..ff13837fd7 100644 --- a/binrz/rz-test/rz_test.h +++ b/binrz/rz-test/rz_test.h @@ -213,6 +213,6 @@ RZ_API void rz_test_test_free(RzTest *test); RZ_API char *rz_test_test_name(RzTest *test); RZ_API bool rz_test_test_broken(RzTest *test); RZ_API RzTestResultInfo *rz_test_run_test(RzTestRunConfig *config, RzTest *test); -RZ_API void rz_test_result_info_free(RZ_NULLABLE RzTestResultInfo *result); +RZ_API void rz_test_test_result_info_free(RzTestResultInfo *result); #endif // RIZIN_RZTEST_H diff --git a/librz/arch/similarity.c b/librz/arch/similarity.c index 1ff9a7181d..92db9c7710 100644 --- a/librz/arch/similarity.c +++ b/librz/arch/similarity.c @@ -276,7 +276,7 @@ static RZ_OWN RzAnalysisMatchResult *analysis_match_result_new(RZ_NONNULL RzAnal RzAnalysisMatchResult *result = NULL; RzList *unmatch_a = rz_list_newf((RzListFree)free); RzList *unmatch_b = rz_list_clone(list_b); - RzThreadPool *pool = rz_th_pool_new(RZ_THREAD_N_CORES_ALL_AVAILABLE); + RzThreadPool *pool = rz_th_pool_new(RZ_THREAD_POOL_ALL_CORES); RzThread *user_thread = NULL; SharedContext shared = { 0 }; MatchUIInfo ui_info = { 0 }; diff --git a/librz/bin/bfile_string.c b/librz/bin/bfile_string.c index 9654253fae..8b16de205d 100644 --- a/librz/bin/bfile_string.c +++ b/librz/bin/bfile_string.c @@ -310,7 +310,7 @@ static void scan_cfstring_table(RzBinFile *bf, HtUP *strings_db, RzPVector /*max_threads = RZ_THREAD_N_CORES_ALL_AVAILABLE; + opt->max_threads = RZ_THREAD_POOL_ALL_CORES; opt->min_length = RZ_BIN_STRING_SEARCH_MIN_STRING; opt->buffer_size = RZ_BIN_STRING_SEARCH_BUFFER_SIZE; opt->max_uni_blocks = RZ_BIN_STRING_SEARCH_MAX_UNI_BLOCKS; diff --git a/librz/core/cbin.c b/librz/core/cbin.c index e636ccc54e..baea0038dc 100644 --- a/librz/core/cbin.c +++ b/librz/core/cbin.c @@ -2595,7 +2595,7 @@ RZ_API bool rz_core_bin_basefind_print(RzCore *core, ut32 pointer_size, RzCmdSta // ensure the last printed line is actually the last expected line // this depends on the number of the threads requested and available // this requires to be called before checking the results - int n_cores = (int)rz_th_max_threads(options.max_threads); + int n_cores = (int)rz_th_request_physical_cores(options.max_threads); rz_cons_gotoxy(1, begin_line + n_cores); } diff --git a/librz/core/cconfig.c b/librz/core/cconfig.c index 772c8a60d3..aec338bb41 100644 --- a/librz/core/cconfig.c +++ b/librz/core/cconfig.c @@ -1114,12 +1114,12 @@ static bool cb_str_escbslash(void *user, void *data) { static bool cb_str_search_max_threads(void *user, void *data) { RzCore *core = (RzCore *)user; RzConfigNode *node = (RzConfigNode *)data; - RzThreadNCores max_threads = rz_th_max_threads(node->i_value); + size_t max_threads = rz_th_physical_core_number(); if (node->value[0] == '?') { - rz_cons_printf("%d\n", max_threads); + rz_cons_printf("%" PFMTSZu "\n", max_threads); return false; } - core->bin->str_search_cfg.max_threads = max_threads; + core->bin->str_search_cfg.max_threads = RZ_MIN(max_threads, node->i_value); return true; } @@ -3692,7 +3692,7 @@ RZ_API int rz_core_config_init(RzCore *core) { /* string search options */ SETB("str.search.reload", true, "When enabled, any change to any option `str.search.*` will reload the bin strings."); - SETICB("str.search.max_threads", RZ_THREAD_N_CORES_ALL_AVAILABLE, &cb_str_search_max_threads, "Maximum core number (0 for all cores)."); + SETICB("str.search.max_threads", RZ_THREAD_POOL_ALL_CORES, &cb_str_search_max_threads, "Maximum core number (0 for all cores)."); SETICB("str.search.min_length", RZ_BIN_STRING_SEARCH_MIN_STRING, &cb_str_search_min_length, "Smallest string length that is possible to find."); SETICB("str.search.buffer_size", RZ_BIN_STRING_SEARCH_BUFFER_SIZE, &cb_str_search_buffer_size, "Maximum buffer size, which will also determine the maximum string length."); SETICB("str.search.max_uni_blocks", RZ_BIN_STRING_SEARCH_MAX_UNI_BLOCKS, &cb_str_search_max_uni_blocks, "Maximum number of unicode blocks."); @@ -3780,7 +3780,7 @@ RZ_API int rz_core_config_init(RzCore *core) { SETI("basefind.alignment", RZ_BASEFIND_BASE_ALIGNMENT, "Basefind alignment in bytes"); SETI("basefind.min.score", RZ_BASEFIND_SCORE_MIN_VALUE, "Basefind min score value to consider it valid"); SETI("basefind.min.string", RZ_BASEFIND_STRING_MIN_LENGTH, "Basefind min string size to find to consider it valid"); - SETI("basefind.max.threads", RZ_THREAD_N_CORES_ALL_AVAILABLE, "Basefind max threads number (when 0 uses all available cores)"); + SETI("basefind.max.threads", RZ_THREAD_POOL_ALL_CORES, "Basefind max threads number (when 0 uses all available cores)"); /* nkeys */ SETPREF("key.s", "", "override step into action"); diff --git a/librz/include/rz_basefind.h b/librz/include/rz_basefind.h index 5337a92dd1..553e680dc3 100644 --- a/librz/include/rz_basefind.h +++ b/librz/include/rz_basefind.h @@ -34,7 +34,7 @@ typedef struct rz_basefind_info_t { typedef bool (*RzBaseFindThreadInfoCb)(const RzBaseFindThreadInfo *th_info, void *user); typedef struct rz_basefind_options_t { - RzThreadNCores max_threads; ///< Max requested number of threads (not guaranteed). + size_t max_threads; ///< Max requested number of threads (not guaranteed). ut32 pointer_size; ///< Pointer size in bits (32 or 64) ut64 start_address; ///< Start search address ut64 end_address; ///< End search address diff --git a/librz/include/rz_bin.h b/librz/include/rz_bin.h index 592981a30f..f46419e038 100644 --- a/librz/include/rz_bin.h +++ b/librz/include/rz_bin.h @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -201,7 +200,7 @@ typedef enum { } RzBinStringSearchMode; typedef struct rz_bin_string_search_opt_t { - RzThreadNCores max_threads; ///< Maximum thread number (normally set to RZ_THREAD_N_CORES_ALL_AVAILABLE). + size_t max_threads; ///< Maximum thread number (normally set to RZ_THREAD_POOL_ALL_CORES). size_t min_length; ///< Smallest string length that is possible to find. size_t buffer_size; ///< Maximum buffer size, which will also determine the maximum string length. size_t max_uni_blocks; ///< Maximum number of unicode blocks diff --git a/librz/include/rz_th.h b/librz/include/rz_th.h index 7332c881f2..913667092c 100644 --- a/librz/include/rz_th.h +++ b/librz/include/rz_th.h @@ -19,13 +19,8 @@ extern "C" { #endif -typedef enum { - RZ_THREAD_N_CORES_ALL_AVAILABLE = 0, -} RzThreadNCores; - -typedef enum { - RZ_THREAD_QUEUE_UNLIMITED = 0, -} RzThreadQueueSize; +#define RZ_THREAD_POOL_ALL_CORES (0) +#define RZ_THREAD_QUEUE_UNLIMITED (0) typedef struct rz_th_sem_t RzThreadSemaphore; typedef struct rz_th_lock_t RzThreadLock; @@ -66,17 +61,17 @@ RZ_API void rz_th_cond_signal_all(RZ_NONNULL RzThreadCond *cond); RZ_API void rz_th_cond_wait(RZ_NONNULL RzThreadCond *cond, RZ_NONNULL RzThreadLock *lock); RZ_API void rz_th_cond_free(RZ_NULLABLE RzThreadCond *cond); -RZ_API RzThreadNCores rz_th_physical_core_number(); -RZ_API RzThreadNCores rz_th_max_threads(RzThreadNCores requested); +RZ_API size_t rz_th_physical_core_number(); +RZ_API size_t rz_th_request_physical_cores(size_t max_cores); -RZ_API RZ_OWN RzThreadPool *rz_th_pool_new(RzThreadNCores max_threads); +RZ_API RZ_OWN RzThreadPool *rz_th_pool_new(size_t max_threads); RZ_API void rz_th_pool_free(RZ_NULLABLE RzThreadPool *pool); RZ_API bool rz_th_pool_add_thread(RZ_NONNULL RzThreadPool *pool, RZ_NONNULL RzThread *thread); RZ_API RZ_BORROW RzThread *rz_th_pool_get_thread(RZ_NONNULL RzThreadPool *pool, size_t index); RZ_API bool rz_th_pool_wait(RZ_NONNULL RzThreadPool *pool); RZ_API size_t rz_th_pool_size(RZ_NONNULL RzThreadPool *pool); -RZ_API RZ_OWN RzThreadQueue *rz_th_queue_new(RzThreadQueueSize max_size, RZ_NULLABLE RzListFree qfree); +RZ_API RZ_OWN RzThreadQueue *rz_th_queue_new(size_t max_size, RZ_NULLABLE RzListFree qfree); RZ_API RZ_OWN RzThreadQueue *rz_th_queue_from_list(RZ_NONNULL RZ_BORROW RzList /**/ *list, RZ_NULLABLE RzListFree qfree); RZ_API RZ_OWN RzThreadQueue *rz_th_queue_from_pvector(RZ_NONNULL RZ_BORROW RzPVector /**/ *vector, RZ_NULLABLE RzListFree qfree); RZ_API void rz_th_queue_free(RZ_NULLABLE RzThreadQueue *queue); @@ -93,8 +88,8 @@ RZ_API void rz_atomic_bool_free(RZ_NULLABLE RzAtomicBool *tbool); RZ_API bool rz_atomic_bool_get(RZ_NONNULL RzAtomicBool *tbool); RZ_API void rz_atomic_bool_set(RZ_NONNULL RzAtomicBool *tbool, bool value); -RZ_API bool rz_th_iterate_list(RZ_NONNULL const RzList /**/ *list, RZ_NONNULL RzThreadIterator iterator, RzThreadNCores max_threads, RZ_NULLABLE void *user); -RZ_API bool rz_th_iterate_pvector(RZ_NONNULL const RzPVector /**/ *pvec, RZ_NONNULL RzThreadIterator iterator, RzThreadNCores max_threads, RZ_NULLABLE void *user); +RZ_API bool rz_th_iterate_list(RZ_NONNULL const RzList /**/ *list, RZ_NONNULL RzThreadIterator iterator, size_t max_threads, RZ_NULLABLE void *user); +RZ_API bool rz_th_iterate_pvector(RZ_NONNULL const RzPVector /**/ *pvec, RZ_NONNULL RzThreadIterator iterator, size_t max_threads, RZ_NULLABLE void *user); #endif /* RZ_API */ diff --git a/librz/include/rz_userconf.h.in b/librz/include/rz_userconf.h.in index e7e48ad4e0..d664018706 100644 --- a/librz/include/rz_userconf.h.in +++ b/librz/include/rz_userconf.h.in @@ -47,7 +47,6 @@ #define IS_IOS @IS_IOS@ #define RZ_BUILD_DEBUG @RZ_BUILD_DEBUG@ #define WITH_SWIFT_DEMANGLER @WITH_SWIFT_DEMANGLER@ -#define N_THREAD_LIMIT @N_THREAD_LIMIT@ #define HAVE_COPYFILE @HAVE_COPYFILE@ #define HAVE_COPY_FILE_RANGE @HAVE_COPY_FILE_RANGE@ #define HAVE_BACKTRACE @HAVE_BACKTRACE@ diff --git a/librz/util/subprocess.c b/librz/util/subprocess.c index fc97bf7364..8b0da9f31e 100644 --- a/librz/util/subprocess.c +++ b/librz/util/subprocess.c @@ -47,18 +47,13 @@ struct rz_subprocess_t { #define INVALID_POINTER_VALUE ((void *)PTRDIFF_MAX) -typedef struct subprocess_windows_t { - RzThreadLock *subproc_mutex; - long refcount; - bool has_procthreadattr; - volatile long pipe_id; - DWORD mode_stdin; - DWORD mode_stdout; - DWORD mode_stderr; -} SubprocessWindows; - -// This structure is used by init/fini -static SubprocessWindows subwin = { 0 }; +static RzThreadLock *subproc_mutex = NULL; +static long refcount = 0; +static bool has_procthreadattr = false; +static volatile long pipe_id = 0; +static DWORD mode_stdin; +static DWORD mode_stdout; +static DWORD mode_stderr; static bool create_pipe_overlap(HANDLE *pipe_read, HANDLE *pipe_write, LPSECURITY_ATTRIBUTES attrs, DWORD sz, DWORD read_mode, DWORD write_mode) { // see https://stackoverflow.com/a/419736 @@ -66,7 +61,7 @@ static bool create_pipe_overlap(HANDLE *pipe_read, HANDLE *pipe_write, LPSECURIT sz = 4096; } WCHAR name[MAX_PATH]; - _snwprintf_s(name, _countof(name), sizeof(name), L"\\\\.\\pipe\\rz-pipe-subproc.%d.%ld", (int)GetCurrentProcessId(), (long)InterlockedIncrement(&subwin.pipe_id)); + _snwprintf_s(name, _countof(name), sizeof(name), L"\\\\.\\pipe\\rz-pipe-subproc.%d.%ld", (int)GetCurrentProcessId(), (long)InterlockedIncrement(&pipe_id)); *pipe_read = CreateNamedPipeW(name, PIPE_ACCESS_INBOUND | read_mode, PIPE_TYPE_BYTE | PIPE_WAIT, 1, sz, sz, 120 * 1000, attrs); if (!*pipe_read) { return FALSE; @@ -84,29 +79,29 @@ static bool create_pipe_overlap(HANDLE *pipe_read, HANDLE *pipe_write, LPSECURIT static RzThreadLock *get_subprocess_lock(void) { RzThreadLock *lock; do { - lock = InterlockedCompareExchangePointer(&subwin.subproc_mutex, INVALID_POINTER_VALUE, INVALID_POINTER_VALUE); + lock = InterlockedCompareExchangePointer(&subproc_mutex, INVALID_POINTER_VALUE, INVALID_POINTER_VALUE); } while (!lock); return lock; } RZ_API bool rz_subprocess_init(void) { - long ref = InterlockedIncrement(&subwin.refcount); + long ref = InterlockedIncrement(&refcount); RzThreadLock *lock = NULL; if (ref == 1) { lock = rz_th_lock_new(false); if (!lock) { - InterlockedExchangePointer(&subwin.subproc_mutex, INVALID_POINTER_VALUE); - InterlockedDecrement(&subwin.refcount); + InterlockedExchangePointer(&subproc_mutex, INVALID_POINTER_VALUE); + InterlockedDecrement(&refcount); return false; } // Enter lock before making it available, so we are the first to run rz_th_lock_enter(lock); - InterlockedExchangePointer(&subwin.subproc_mutex, lock); + InterlockedExchangePointer(&subproc_mutex, lock); } else { // Spin until theres a lock available or lock initialization failed lock = get_subprocess_lock(); if (lock == INVALID_POINTER_VALUE) { - InterlockedDecrement(&subwin.refcount); + InterlockedDecrement(&refcount); return false; } rz_th_lock_enter(lock); @@ -118,12 +113,12 @@ RZ_API bool rz_subprocess_init(void) { } // Save current console mode - GetConsoleMode(GetStdHandle(STD_INPUT_HANDLE), &subwin.mode_stdin); - GetConsoleMode(GetStdHandle(STD_OUTPUT_HANDLE), &subwin.mode_stdout); - GetConsoleMode(GetStdHandle(STD_ERROR_HANDLE), &subwin.mode_stderr); + GetConsoleMode(GetStdHandle(STD_INPUT_HANDLE), &mode_stdin); + GetConsoleMode(GetStdHandle(STD_OUTPUT_HANDLE), &mode_stdout); + GetConsoleMode(GetStdHandle(STD_ERROR_HANDLE), &mode_stderr); #if NTDDI_VERSION >= NTDDI_VISTA - if (!subwin.has_procthreadattr && IsWindowsVistaOrGreater()) { + if (!has_procthreadattr && IsWindowsVistaOrGreater()) { HMODULE kernel32 = LoadLibraryW(L"kernel32"); if (!kernel32) { rz_sys_perror("LoadLibraryW(L\"kernel32\")"); @@ -133,7 +128,7 @@ RZ_API bool rz_subprocess_init(void) { lpUpdateProcThreadAttribute = (UpdateProcThreadAttribute_t)GetProcAddress(kernel32, "UpdateProcThreadAttribute"); lpDeleteProcThreadAttributeList = (DeleteProcThreadAttributeList_t)GetProcAddress(kernel32, "DeleteProcThreadAttributeList"); if (lpInitializeProcThreadAttributeList && lpUpdateProcThreadAttribute && lpDeleteProcThreadAttributeList) { - subwin.has_procthreadattr = true; + has_procthreadattr = true; } FreeLibrary(kernel32); } @@ -145,22 +140,22 @@ leave: RZ_API void rz_subprocess_fini(void) { RzThreadLock *lock = NULL; do { - if (InterlockedCompareExchange(&subwin.refcount, -1, -1) == 0) { + if (InterlockedCompareExchange(&refcount, -1, -1) == 0) { // Shouldn't happen, someone called this function excessively rz_warn_if_reached(); return; } - lock = InterlockedExchangePointer(&subwin.subproc_mutex, NULL); + lock = InterlockedExchangePointer(&subproc_mutex, NULL); } while (!lock); - if (InterlockedDecrement(&subwin.refcount) > 0) { - InterlockedExchangePointer(&subwin.subproc_mutex, lock); + if (InterlockedDecrement(&refcount) > 0) { + InterlockedExchangePointer(&subproc_mutex, lock); return; } SetEnvironmentVariableW(L"RZ_PIPE_PATH", NULL); // Restore console mode - SetConsoleMode(GetStdHandle(STD_INPUT_HANDLE), subwin.mode_stdin); - SetConsoleMode(GetStdHandle(STD_OUTPUT_HANDLE), subwin.mode_stdout); - SetConsoleMode(GetStdHandle(STD_ERROR_HANDLE), subwin.mode_stderr); + SetConsoleMode(GetStdHandle(STD_INPUT_HANDLE), mode_stdin); + SetConsoleMode(GetStdHandle(STD_OUTPUT_HANDLE), mode_stdout); + SetConsoleMode(GetStdHandle(STD_ERROR_HANDLE), mode_stderr); rz_th_lock_free(lock); } @@ -378,7 +373,7 @@ RZ_API RZ_OWN RzSubprocess *rz_subprocess_start_opt(RZ_NONNULL const RzSubproces STARTUPINFOW *start_info = &start_info_short; #if NTDDI_VERSION >= NTDDI_VISTA STARTUPINFOEXW start_infoex = { .StartupInfo.cb = sizeof(STARTUPINFOEXW) }; - if (subwin.has_procthreadattr) { + if (has_procthreadattr) { SIZE_T attr_list_size = 0; if (!lpInitializeProcThreadAttributeList(NULL, 1, 0, &attr_list_size) && GetLastError() != ERROR_INSUFFICIENT_BUFFER) { @@ -751,33 +746,28 @@ struct rz_subprocess_t { int slave_fd; }; -typedef struct subprocess_unix_t { - RzPVector /**/ subprocs; - RzThreadLock *subprocs_mutex; - int sigchld_pipe[2]; - RzThread *sigchld_thread; -} SubprocessUnix; - -// This structure is used by init/fini -static SubprocessUnix subnix = { 0 }; +static RzPVector subprocs; +static RzThreadLock *subprocs_mutex; +static int sigchld_pipe[2]; +static RzThread *sigchld_thread; static void subprocess_lock(void) { - rz_th_lock_enter(subnix.subprocs_mutex); + rz_th_lock_enter(subprocs_mutex); } static void subprocess_unlock(void) { - rz_th_lock_leave(subnix.subprocs_mutex); + rz_th_lock_leave(subprocs_mutex); } static void handle_sigchld(int sig) { ut8 b = 1; - rz_xwrite(subnix.sigchld_pipe[1], &b, 1); + rz_xwrite(sigchld_pipe[1], &b, 1); } static void *sigchld_th(void *th) { while (true) { ut8 b; - ssize_t rd = read(subnix.sigchld_pipe[0], &b, 1); + ssize_t rd = read(sigchld_pipe[0], &b, 1); if (rd <= 0) { if (rd < 0) { if (errno == EINTR) { @@ -799,7 +789,7 @@ static void *sigchld_th(void *th) { subprocess_lock(); void **it; RzSubprocess *proc = NULL; - rz_pvector_foreach (&subnix.subprocs, it) { + rz_pvector_foreach (&subprocs, it) { RzSubprocess *p = *it; if (p->pid == pid) { proc = p; @@ -825,27 +815,27 @@ static void *sigchld_th(void *th) { } RZ_API bool rz_subprocess_init(void) { - rz_pvector_init(&subnix.subprocs, NULL); - subnix.subprocs_mutex = rz_th_lock_new(true); - if (!subnix.subprocs_mutex) { + rz_pvector_init(&subprocs, NULL); + subprocs_mutex = rz_th_lock_new(true); + if (!subprocs_mutex) { return false; } - if (rz_sys_pipe(subnix.sigchld_pipe, true) == -1) { + if (rz_sys_pipe(sigchld_pipe, true) == -1) { perror("pipe"); - rz_th_lock_free(subnix.subprocs_mutex); + rz_th_lock_free(subprocs_mutex); return false; } - subnix.sigchld_thread = rz_th_new(sigchld_th, NULL); - if (!subnix.sigchld_thread) { - rz_sys_pipe_close(subnix.sigchld_pipe[0]); - rz_sys_pipe_close(subnix.sigchld_pipe[1]); - rz_th_lock_free(subnix.subprocs_mutex); + sigchld_thread = rz_th_new(sigchld_th, NULL); + if (!sigchld_thread) { + rz_sys_pipe_close(sigchld_pipe[0]); + rz_sys_pipe_close(sigchld_pipe[1]); + rz_th_lock_free(subprocs_mutex); return false; } if (rz_sys_signal(SIGCHLD, handle_sigchld) < 0) { - rz_sys_pipe_close(subnix.sigchld_pipe[0]); - rz_sys_pipe_close(subnix.sigchld_pipe[1]); - rz_th_lock_free(subnix.subprocs_mutex); + rz_sys_pipe_close(sigchld_pipe[0]); + rz_sys_pipe_close(sigchld_pipe[1]); + rz_th_lock_free(subprocs_mutex); return false; } return true; @@ -854,13 +844,13 @@ RZ_API bool rz_subprocess_init(void) { RZ_API void rz_subprocess_fini(void) { rz_sys_signal(SIGCHLD, SIG_IGN); ut8 b = 0; - rz_xwrite(subnix.sigchld_pipe[1], &b, 1); - rz_sys_pipe_close(subnix.sigchld_pipe[1]); - rz_th_wait(subnix.sigchld_thread); - rz_sys_pipe_close(subnix.sigchld_pipe[0]); - rz_th_free(subnix.sigchld_thread); - rz_pvector_clear(&subnix.subprocs); - rz_th_lock_free(subnix.subprocs_mutex); + rz_xwrite(sigchld_pipe[1], &b, 1); + rz_sys_pipe_close(sigchld_pipe[1]); + rz_th_wait(sigchld_thread); + rz_sys_pipe_close(sigchld_pipe[0]); + rz_th_free(sigchld_thread); + rz_pvector_clear(&subprocs); + rz_th_lock_free(subprocs_mutex); } static char **create_child_env(const char *envvars[], const char *envvals[], size_t env_size) { @@ -1178,7 +1168,7 @@ no_term_change: rz_sys_pipe_close(stderr_pipe[1]); } - rz_pvector_push(&subnix.subprocs, proc); + rz_pvector_push(&subprocs, proc); subprocess_unlock(); return proc; @@ -1457,7 +1447,7 @@ RZ_API void rz_subprocess_free(RzSubprocess *proc) { return; } subprocess_lock(); - rz_pvector_remove_data(&subnix.subprocs, proc); + rz_pvector_remove_data(&subprocs, proc); subprocess_unlock(); rz_strbuf_fini(&proc->out); rz_strbuf_fini(&proc->err); diff --git a/librz/util/thread_iterators.c b/librz/util/thread_iterators.c index d3a04faafd..e67c928402 100644 --- a/librz/util/thread_iterators.c +++ b/librz/util/thread_iterators.c @@ -10,7 +10,7 @@ #include #include -static bool th_run_iterator(RzThreadFunction th_cb, void *context, RzThreadNCores max_threads) { +static bool th_run_iterator(RzThreadFunction th_cb, void *context, size_t max_threads) { RzThreadPool *pool = rz_th_pool_new(max_threads); if (!pool) { RZ_LOG_ERROR("th: failed to allocate thread pool\n"); @@ -70,7 +70,7 @@ static void *thread_iterate_list_cb(th_list_ctx_t *context) { * * \return On error returns false, otherwise true. */ -RZ_API bool rz_th_iterate_list(RZ_NONNULL const RzList /**/ *list, RZ_NONNULL RzThreadIterator iterator, RzThreadNCores max_threads, RZ_NULLABLE void *user) { +RZ_API bool rz_th_iterate_list(RZ_NONNULL const RzList /**/ *list, RZ_NONNULL RzThreadIterator iterator, size_t max_threads, RZ_NULLABLE void *user) { rz_return_val_if_fail(list && iterator, false); if (rz_list_length(list) < 1) { // nothing to do, but return true @@ -138,7 +138,7 @@ static void *thread_iterate_pvec_cb(th_vec_ctx_t *context) { * * \return On error returns false, otherwise true. */ -RZ_API bool rz_th_iterate_pvector(RZ_NONNULL const RzPVector /**/ *pvec, RZ_NONNULL RzThreadIterator iterator, RzThreadNCores max_threads, RZ_NULLABLE void *user) { +RZ_API bool rz_th_iterate_pvector(RZ_NONNULL const RzPVector /**/ *pvec, RZ_NONNULL RzThreadIterator iterator, size_t max_threads, RZ_NULLABLE void *user) { rz_return_val_if_fail(pvec && iterator, false); if (rz_pvector_len(pvec) < 1) { // nothing to do, but return true diff --git a/librz/util/thread_pool.c b/librz/util/thread_pool.c index 5a894bce8f..4938e9caac 100644 --- a/librz/util/thread_pool.c +++ b/librz/util/thread_pool.c @@ -3,7 +3,6 @@ #include #include "thread.h" -#include /** * \brief RzThreadPool is a structure which handles n-threads threads @@ -20,11 +19,11 @@ struct rz_th_pool_t { * * \return The number of available physical cores (always >= 1) */ -RZ_API RzThreadNCores rz_th_physical_core_number() { +RZ_API size_t rz_th_physical_core_number() { #ifdef __WINDOWS__ SYSTEM_INFO sysinfo; GetSystemInfo(&sysinfo); - return (RzThreadNCores)sysinfo.dwNumberOfProcessors; + return sysinfo.dwNumberOfProcessors; #elif __APPLE__ || __FreeBSD__ || __OpenBSD__ || __DragonFly__ || __NetBSD__ int os_status = 0; int mib[4]; @@ -58,34 +57,30 @@ RZ_API RzThreadNCores rz_th_physical_core_number() { // this is needed because the upper bits are set on bsd platforms n_cpus &= UT32_MAX; - return (RzThreadNCores)n_cpus; + return n_cpus; #elif __HAIKU__ system_info info; get_system_info(&info); - return (RzThreadNCores)info.cpu_count; + return info.cpu_count; #else - return (RzThreadNCores)sysconf(_SC_NPROCESSORS_ONLN); + return sysconf(_SC_NPROCESSORS_ONLN); #endif } /** - * \brief Returns the maximum number of threads available unless it exeeds N_THREAD_LIMIT. - * When set to 0, it will be the max number of cores. + * \brief Returns the maximum number of cores available regardless of the number of cores requested. + * When set to 0, it will be the max number of physical cores. * - * \param[in] max_threads The maximum number of threads to request + * \param[in] max_cores The maximum number of physical cores to request * - * \return The max number of threads requested + * \return The actual max number of cores available */ -RZ_API RzThreadNCores rz_th_max_threads(RzThreadNCores requested) { - const size_t n_thread_limit = N_THREAD_LIMIT; - RzThreadNCores n_cores = rz_th_physical_core_number(); - if (requested <= RZ_THREAD_N_CORES_ALL_AVAILABLE) { +RZ_API size_t rz_th_request_physical_cores(size_t max_cores) { + size_t n_cores = rz_th_physical_core_number(); + if (!max_cores) { return n_cores; - } else if (n_thread_limit < (size_t)requested) { - RZ_LOG_WARN("The number of requested threads is higher than the thread limit (%" PFMTSZu ").\n", n_thread_limit); - return n_thread_limit; } - return requested; + return RZ_MIN(n_cores, max_cores); } /** @@ -98,13 +93,13 @@ RZ_API RzThreadNCores rz_th_max_threads(RzThreadNCores requested) { * \param max_threads The maximum number of threads needed in the pool * \return RzThreadPool The RzThreadPool structure */ -RZ_API RZ_OWN RzThreadPool *rz_th_pool_new(RzThreadNCores max_threads) { +RZ_API RZ_OWN RzThreadPool *rz_th_pool_new(size_t max_threads) { RzThreadPool *pool = RZ_NEW0(RzThreadPool); if (!pool) { return NULL; } - pool->size = (size_t)rz_th_max_threads(max_threads); + pool->size = rz_th_request_physical_cores(max_threads); pool->threads = RZ_NEWS0(RzThread *, pool->size); if (!pool->threads) { free(pool); diff --git a/librz/util/thread_queue.c b/librz/util/thread_queue.c index 0abfb1af79..2248cf5f27 100644 --- a/librz/util/thread_queue.c +++ b/librz/util/thread_queue.c @@ -17,7 +17,7 @@ struct rz_th_queue_t { RzThreadLock *lock; RzThreadCond *cond; - RzThreadQueueSize max_size; + size_t max_size; RzList /**/ *list; }; @@ -29,7 +29,7 @@ struct rz_th_queue_t { * * \return On success returns a valid pointer, otherwise NULL */ -RZ_API RZ_OWN RzThreadQueue *rz_th_queue_new(RzThreadQueueSize max_size, RZ_NULLABLE RzListFree qfree) { +RZ_API RZ_OWN RzThreadQueue *rz_th_queue_new(size_t max_size, RZ_NULLABLE RzListFree qfree) { RzThreadQueue *queue = RZ_NEW0(RzThreadQueue); if (!queue) { return NULL; diff --git a/meson.build b/meson.build index 3e42ac85e2..a8153a2ee5 100644 --- a/meson.build +++ b/meson.build @@ -405,7 +405,6 @@ foreach it : ccs it_userconf.set10('HAVE_LZMA', get_option('use_lzma')) it_userconf.set10('HAVE_ZLIB', get_option('use_zlib')) it_userconf.set10('SUPPORTS_PCRE2_JIT', pcre2_jit_supported) - it_userconf.set('N_THREAD_LIMIT', get_option('n_thread_limit')) if it_machine.system() == 'freebsd' or it_machine.system() == 'dragonfly' add_project_link_arguments('-Wl,--unresolved-symbols,ignore-in-object-files', language: 'c', native: it_native) diff --git a/meson_options.txt b/meson_options.txt index 5cba4c4ad0..7abaf40e1a 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -7,7 +7,6 @@ option('blob', type: 'boolean', value: false, description: 'Compile just one bin option('subprojects_check', type: 'boolean', value: true, description: 'Check if git subprojects are up-to-date. Might be useful to disable this when developing on a different subproject version') option('portable', type: 'boolean', value: false, description: 'Make rizin installation moveable, by using relative paths instead of absolute ones') option('extra_prefix', type: 'string', value: '', description: 'Extra load path prefix (absolute path) for plugins, sdb, sigdb, etc.') -option('n_thread_limit', type: 'integer', min: 1, value: 32767) option('rizin_wwwroot', type: 'string', value: '', description: 'Install path for www files') option('rizin_sdb', type: 'string', value: '', description: 'Install path for all SDB files') diff --git a/test/unit/test_rz_test.c b/test/unit/test_rz_test.c index 6347441028..5690ff57ba 100644 --- a/test/unit/test_rz_test.c +++ b/test/unit/test_rz_test.c @@ -59,7 +59,7 @@ bool test_rz_test_fix(void) { RzTestDatabase *db = rz_test_test_database_new(); database_load(db, FILENAME, 1); - RzPVector *results = rz_pvector_new((RzPVectorFree)rz_test_result_info_free); + RzPVector *results = rz_pvector_new((RzPVectorFree)rz_test_test_result_info_free); RzTestResultInfo *result0 = RZ_NEW0(RzTestResultInfo); rz_pvector_push(results, result0); diff --git a/test/unit/test_threads.c b/test/unit/test_threads.c index 45ed5d689b..47ba3472c0 100644 --- a/test/unit/test_threads.c +++ b/test/unit/test_threads.c @@ -5,31 +5,15 @@ #include #include #include -#include #include "minunit.h" -bool test_thread_limit(void) { - const RzThreadNCores n_thread_limit = N_THREAD_LIMIT; - const RzThreadNCores n_cores = rz_th_physical_core_number(); - - // ensure the core count is returned. - RzThreadNCores requested = rz_th_max_threads(RZ_THREAD_N_CORES_ALL_AVAILABLE); - mu_assert_eq(requested, n_cores, "RZ_THREAD_N_CORES_ALL_AVAILABLE == rz_th_physical_core_number"); - - // ensure the thread limit is returned. - requested = rz_th_max_threads(n_thread_limit + 1); - mu_assert_eq(requested, n_thread_limit, "N_THREAD_LIMIT == rz_th_max_threads(LIMIT + 1)"); - - mu_end; -} - bool test_thread_pool_cores(void) { - RzThreadNCores cores = rz_th_physical_core_number(); + size_t cores = rz_th_physical_core_number(); - RzThreadPool *pool = rz_th_pool_new(RZ_THREAD_N_CORES_ALL_AVAILABLE); - mu_assert_notnull(pool, "rz_th_pool_new(RZ_THREAD_N_CORES_ALL_AVAILABLE) null check"); + RzThreadPool *pool = rz_th_pool_new(RZ_THREAD_POOL_ALL_CORES); + mu_assert_notnull(pool, "rz_th_pool_new(RZ_THREAD_POOL_ALL_CORES) null check"); size_t pool_size = rz_th_pool_size(pool); - mu_assert_eq(pool_size, cores, "rz_th_pool_new(RZ_THREAD_N_CORES_ALL_AVAILABLE) core count check"); + mu_assert_eq(pool_size, cores, "rz_th_pool_new(RZ_THREAD_POOL_ALL_CORES) core count check"); rz_th_pool_free(pool); if (cores > 1) { @@ -164,7 +148,7 @@ bool test_thread_iterator_list(void) { rz_list_append(list, &bool4); // test values are accessed - res = rz_th_iterate_list(list, (RzThreadIterator)thread_set_bool_arg, RZ_THREAD_N_CORES_ALL_AVAILABLE, &bool_user); + res = rz_th_iterate_list(list, (RzThreadIterator)thread_set_bool_arg, RZ_THREAD_POOL_ALL_CORES, &bool_user); mu_assert_true(res, "list is not empty and must return true"); mu_assert_true(bool_user, "bool_user must be true"); mu_assert_true(bool0, "bool0 must be true"); @@ -184,7 +168,7 @@ bool test_thread_iterator_list(void) { rz_list_append(list, NULL); rz_list_append(list, NULL); rz_list_append(list, NULL); - res = rz_th_iterate_list(list, (RzThreadIterator)thread_set_bool_arg, RZ_THREAD_N_CORES_ALL_AVAILABLE, &bool_user); + res = rz_th_iterate_list(list, (RzThreadIterator)thread_set_bool_arg, RZ_THREAD_POOL_ALL_CORES, &bool_user); mu_assert_true(res, "pvec is not empty and must return true"); mu_assert_false(bool_user, "bool_user must be false"); @@ -215,7 +199,7 @@ bool test_thread_iterator_pvec(void) { rz_pvector_push(pvec, &bool4); // test values are accessed - res = rz_th_iterate_pvector(pvec, (RzThreadIterator)thread_set_bool_arg, RZ_THREAD_N_CORES_ALL_AVAILABLE, &bool_user); + res = rz_th_iterate_pvector(pvec, (RzThreadIterator)thread_set_bool_arg, RZ_THREAD_POOL_ALL_CORES, &bool_user); mu_assert_true(res, "pvec is not empty and must return true"); mu_assert_true(bool_user, "bool_user must be true"); mu_assert_true(bool0, "bool0 must be true"); @@ -231,7 +215,7 @@ bool test_thread_iterator_pvec(void) { rz_pvector_set(pvec, 2, NULL); rz_pvector_set(pvec, 3, NULL); rz_pvector_set(pvec, 4, NULL); - res = rz_th_iterate_pvector(pvec, (RzThreadIterator)thread_set_bool_arg, RZ_THREAD_N_CORES_ALL_AVAILABLE, &bool_user); + res = rz_th_iterate_pvector(pvec, (RzThreadIterator)thread_set_bool_arg, RZ_THREAD_POOL_ALL_CORES, &bool_user); mu_assert_true(res, "pvec is not empty and must return true"); mu_assert_false(bool_user, "bool_user must be false"); @@ -240,7 +224,6 @@ bool test_thread_iterator_pvec(void) { } int all_tests() { - mu_run_test(test_thread_limit); mu_run_test(test_thread_pool_cores); mu_run_test(test_thread_queue); mu_run_test(test_thread_ht);