diff --git a/binrz/rz-test/run.c b/binrz/rz-test/run.c index 227ac1ea21..570a6c1fd5 100644 --- a/binrz/rz-test/run.c +++ b/binrz/rz-test/run.c @@ -119,6 +119,7 @@ static RzSubprocessOutput *run_rz_test(RzTestRunConfig *config, ut64 timeout_ms, rz_pvector_push(&args, "-escr.color=0"); rz_pvector_push(&args, "-escr.interactive=0"); rz_pvector_push(&args, "-eflirt.sigdb.load.system=false"); + rz_pvector_push(&args, "-esearch.show_progress=false"); rz_pvector_push(&args, "-eflirt.sigdb.load.home=false"); rz_pvector_push(&args, "-N"); RzListIter *it; diff --git a/librz/core/csearch.c b/librz/core/csearch.c new file mode 100644 index 0000000000..67d25d3228 --- /dev/null +++ b/librz/core/csearch.c @@ -0,0 +1,223 @@ +// SPDX-FileCopyrightText: 2024 deroad +// SPDX-FileCopyrightText: 2024 Rot127 +// SPDX-License-Identifier: LGPL-3.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * \brief Sets up the search find options according to the core config. + * + * \param core The core to get the config from. + * + * \return The find options to use. Or NULL in case of failure. + */ +RZ_API RZ_OWN RzSearchFindOpt *rz_core_setup_default_search_find_opts(RzCore *core) { + rz_return_val_if_fail(core, NULL); + RzSearchFindOpt *fopts = rz_search_find_opt_new(); + if (!fopts) { + RZ_LOG_ERROR("Failed allocating find options.\n"); + return NULL; + } + if (!(rz_search_find_opt_set_inverse_match(fopts, rz_config_get_b(core->config, "search.inverse")) && + rz_search_find_opt_set_overlap_match(fopts, rz_config_get_b(core->config, "search.overlap")) && + rz_search_find_opt_set_alignment(fopts, rz_config_get_i(core->config, "search.align")))) { + RZ_LOG_ERROR("Failed set find options.\n"); + rz_search_find_opt_free(fopts); + return NULL; + } + return fopts; +} + +/** + * \brief Sets up the search parameters according to the core IO layer and config. + * + * \param core The core to get the IO maps, settings and other relevant information from. + * \param search_opts Search options to set up. Only fields to search behavior will be set (max_threads, max hits). Can be NULL. + * + * \return The boundaries to search in. Or NULL in case of failure. + */ +RZ_API RZ_OWN RzList /**/ *rz_core_setup_io_search_parameters(RzCore *core, RZ_NULLABLE RZ_OUT RzSearchOpt *search_opts) { + rz_return_val_if_fail(core && core->io && core->config, NULL); + RzList *boundaries = NULL; + + if (!core->io) { + RZ_LOG_ERROR("core: RzIO is not available.\n"); + return NULL; + } + + boundaries = rz_core_get_boundaries_select(core, "search.from", "search.to", "search.in"); + if (!boundaries || rz_list_empty(boundaries)) { + ut64 from = rz_config_get_i(core->config, "search.from"); + ut64 to = rz_config_get_i(core->config, "search.to"); + RZ_LOG_ERROR("core: Failed to get search boundaries within [0x%" PFMT64x ", 0x%" PFMT64x "].\n", from, to); + goto fail; + } + + if (search_opts) { + // Set search options known by core. + ut32 max_threads = rz_th_max_threads(rz_config_get_i(core->config, "search.max_threads")); + ut32 max_hits = rz_config_get_i(core->config, "search.maxhits"); + if (!(rz_search_opt_set_max_threads(search_opts, max_threads) && + rz_search_opt_set_max_hits(search_opts, max_hits))) { + RZ_LOG_ERROR("core: Failed to setup search options.\n"); + goto fail; + } + + RzSearchFindOpt *fopts = rz_core_setup_default_search_find_opts(core); + if (!fopts) { + RZ_LOG_ERROR("Failed setup find options.\n"); + goto fail; + } + rz_search_opt_set_find_options(search_opts, fopts); + } + + return boundaries; +fail: + rz_list_free(boundaries); + return NULL; +} + +static bool default_search_no_cancel(void *user, size_t n_hits, RzSearchCancelReason invoke_reason) { + return rz_cons_is_breaked(); +} + +static RzList /**/ *perform_search_on_core_io(RzCore *core, RZ_BORROW RzSearchOpt *search_opts, RZ_BORROW RzList /**/ *boundaries, RZ_BORROW RzSearchCollection *collection) { + RzList *hits = NULL; + + hits = rz_search_on_io(search_opts, collection, core->io, boundaries); + if (!hits) { + ut64 from = rz_config_get_i(core->config, "search.from"); + ut64 to = rz_config_get_i(core->config, "search.to"); + RZ_LOG_ERROR("core: Failed to search within [0x%" PFMT64x ", 0x%" PFMT64x "].\n", from, to); + } + return hits; +} + +/** + * \brief Finds a byte array in the IO layer of the given core and core configuration. + * + * \param core The RzCore core. + * \param opt The search options to apply. If it is NULL a default set of options is used. + * \param pattern The bytes pattern to search. + * + * \return On success returns a valid pointer, otherwise NULL + */ +RZ_API RZ_OWN RzList /**/ *rz_core_search_bytes(RZ_NONNULL RzCore *core, RZ_BORROW RZ_NULLABLE RzSearchOpt *user_opts, RZ_NONNULL RZ_OWN RzSearchBytesPattern *pattern) { + rz_return_val_if_fail(core && core->config && pattern, NULL); + if (rz_search_bytes_pattern_len(pattern) == 0) { + RZ_LOG_ERROR("core: Cannot search for byte pattern if 'length' == 0.\n"); + rz_search_bytes_pattern_free(pattern); + return NULL; + } + + RzList *hits = NULL; + RzList *boundaries = NULL; + RzSearchOpt *search_opts = NULL; + + RzSearchCollection *collection = rz_search_collection_bytes(); + if (!collection || + !rz_search_collection_bytes_add_pattern(collection, pattern)) { + RZ_LOG_ERROR("core: Failed to initialize search collection.\n"); + rz_search_bytes_pattern_free(pattern); + goto quit; + } + + if (!user_opts) { + search_opts = rz_search_opt_new(); + if (!rz_search_opt_set_cancel_cb(search_opts, default_search_no_cancel, NULL)) { + RZ_LOG_ERROR("search: Failed to setup callback for search options.\n"); + goto quit; + } + } + + // Don't pass the user provided search options. + // They were set up by the user and we respect them. + boundaries = rz_core_setup_io_search_parameters(core, user_opts ? NULL : search_opts); + if (!boundaries) { + RZ_LOG_ERROR("core: Setting up search from core failed.\n"); + goto quit; + } + if (!rz_search_opt_set_elemet_size(user_opts ? user_opts : search_opts, rz_search_bytes_pattern_len(pattern))) { + RZ_LOG_ERROR("search: Failed to update chunk size in the search options.\n"); + goto quit; + } + + hits = perform_search_on_core_io(core, user_opts ? user_opts : search_opts, boundaries, collection); + +quit: + rz_list_free(boundaries); + rz_search_opt_free(search_opts); + rz_search_collection_free(collection); + return hits; +} + +/** + * \brief Finds a string within the `search.in` boundaries. + * + * \param core The RzCore core. + * \param opt The search options to apply. If NULL, a default set of options is used. + * \param[in] re_pattern The regex pattern to search. + * \param[in] flags The regex flags to the \p re_pattern. + * \param[in] expected The expected encoding. + * + * \return On success returns a valid pointer to a list of search hits, otherwise NULL. + */ +RZ_API RZ_OWN RzList /**/ *rz_core_search_string(RZ_NONNULL RzCore *core, RZ_BORROW RZ_NONNULL RzSearchOpt *user_opts, RZ_NONNULL const char *re_pattern, RzRegexFlags flags, RzStrEnc expected) { + rz_return_val_if_fail(core && user_opts && re_pattern, NULL); + + if (RZ_STR_ISEMPTY(re_pattern)) { + RZ_LOG_ERROR("core: invalid string: empty string.\n"); + return NULL; + } + if (strlen(re_pattern) >= core->bin->str_search_cfg.max_length) { + RZ_LOG_ERROR("core: String to search is larger then search.str.max_length.\n"); + return NULL; + } + + // Copy RzUtilStrScanOptions from RzBin + RzUtilStrScanOptions scan_opt = { + // buf_size is effectively the maximum string length. + // Gets renamed with the refactor. + .max_str_length = core->bin->str_search_cfg.max_length, + .min_str_length = core->bin->str_search_cfg.min_length, + .prefer_big_endian = core->analysis->big_endian, + .check_ascii_freq = core->bin->str_search_cfg.check_ascii_freq, + }; + + RzList *hits = NULL; + RzList *boundaries = NULL; + RzSearchOpt *search_opts = NULL; + + RzSearchCollection *collection = rz_search_collection_strings(&scan_opt, expected, flags); + if (!collection || + !rz_search_collection_string_add(collection, re_pattern, flags)) { + rz_search_collection_free(collection); + RZ_LOG_ERROR("core: Failed to initialize search collection.\n"); + return NULL; + } + boundaries = rz_core_setup_io_search_parameters(core, user_opts ? NULL : search_opts); + if (!boundaries) { + RZ_LOG_ERROR("core: Setting up search from core failed.\n"); + goto quit; + } + if (!rz_search_opt_set_elemet_size(user_opts ? user_opts : search_opts, scan_opt.max_str_length)) { + RZ_LOG_ERROR("search: Failed to update chunk size in the search options.\n"); + goto quit; + } + + hits = perform_search_on_core_io(core, user_opts ? user_opts : search_opts, boundaries, collection); + +quit: + rz_list_free(boundaries); + rz_search_opt_free(search_opts); + rz_search_collection_free(collection); + return hits; +} diff --git a/librz/core/meson.build b/librz/core/meson.build index 296b0df3b5..abf5394ce4 100644 --- a/librz/core/meson.build +++ b/librz/core/meson.build @@ -44,6 +44,7 @@ rz_core_sources = [ 'cplugin.c', 'cprint.c', 'creg.c', + 'csearch.c', 'csign.c', 'ctypes.c', 'cvfile.c', diff --git a/librz/include/rz_core.h b/librz/include/rz_core.h index 4d7034b257..7c4fa1e2b0 100644 --- a/librz/include/rz_core.h +++ b/librz/include/rz_core.h @@ -282,7 +282,8 @@ struct rz_core_t { RzLang *lang; RzDebug *dbg; RzFlag *flags; - RzSearch *search; + char *lastsearch; ///< Legacy search. Will be removed + RzSearch *search; ///< Legacy search. Will be removed RzEgg *egg; RzCrypto *crypto; RzAGraph *graph; @@ -318,7 +319,6 @@ struct rz_core_t { int curtab; // current tab int seltab; // selected tab char *cmdremote; - char *lastsearch; char *cmdfilter; char *curtheme; bool break_loop; @@ -1083,6 +1083,7 @@ RZ_API void rz_core_rtr_cmd(RzCore *core, const char *input); RZ_API int rz_core_rtr_http(RzCore *core, int launch, int browse, const char *path); RZ_API int rz_core_rtr_gdb(RzCore *core, int launch, const char *path); +/// Legacy search RZ_API int rz_core_search_preludes(RzCore *core, bool log); RZ_API int rz_core_search_prelude(RzCore *core, ut64 from, ut64 to, const ut8 *buf, int blen, const ut8 *mask, int mlen); @@ -1348,6 +1349,12 @@ RZ_API void rz_core_analysis_bytes_il(RZ_NONNULL RzCore *core, ut64 len, ut64 nu RZ_API bool rz_core_disasm_until_ret(RZ_NONNULL RzCore *core, ut64 addr, int limit, RzOutputMode mode, bool ret_val, RZ_NULLABLE RZ_OUT RzStrBuf *buf); +RZ_API RZ_OWN RzList /**/ *rz_core_setup_io_search_parameters(RzCore *core, RZ_NULLABLE RZ_OUT RzSearchOpt *search_opts); +RZ_API RZ_OWN RzSearchFindOpt *rz_core_setup_default_search_find_opts(RzCore *core); + +RZ_API RZ_OWN RzList /**/ *rz_core_search_bytes(RZ_NONNULL RzCore *core, RZ_BORROW RZ_NULLABLE RzSearchOpt *user_opts, RZ_NONNULL RZ_OWN RzSearchBytesPattern *pattern); +RZ_API RZ_OWN RzList /**/ *rz_core_search_string(RZ_NONNULL RzCore *core, RZ_BORROW RZ_NONNULL RzSearchOpt *user_opts, RZ_NONNULL const char *string, RzRegexFlags flags, RzStrEnc expected); + #endif #ifdef __cplusplus diff --git a/librz/include/rz_search.h b/librz/include/rz_search.h index e273bd6930..5e80406051 100644 --- a/librz/include/rz_search.h +++ b/librz/include/rz_search.h @@ -5,6 +5,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -44,10 +45,10 @@ typedef struct rz_search_keyword_t { ut64 last; // last hit hint } RzSearchKeyword; -typedef struct rz_search_hit_t { +typedef struct { RzSearchKeyword *kw; ut64 addr; -} RzSearchHit; +} RzSearchLegacyHit; typedef int (*RzSearchCallback)(RzSearchKeyword *kw, void *user, ut64 where); @@ -109,7 +110,7 @@ RZ_API int rz_search_deltakey_update(RzSearch *s, ut64 from, const ut8 *buf, int RZ_API int rz_search_strings_update(RzSearch *s, ut64 from, const ut8 *buf, int len); RZ_API int rz_search_regexp_update(RzSearch *s, ut64 from, const ut8 *buf, int len); // Returns 2 if search.maxhits is reached, 0 on error, otherwise 1 -RZ_API int rz_search_hit_new(RzSearch *s, RzSearchKeyword *kw, ut64 addr); +RZ_API int rz_search_legacy_hit_new(RzSearch *s, RzSearchKeyword *kw, ut64 addr); RZ_API void rz_search_set_distance(RzSearch *s, int dist); RZ_API int rz_search_set_string_limits(RzSearch *s, ut32 min, ut32 max); // dup again? // RZ_API int rz_search_set_callback(RzSearch *s, int (*callback)(struct rz_search_kw_t *, void *, ut64), void *user); @@ -120,9 +121,98 @@ RZ_API int rz_search_begin(RzSearch *s); RZ_API void rz_search_pattern_size(RzSearch *s, int size); RZ_API int rz_search_pattern(RzSearch *s, ut64 from, ut64 to); +#endif // RZ_API + +// +// New search. +// Everything above is only there to not break the build. +// + +RZ_LIB_VERSION_HEADER(rz_search); + +/** + * \brief Private search options for the search module. Use the rz_search_opt_*() functions to edit it. + */ +typedef struct rz_search_opt_t RzSearchOpt; + +/** + * \brief Options for the find() callback of the different searches. + */ +typedef struct rz_search_find_opt_t RzSearchFindOpt; + +typedef struct rz_search_collection_t RzSearchCollection; + +typedef struct rz_search_hit_t { + char *hit_desc; ///< Hit description (can be NULL) + ut64 address; ///< Address/offset of the matched data. + size_t size; ///< Size of the matched data (can be 0), in bytes. +} RzSearchHit; + +typedef enum { + RZ_SEARCH_CANCEL_REGULAR_CHECK, ///< Regular cancel check. Repeated every RZ_SEARCH_CANCEL_CHECK_INTERVAL_USEC microseconds. + RZ_SEARCH_CANCEL_SIGINT, ///< Interrupt signal (likely ctrl + c). +} RzSearchCancelReason; + +typedef struct rz_search_bytes_pattern_t RzSearchBytesPattern; + +RZ_API RZ_OWN char *rz_search_hit_flag_name(RZ_NONNULL const RzSearchHit *hit, size_t hit_id, RZ_NULLABLE const char *prefix); + +RZ_API void rz_search_bytes_pattern_free(RZ_NULLABLE RZ_OWN RzSearchBytesPattern *hp); +RZ_API RZ_OWN RzSearchBytesPattern *rz_search_bytes_pattern_copy(RZ_NONNULL RZ_BORROW RzSearchBytesPattern *hp); +RZ_API RZ_OWN RzSearchBytesPattern *rz_search_bytes_pattern_new(RZ_OWN ut8 *bytes, RZ_NULLABLE RZ_OWN ut8 *mask, size_t length, RZ_NULLABLE const char *pattern_desc, bool compile_regex); +RZ_API RZ_OWN RzSearchBytesPattern *rz_search_parse_byte_pattern(const char *byte_pattern, RZ_NULLABLE const char *pattern_desc); +RZ_API size_t rz_search_bytes_pattern_len(RZ_NONNULL const RzSearchBytesPattern *hp); +RZ_API const char *rz_search_bytes_pattern_desc(RZ_NONNULL const RzSearchBytesPattern *bp); + +/** + * \brief The cancel callback. It is invoked to check, if the search should be stopped. + * + * \param user The private user data. + * \param n_hits Number of hits already found during the search. + * \param invoe_reason The reason it is called. + * + * \return True, if the search should be canceled. + * \return False, if the search should continue. + */ +typedef bool (*RzSearchCancelCallback)(void *user, size_t n_hits, RzSearchCancelReason invoke_reason); + +RZ_API RZ_OWN RzSearchOpt *rz_search_opt_new(); +RZ_API void rz_search_opt_free(RZ_NULLABLE RzSearchOpt *opt); +RZ_API bool rz_search_opt_set_max_hits(RZ_NONNULL RzSearchOpt *opt, size_t max_hits); +RZ_API bool rz_search_opt_set_elemet_size(RZ_NONNULL RzSearchOpt *opt, ut64 chunk_size); +RZ_API bool rz_search_opt_set_max_threads(RZ_NONNULL RzSearchOpt *opt, RzThreadNCores max_threads); +RZ_API bool rz_search_opt_set_cancel_cb(RZ_NONNULL RzSearchOpt *opt, RzSearchCancelCallback callback, void *user); +RZ_API bool rz_search_opt_set_find_options(RZ_NONNULL RzSearchOpt *opt, RZ_OWN RzSearchFindOpt *find_opts); + +RZ_API RZ_OWN RzSearchFindOpt *rz_search_find_opt_new(); +RZ_API void rz_search_find_opt_free(RZ_NULLABLE RzSearchFindOpt *opt); +RZ_API bool rz_search_find_opt_set_inverse_match(RZ_NONNULL RzSearchFindOpt *opt, bool inverse_match); +RZ_API bool rz_search_find_opt_set_overlap_match(RZ_NONNULL RzSearchFindOpt *opt, bool overlap_match); +RZ_API bool rz_search_find_opt_set_alignment(RZ_NONNULL RzSearchFindOpt *opt, size_t alignment); + +RZ_API RZ_OWN RzSearchCollection *rz_search_collection_aes_keys(); + +RZ_API RZ_OWN RzSearchCollection *rz_search_collection_private_keys(); + +RZ_API RZ_OWN RzSearchCollection *rz_search_collection_regex(); +RZ_API bool rz_search_collection_regex_add(RZ_NONNULL RzSearchCollection *col, RZ_NONNULL const char *regex, bool caseless); + +RZ_API RZ_OWN RzSearchCollection *rz_search_collection_bytes(); +RZ_API bool rz_search_collection_bytes_add(RZ_NONNULL RzSearchCollection *col, RZ_NULLABLE const char *pattern_desc, RZ_NONNULL const ut8 *bytes, RZ_NULLABLE const ut8 *mask, size_t length); +RZ_API bool rz_search_collection_bytes_add_pattern(RZ_NONNULL RzSearchCollection *col, RZ_NONNULL RZ_OWN RzSearchBytesPattern *bytes_pattern); + +RZ_API RZ_OWN RzSearchCollection *rz_search_collection_strings(RZ_NONNULL RzUtilStrScanOptions *opts, RzStrEnc expected, RzRegexFlags re_flags); +RZ_API bool rz_search_collection_string_add(RZ_NONNULL RzSearchCollection *col, RZ_NONNULL const char *regex_pattern, RzRegexFlags re_flags); + +RZ_API RZ_OWN RzSearchCollection *rz_search_collection_magic(RZ_NONNULL const char *magic_dir); + +RZ_API bool rz_search_collection_match_any(RZ_NULLABLE RzSearchCollection *sc, RZ_NONNULL const ut8 *buffer, size_t length); +RZ_API void rz_search_collection_free(RZ_NULLABLE RzSearchCollection *sc); + +RZ_API RZ_OWN RzList /**/ *rz_search_on_io(RZ_BORROW RZ_NONNULL RzSearchOpt *opt, RZ_BORROW RZ_NONNULL RzSearchCollection *col, RZ_BORROW RZ_NONNULL RzIO *io, RZ_BORROW RZ_NONNULL RzList /**/ *search_in); + #ifdef __cplusplus } #endif #endif -#endif diff --git a/librz/include/rz_util/rz_str_search.h b/librz/include/rz_util/rz_str_search.h index 75c0ccc715..5162b2e550 100644 --- a/librz/include/rz_util/rz_str_search.h +++ b/librz/include/rz_util/rz_str_search.h @@ -14,22 +14,46 @@ extern "C" { * Represent a detected string. */ typedef struct { - char *string; ///< Pointer to the string - ut64 addr; ///< Address of the string in the RzBuffer + char *string; ///< The detected string. Note that this one is always in UTF-8. No matter what the ecoding is in memory. + RzRegex *regex; ///< Regex matching the string. If set, the string member is the pattern. + ut64 addr; ///< Address/offset of the string in the RzBuffer ut32 size; ///< Size of buffer containing the string in bytes ut32 length; ///< Length of string in chars - RzStrEnc type; ///< String type + RzStrEnc type; ///< String encoding in memory. } RzDetectedString; /** * Defines the search parameters for rz_scan_strings */ typedef struct { - size_t buf_size; ///< Maximum size of a detected string - size_t max_uni_blocks; ///< Maximum number of unicode blocks + size_t max_str_length; ///< Maximum size of a detected string. size_t min_str_length; ///< Minimum string length bool prefer_big_endian; ///< True if the preferred endianess for UTF strings is big-endian bool check_ascii_freq; ///< If true, perform check on ASCII frequencies when looking for false positives + /** + * \brief Map UTF-8 byte offsets to memory offsets. + * The string scan function always returns UTF-8 strings. + * Independent what encoding the strings have in memory. + * Sometimes it is necessary to know the offsets of the real encoding. + * This maps an UTF-8 code point offset to the original code point offset in memory. + * The keys are ut64 values. With the upper 32bits holding the index into the + * "detected string list" returned by rz_scan_strings_whole_buf(). + * The lower 32bits are the offset into the UTF-8 string. + * The value is the offset into the memory. Relevant to the buffer + * The string was found in. + * + * Example: + * + * Buffer (UTF-16): 0x00, 0x41, 0x00, 0x41, 0x00, 0x00, 0x00, 0x42, 0x00, 0x42 + * Found strings (UTF-8): [ "AA", "BB" ] + * Map: { + * 0x0000000000000000: 0, + * 0x0000000000000001: 2, + * 0x0000000100000000: 6, + * 0x0000000100000001: 8 + * } + */ + RZ_NULLABLE HtUU *utf8_to_mem_offset_map; } RzUtilStrScanOptions; RZ_API void rz_detected_string_free(RzDetectedString *str); diff --git a/librz/meson.build b/librz/meson.build index bc57eb27e5..bf29e95458 100644 --- a/librz/meson.build +++ b/librz/meson.build @@ -14,8 +14,8 @@ subdir('diff') subdir('io') subdir('bp') subdir('syscall') -subdir('search') subdir('magic') +subdir('search') subdir('flag') subdir('reg') subdir('type') diff --git a/librz/search/aes-find.c b/librz/search/aes-find.c index 5934d28fda..e04602c66c 100644 --- a/librz/search/aes-find.c +++ b/librz/search/aes-find.c @@ -51,7 +51,7 @@ RZ_API int rz_search_aes_update(RzSearch *s, ut64 from, const ut8 *buf, int len) for (i = 0; i < last; i++) { if (aes128_key_test(buf + i)) { kw->keyword_length = AES128_KEY_LENGTH; - t = rz_search_hit_new(s, kw, from + i); + t = rz_search_legacy_hit_new(s, kw, from + i); if (!t) { return -1; } @@ -62,7 +62,7 @@ RZ_API int rz_search_aes_update(RzSearch *s, ut64 from, const ut8 *buf, int len) } if (len - i - AES192_SEARCH_LENGTH >= 0 && aes192_key_test(buf + i)) { kw->keyword_length = AES192_KEY_LENGTH; - t = rz_search_hit_new(s, kw, from + i); + t = rz_search_legacy_hit_new(s, kw, from + i); if (!t) { return -1; } @@ -73,7 +73,7 @@ RZ_API int rz_search_aes_update(RzSearch *s, ut64 from, const ut8 *buf, int len) } if (len - i - AES256_SEARCH_LENGTH >= 0 && aes256_key_test(buf + i)) { kw->keyword_length = AES256_KEY_LENGTH; - t = rz_search_hit_new(s, kw, from + i); + t = rz_search_legacy_hit_new(s, kw, from + i); if (!t) { return -1; } @@ -86,4 +86,4 @@ RZ_API int rz_search_aes_update(RzSearch *s, ut64 from, const ut8 *buf, int len) } } return -1; -} \ No newline at end of file +} diff --git a/librz/search/collection.c b/librz/search/collection.c new file mode 100644 index 0000000000..0c6831c661 --- /dev/null +++ b/librz/search/collection.c @@ -0,0 +1,91 @@ +// SPDX-FileCopyrightText: 2024 RizinOrg +// SPDX-FileCopyrightText: 2024 deroad +// SPDX-License-Identifier: LGPL-3.0-only + +#include +#include "search_internal.h" + +static RZ_OWN RzSearchCollection *rz_search_collection_new(RzSearchSpace space, RZ_NONNULL void *find, RZ_NONNULL RzSearchIsEmptyCallback is_empty, RZ_NULLABLE RzSearchFreeCallback free, RZ_NULLABLE void *user) { + rz_return_val_if_fail(find && is_empty, NULL); + RzSearchCollection *sc = RZ_NEW0(RzSearchCollection); + if (!sc) { + RZ_LOG_ERROR("search: failed to allocate RzSearchCollection\n"); + return NULL; + } + sc->space = space; + sc->find = find; + sc->is_empty = is_empty; + sc->free = free; + sc->user = user; + return sc; +} + +/** + * \brief Initialize a new RzSearchCollection over a graph. + * + * \param[in] find The find callback to set + * \param[in] is_empty The callback to use to check if collection is empty + * \param[in] free The callback to use to free the context + * \param user The additional context needed. + * + * \return On success returns a valid pointer, otherwise NULL. + */ +RZ_IPI RZ_OWN RzSearchCollection *rz_search_collection_new_graph_space(RZ_NONNULL RzSearchFindGraphCallback find, RZ_NONNULL RzSearchIsEmptyCallback is_empty, RZ_NULLABLE RzSearchFreeCallback free, RZ_NULLABLE void *user) { + rz_return_val_if_fail(find && is_empty, NULL); + return rz_search_collection_new(RZ_SEARCH_SPACE_GRAPH, find, is_empty, free, user); +} + +/** + * \brief Initialize a new RzSearchCollection over bytes. + * + * \param[in] find The find callback to set + * \param[in] is_empty The callback to use to check if collection is empty + * \param[in] free The callback to use to free the context + * \param user The additional context needed. + * + * \return On success returns a valid pointer, otherwise NULL. + */ +RZ_IPI RZ_OWN RzSearchCollection *rz_search_collection_new_bytes_space(RZ_NONNULL RzSearchFindBytesCallback find, RZ_NONNULL RzSearchIsEmptyCallback is_empty, RZ_NULLABLE RzSearchFreeCallback free, RZ_NULLABLE void *user) { + rz_return_val_if_fail(find && is_empty, NULL); + return rz_search_collection_new(RZ_SEARCH_SPACE_BYTES, find, is_empty, free, user); +} + +/** + * \brief Frees a RzSearchCollection structure + * + * \param[in] sc The RzSearchCollection pointer to free + */ +RZ_API void rz_search_collection_free(RZ_NULLABLE RzSearchCollection *sc) { + if (!sc) { + return; + } + if (sc->free) { + sc->free(sc->user); + } + free(sc); +} + +/** + * \brief Checks if a given RzSearchCollection has an expected find callback + * + * \param col The RzSearchCollection to test + * \param[in] expected The expected find callback + * + * \return Returns true when the RzSearchCollection callback matches the expected one. + */ +RZ_IPI bool rz_search_collection_has_find_callback(RZ_NONNULL RzSearchCollection *col, RZ_NONNULL void *expected) { + rz_return_val_if_fail(col && expected, false); + return col->find == expected; +} + +/** + * \brief Checks if a given RzSearchCollection is empty + * + * \param col The RzSearchCollection to test + * + * \return Returns true when the RzSearchCollection is empty. + */ +RZ_IPI bool rz_search_collection_is_empty(RZ_NONNULL RzSearchCollection *col) { + rz_return_val_if_fail(col && col->is_empty, false); + return col->is_empty(col->user); +} diff --git a/librz/search/meson.build b/librz/search/meson.build index ab0bf39e86..b10f24e3e0 100644 --- a/librz/search/meson.build +++ b/librz/search/meson.build @@ -5,11 +5,15 @@ rz_search_sources = [ 'regexp.c', 'privkey-find.c', 'search.c', + 'collection.c', + 'options.c', + 'bytes_search.c', + 'string_search.c', ] rz_search = library('rz_search', rz_search_sources, include_directories: [platform_inc], - dependencies: [rz_util_dep], + dependencies: [rz_util_dep, rz_io_dep, rz_magic_dep], install: true, implicit_include_directories: false, install_rpath: rpath_lib, @@ -25,5 +29,5 @@ meson.override_dependency('rz_search', rz_search_dep) modules += { 'rz_search': { 'target': rz_search, - 'dependencies': ['rz_util'] + 'dependencies': ['rz_util', 'rz_io', 'rz_magic'] }} diff --git a/librz/search/options.c b/librz/search/options.c new file mode 100644 index 0000000000..11157588b9 --- /dev/null +++ b/librz/search/options.c @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: 2024 RizinOrg +// SPDX-FileCopyrightText: 2024 deroad +// SPDX-License-Identifier: LGPL-3.0-only + +#include +#include "search_internal.h" + +RZ_API RZ_OWN RzSearchOpt *rz_search_opt_new() { + RzSearchOpt *opt = RZ_NEW0(RzSearchOpt); + if (!opt) { + return NULL; + } + opt->max_threads = RZ_THREAD_N_CORES_ALL_AVAILABLE; + opt->chunk_size = RZ_SEARCH_DEFAULT_CHUNK_SIZE; + return opt; +} + +RZ_API void rz_search_opt_free(RZ_NULLABLE RzSearchOpt *opt) { + if (!opt) { + return; + } + rz_search_find_opt_free(opt->find_opts); + free(opt); +} + +RZ_API bool rz_search_opt_set_max_hits(RZ_NONNULL RzSearchOpt *opt, size_t max_hits) { + rz_return_val_if_fail(opt, false); + opt->max_hits = max_hits; + return true; +} + +static bool set_chunk_size(RZ_NONNULL RzSearchOpt *opt, ut64 chunk_size) { + rz_return_val_if_fail(opt, false); + if (chunk_size < RZ_SEARCH_MIN_CHUNK_SIZE || chunk_size > RZ_SEARCH_MAX_CHUNK_SIZE) { + RZ_LOG_ERROR("search: Chunk size is not in range of %#" PFMT64x "-%#" PFMT64x " bytes.\n", + RZ_SEARCH_MIN_CHUNK_SIZE, + RZ_SEARCH_MAX_CHUNK_SIZE); + return false; + } + opt->chunk_size = chunk_size; + return true; +} + +static bool element_chunk_ratio_ok(ut64 element_size, ut64 chunk_size) { + if (element_size >= chunk_size) { + return false; + } + return (chunk_size / element_size) >= RZ_SEARCH_MIN_ELEMENTS_PER_CHUNK; +} + +RZ_API bool rz_search_opt_set_elemet_size(RZ_NONNULL RzSearchOpt *opt, ut64 element_size) { + rz_return_val_if_fail(opt, false); + if (!element_chunk_ratio_ok(element_size, opt->chunk_size)) { + if (!set_chunk_size(opt, element_size * RZ_SEARCH_MIN_ELEMENTS_PER_CHUNK)) { + RZ_LOG_ERROR("search: Element to search is too big.\n"); + return false; + } + } + opt->element_size = element_size; + return true; +} + +RZ_API bool rz_search_opt_set_max_threads(RZ_NONNULL RzSearchOpt *opt, RzThreadNCores max_threads) { + rz_return_val_if_fail(opt, false); + opt->max_threads = max_threads; + return true; +} + +RZ_API bool rz_search_opt_set_cancel_cb(RZ_NONNULL RzSearchOpt *opt, RzSearchCancelCallback callback, void *user) { + rz_return_val_if_fail(opt, false); + opt->cancel_cb = callback; + opt->cancel_usr = user; + return true; +} + +RZ_API bool rz_search_opt_set_find_options(RZ_NONNULL RzSearchOpt *opt, RZ_OWN RzSearchFindOpt *find_opts) { + rz_return_val_if_fail(opt, false); + opt->find_opts = find_opts; + return true; +} + +RZ_API RZ_OWN RzSearchFindOpt *rz_search_find_opt_new() { + return RZ_NEW0(RzSearchFindOpt); +} + +RZ_API void rz_search_find_opt_free(RZ_NULLABLE RzSearchFindOpt *opt) { + free(opt); +} + +RZ_API bool rz_search_find_opt_set_inverse_match(RZ_NONNULL RzSearchFindOpt *opt, bool inverse_match) { + rz_return_val_if_fail(opt, false); + opt->match_inverse = inverse_match; + return true; +} + +RZ_API bool rz_search_find_opt_get_inverse_match(RZ_NONNULL RzSearchFindOpt *opt) { + rz_return_val_if_fail(opt, false); + return opt->match_inverse; +} + +RZ_API bool rz_search_find_opt_set_overlap_match(RZ_NONNULL RzSearchFindOpt *opt, bool overlap_match) { + rz_return_val_if_fail(opt, false); + opt->match_overlap = overlap_match; + return true; +} + +RZ_API bool rz_search_find_opt_get_overlap_match(RZ_NONNULL RzSearchFindOpt *opt) { + rz_return_val_if_fail(opt, false); + return opt->match_overlap; +} + +RZ_API bool rz_search_find_opt_set_alignment(RZ_NONNULL RzSearchFindOpt *opt, size_t alignment) { + rz_return_val_if_fail(opt, false); + opt->alignment = alignment; + return true; +} + +RZ_API ut16 rz_search_find_opt_get_alignment(RZ_NONNULL RzSearchFindOpt *opt) { + rz_return_val_if_fail(opt, 0); + return opt->alignment; +} diff --git a/librz/search/privkey-find.c b/librz/search/privkey-find.c index 7f9deb000c..4c8c99167d 100644 --- a/librz/search/privkey-find.c +++ b/librz/search/privkey-find.c @@ -110,7 +110,7 @@ RZ_API int rz_search_privkey_update(RzSearch *s, ut64 from, const ut8 *buf, int if (check_fields(buf + index)) { parse_next_field(buf + index, &kw->keyword_length); - t = rz_search_hit_new(s, kw, from + index); + t = rz_search_legacy_hit_new(s, kw, from + index); if (t > 1) { return s->nhits - old_nhits; } diff --git a/librz/search/regexp.c b/librz/search/regexp.c index e7a0c6e46e..22bfafc726 100644 --- a/librz/search/regexp.c +++ b/librz/search/regexp.c @@ -37,7 +37,7 @@ RZ_API int rz_search_regexp_update(RzSearch *s, ut64 from, const ut8 *buf, int l rz_pvector_foreach (matches, it) { RzRegexMatch *m = *it; kw->keyword_length = m->len; // For a regex search, the keyword can be of variable length - int t = rz_search_hit_new(s, kw, from + m->start); + int t = rz_search_legacy_hit_new(s, kw, from + m->start); if (t == 0) { ret = -1; rz_pvector_free(matches); diff --git a/librz/search/search.c b/librz/search/search.c index 68cc905fa9..da1ba206d7 100644 --- a/librz/search/search.c +++ b/librz/search/search.c @@ -1,9 +1,15 @@ -// SPDX-FileCopyrightText: 2008-2016 pancake +// SPDX-FileCopyrightText: 2024 RizinOrg +// SPDX-FileCopyrightText: 2024 deroad // SPDX-License-Identifier: LGPL-3.0-only -#include #include -#include +#include +#include +#include +#include +#include +#include +#include "search_internal.h" // Experimental search engine (fails, because stops at first hit of every block read #define USE_BMH 0 @@ -74,8 +80,7 @@ RZ_API int rz_search_strings_update(RzSearch *s, ut64 from, const ut8 *buf, int rz_return_val_if_fail(s && buf && len, -1); RzUtilStrScanOptions scan_opt = { - .buf_size = len, - .max_uni_blocks = s->string_max, + .max_str_length = len, .min_str_length = s->string_min, .prefer_big_endian = false, }; @@ -97,7 +102,7 @@ RZ_API int rz_search_strings_update(RzSearch *s, ut64 from, const ut8 *buf, int rz_list_foreach (s->kws, iter, kw) { RzDetectedString *dstr; rz_list_foreach (str_list, iter2, dstr) { - rz_search_hit_new(s, kw, dstr->addr); + rz_search_legacy_hit_new(s, kw, dstr->addr); matches++; } } @@ -139,7 +144,7 @@ RZ_API int rz_search_begin(RzSearch *s) { } // Returns 2 if search.maxhits is reached, 0 on error, otherwise 1 -RZ_API int rz_search_hit_new(RzSearch *s, RzSearchKeyword *kw, ut64 addr) { +RZ_API int rz_search_legacy_hit_new(RzSearch *s, RzSearchKeyword *kw, ut64 addr) { if (s->align > 1 && (addr % s->align)) { eprintf("0x%08" PFMT64x " unaligned\n", addr); return 1; @@ -165,7 +170,7 @@ RZ_API int rz_search_hit_new(RzSearch *s, RzSearchKeyword *kw, ut64 addr) { } kw->count++; s->nhits++; - RzSearchHit *hit = RZ_NEW0(RzSearchHit); + RzSearchLegacyHit *hit = RZ_NEW0(RzSearchLegacyHit); if (hit) { hit->kw = kw; hit->addr = addr; @@ -233,7 +238,7 @@ RZ_API int rz_search_deltakey_update(RzSearch *s, ut64 from, const ut8 *buf, int j++; } if (j == kw->keyword_length) { - int t = rz_search_hit_new(s, kw, s->bckwrds ? from - kw->keyword_length - 1 - i + left->len : from + i - left->len); + int t = rz_search_legacy_hit_new(s, kw, s->bckwrds ? from - kw->keyword_length - 1 - i + left->len : from + i - left->len); kw->last += s->bckwrds ? 0 : 1; if (!t) { return -1; @@ -257,7 +262,7 @@ RZ_API int rz_search_deltakey_update(RzSearch *s, ut64 from, const ut8 *buf, int j++; } if (j == kw->keyword_length) { - int t = rz_search_hit_new(s, kw, s->bckwrds ? from - kw->keyword_length - 1 - i : from + i); + int t = rz_search_legacy_hit_new(s, kw, s->bckwrds ? from - kw->keyword_length - 1 - i : from + i); kw->last += s->bckwrds ? 0 : 1; if (!t) { return -1; @@ -390,7 +395,7 @@ RZ_API int rz_search_mybinparse_update(RzSearch *s, ut64 from, const ut8 *buf, i : 0; for (; i + kw->keyword_length <= len1 && i < left->len; i++) { if (brute_force_match(s, kw, left->data, i) != s->inverse) { - int t = rz_search_hit_new(s, kw, s->bckwrds ? from - kw->keyword_length - i + left->len : from + i - left->len); + int t = rz_search_legacy_hit_new(s, kw, s->bckwrds ? from - kw->keyword_length - i + left->len : from + i - left->len); if (!t) { return -1; } @@ -407,7 +412,7 @@ RZ_API int rz_search_mybinparse_update(RzSearch *s, ut64 from, const ut8 *buf, i : 0; for (; i + kw->keyword_length <= len; i++) { if (brute_force_match(s, kw, buf, i) != s->inverse) { - int t = rz_search_hit_new(s, kw, s->bckwrds ? from - kw->keyword_length - i : from + i); + int t = rz_search_legacy_hit_new(s, kw, s->bckwrds ? from - kw->keyword_length - i : from + i); if (!t) { return -1; } @@ -475,7 +480,7 @@ RZ_API int rz_search_update_i(RzSearch *s, ut64 from, const ut8 *buf, long len) } static int listcb(RzSearchKeyword *k, void *user, ut64 addr) { - RzSearchHit *hit = RZ_NEW0(RzSearchHit); + RzSearchLegacyHit *hit = RZ_NEW0(RzSearchLegacyHit); if (!hit) { return 0; } @@ -536,3 +541,268 @@ RZ_API void rz_search_kw_reset(RzSearch *s) { rz_list_purge(s->hits); RZ_FREE(s->data); } + +// +// New search. +// Everything above is only there to not break the build. +// + +#include "search_internal.h" + +// RZ_LIB_VERSION(rz_search); + +typedef struct search_ctx { + RzIO *io; ///< the RzIO struct to use + RzThreadLock *io_lock; + RzSearchCollection *col; ///< collection to use + RzSearchOpt *opt; ///< User options + RzThreadQueue /* RzSearchHits */ *hits; ///< Hits list + RzAtomicBool *loop; ///< If set, the execution will continue until it terminates. If unset, the execution cancels. +} search_ctx_t; + +static void *search_cancel_th(void *user) { + search_ctx_t *ctx = (search_ctx_t *)user; + RzSearchOpt *opt = ctx->opt; + + do { + size_t n_hits = rz_th_queue_size(ctx->hits); + if (opt->cancel_cb(opt->cancel_usr, n_hits, RZ_SEARCH_CANCEL_REGULAR_CHECK)) { + rz_atomic_bool_set(ctx->loop, false); + break; + } + rz_sys_usleep(RZ_SEARCH_CANCEL_CHECK_INTERVAL_USEC); + } while (rz_atomic_bool_get(ctx->loop)); + + return NULL; +} + +static bool search_iterator_io_map_cb(void *element, void *user) { + search_ctx_t *ctx = (search_ctx_t *)user; + RzInterval *window = (RzInterval *)element; + if (!window) { + return rz_atomic_bool_get(ctx->loop); + } + if (!ctx->opt) { + RZ_LOG_ERROR("No search options given.\n"); + return false; + } + + RzSearchCollection *col = ctx->col; + + ut64 at = window->addr; + ut64 size = window->size; + + rz_th_lock_enter(ctx->io_lock); + RzBuffer *buffer = rz_io_nread_at_new_buf(ctx->io, at, size); + if (!buffer || rz_buf_size(buffer) != size) { + RZ_LOG_ERROR("search: failed to read at 0x%08" PFMT64x " (0x%08" PFMT64x " bytes)\n", at, size); + rz_th_lock_leave(ctx->io_lock); + goto failure; + } + rz_th_lock_leave(ctx->io_lock); + + RzSearchFindBytesCallback find = col->find; + if (!find(ctx->opt->find_opts, col->user, at, buffer, ctx->hits)) { + RZ_LOG_ERROR("search: failed search at 0x%08" PFMT64x "\n", at); + goto failure; + } + + rz_buf_free(buffer); + return rz_atomic_bool_get(ctx->loop); + +failure: + rz_buf_free(buffer); + rz_atomic_bool_set(ctx->loop, false); + return false; +} + +static RzList /**/ *assemble_search_window_list(RzList /**/ *search_in, RzSearchOpt *opt) { + rz_return_val_if_fail(search_in && opt && opt->element_size, NULL); + RzList *list = rz_list_newf(free); + if (!list) { + return NULL; + } + + RzIOMap *map; + RzListIter *iter; + rz_list_foreach (search_in, iter, map) { + ut64 start = map->itv.addr; + ut64 end = start + map->itv.size; + for (size_t chunk_begin = start; chunk_begin < end; chunk_begin += opt->chunk_size) { + ut64 window_size = opt->chunk_size + opt->element_size - 1; + if (chunk_begin + window_size > end) { + window_size = end - chunk_begin; + } + + RzInterval *window = RZ_NEW0(RzInterval); + window->addr = chunk_begin; + window->size = window_size; + rz_list_append(list, window); + } + } + return list; +} + +/** + * \brief Perform a search within the given search maps of a collection + * + * \param opt The RzSearchOpt to use + * \param col The RzSearchCollection to use + * \param io The RzIO layer to use + * \param search_in The search maps for the boundaries + * + * \return On success returns all the hits. + */ +RZ_API RZ_OWN RzList /**/ *rz_search_on_io( + RZ_BORROW RZ_NONNULL RzSearchOpt *opt, + RZ_BORROW RZ_NONNULL RzSearchCollection *col, + RZ_BORROW RZ_NONNULL RzIO *io, + RZ_BORROW RZ_NONNULL RzList /**/ *search_in) { + rz_return_val_if_fail(opt && col && io && search_in, NULL); + search_ctx_t ctx = { 0 }; + RzList *results = NULL; + RzThreadQueue *hits = NULL; + RzList /* RzInterval */ *windows = NULL; + RzThread *cancel_th = NULL; + + if (!rz_search_collection_on_bytes_space(col)) { + RZ_LOG_ERROR("search: The search collection is not initialized for byte space.\n"); + return NULL; + } + + if (opt->chunk_size < RZ_SEARCH_MIN_CHUNK_SIZE) { + RZ_LOG_ERROR("search: cannot search when buffer size is less than %#" PFMT64x " bytes.\n", RZ_SEARCH_MIN_CHUNK_SIZE); + return NULL; + } + + if (rz_list_empty(search_in)) { + RZ_LOG_ERROR("search: cannot search in an empty RzIOMap list.\n"); + return NULL; + } + + if (rz_search_collection_is_empty(col)) { + RZ_LOG_ERROR("search: cannot perform the search when the search collection is empty.\n"); + return NULL; + } + + hits = rz_th_queue_new(RZ_THREAD_QUEUE_UNLIMITED, (RzListFree)rz_search_hit_free); + if (!hits) { + RZ_LOG_ERROR("search: cannot allocate RzSearchHit queue.\n"); + return NULL; + } + + windows = assemble_search_window_list(search_in, opt); + if (!windows) { + RZ_LOG_ERROR("search: Could not prepare search window queue.\n"); + rz_list_free(windows); + return NULL; + } + + ctx.col = col; + ctx.opt = opt; + ctx.io = io; + ctx.io_lock = rz_th_lock_new(false); + ctx.loop = rz_atomic_bool_new(true); + ctx.hits = hits; + + if (opt->cancel_cb) { + // create cancel thread + cancel_th = rz_th_new(search_cancel_th, &ctx); + if (!cancel_th) { + RZ_LOG_ERROR("search: cannot allocate cancel thread.\n"); + rz_th_queue_free(hits); + rz_atomic_bool_free(ctx.loop); + return NULL; + } + } + + if (!rz_th_iterate_list(windows, search_iterator_io_map_cb, opt->max_threads, &ctx)) { + RZ_LOG_ERROR("search: cannot iterate over list.\n"); + } else { + results = rz_th_queue_pop_all(hits); + } + + if (cancel_th) { + // stop & free cancel thread. + rz_atomic_bool_set(ctx.loop, false); + rz_th_wait(cancel_th); + rz_th_free(cancel_th); + rz_atomic_bool_free(ctx.loop); + } + + rz_th_lock_free(ctx.io_lock); + rz_list_free(windows); + rz_th_queue_free(hits); + return results; +} + +/** + * \brief Allocate and initialize a new RzSearchHit + * + * \param[in] hit_desc The hit description linked to the hit (can be NULL) + * \param[in] address The address where the hit happened + * \param[in] size The size of the hit data (can be 0) + * + * \return On success returns a valid pointer, otherwise NULL + */ +RZ_IPI RZ_OWN RzSearchHit *rz_search_hit_new(const char *hit_desc, ut64 address, size_t size) { + RzSearchHit *hit = RZ_NEW0(RzSearchHit); + if (!hit) { + return NULL; + } + hit->hit_desc = rz_str_dup(hit_desc); + hit->address = address; + hit->size = size; + return hit; +} + +/** + * \brief Frees a RzSearchHit structure + * + * \param hit The RzSearchHit pointer to free + */ +RZ_IPI void rz_search_hit_free(RZ_NULLABLE RzSearchHit *hit) { + if (!hit) { + return; + } + free(hit->hit_desc); + free(hit); +} + +/** + * \brief Get a flag name describing the hit. The flag name can be customized. + * + * \param hit The RzSearchHit to build the flag name for. + * \param hit_id The id number of the hit. + * \param prefix An optional prefix for the flag. Defaults to "hit". + * + * Example: + * + * hit = { address = 0x110, hit_desc = "bytes", size = 0x10 } + * prefix = "sb" + * + * Result = sb.bytes.0 + + * hit = { address = 0x110, hit_desc = NULL, size = 0x10 } + * prefix = NULL + * + * Result = hit.0 + * + * \return A flag of \p hit, or NULL in case of failure. + */ +RZ_API RZ_OWN char *rz_search_hit_flag_name(RZ_NONNULL const RzSearchHit *hit, + size_t hit_id, + RZ_NULLABLE const char *prefix) { + rz_return_val_if_fail(hit, NULL); + RzStrBuf *buf = rz_strbuf_new(""); + if (!buf) { + return NULL; + } + rz_strbuf_appendf(buf, "%s", prefix ? prefix : "hit"); + if (hit->hit_desc) { + rz_strbuf_appendf(buf, ".%s", hit->hit_desc); + } + rz_strbuf_appendf(buf, ".%" PFMTSZd, hit_id); + + return rz_strbuf_drain(buf); +} diff --git a/librz/search/search_internal.h b/librz/search/search_internal.h new file mode 100644 index 0000000000..1ba9a41291 --- /dev/null +++ b/librz/search/search_internal.h @@ -0,0 +1,141 @@ +// SPDX-FileCopyrightText: 2024 RizinOrg +// SPDX-FileCopyrightText: 2024 deroad +// SPDX-License-Identifier: LGPL-3.0-only + +#ifndef RZ_SEARCH_INTERNAL_H +#define RZ_SEARCH_INTERNAL_H + +#include +#include +#include + +#define RZ_SEARCH_AES_LENGTH 40 +#define RZ_SEARCH_PRIVATE_KEY_LENGTH 11 +#define RZ_SEARCH_MAX_HEX_PATTERN UT16_MAX + +/** + * \brief The number of elements per search chunk. + * Note, the actual buffer, passed to the find() callback, + * will be bigger by sizeof(element) - 1. + * To also find elements crossing chunk boundaries. + * + * ATTENTION: If you change this value, update the test + * in cmd_search_x::"search over boundary" + */ +#define RZ_SEARCH_MIN_ELEMENTS_PER_CHUNK 64u + +/** + * \brief Minimal buffer size for each find() thread in bytes. + */ +#define RZ_SEARCH_MIN_CHUNK_SIZE 32ull + +/** + * \brief Default buffer size for each find() thread in bytes. + * Size: 4096 + * + * ATTENTION: If you change this value, update the test + * in cmd_search_x::"search over boundary" + */ +#define RZ_SEARCH_DEFAULT_CHUNK_SIZE 0x1000ull + +/** + * \brief Maximum buffer size to check in each find() thread in bytes. + * Size: 4G + */ +#define RZ_SEARCH_MAX_CHUNK_SIZE 0x100000000ull +#define RZ_SEARCH_CANCEL_CHECK_INTERVAL_USEC 100 * 1000 + +/** + * \brief The callback to free the private user data in the RzSearchCollection. + * + * \param user The private user data to free. + */ +typedef void (*RzSearchFreeCallback)(void *user); + +/** + * \brief The callback to check if the search collection is considered empty. + * + * \param user The private user data. + */ +typedef bool (*RzSearchIsEmptyCallback)(void *user); + +/** + * \brief A callback checking a chunk of bytes if it matches the search criteria. + * + * \param user The private user data. + * \param address The address associated with the given bytes. + * \param buffer The bytes buffer. + * \param The queue to push new hits onto. + * + * \return True, if a match was found. + * \return False otherwise. + */ +typedef bool (*RzSearchFindBytesCallback)(RZ_NULLABLE RzSearchFindOpt *fopt, void *user, ut64 address, const RzBuffer *buffer, RZ_OUT RzThreadQueue *hits); + +/** + * \brief A callback to search a graph for a pattern. + * + * \param user The private user data. + * \param graph The graph to search in. + * \param The queue to push new hits onto. + * + * \return True, if a match was found. + * \return False otherwise. + */ +typedef bool (*RzSearchFindGraphCallback)(RzSearchFindOpt *fopt, void *user, const RzGraph *graph, RZ_OUT RzThreadQueue *hits); + +typedef enum { + RZ_SEARCH_SPACE_BYTES = 0, ///< The search is performed on bytes. + RZ_SEARCH_SPACE_GRAPH, ///< The search is performed on a graph. + RZ_SEARCH_SPACE_KB, ///< The search is performed on the knowledge base. +} RzSearchSpace; + +struct rz_search_collection_t { + void *user; ///< Context defined by the various collections + RzSearchSpace space; ///< The search space of this collection. + void *find; ///< Callback to do the search in a given chunk. The callback type depends on \ref rz_search_collection_t.space. + RzSearchIsEmptyCallback is_empty; ///< Callback used to check if the collection is empty. + RzSearchFreeCallback free; ///< Callback used to free the collection. +}; + +struct rz_search_bytes_pattern_t { + const char *pattern_desc; ///< Pattern description string. + ut8 *bytes; ///< Pattern bytes. + ut8 *mask; ///< Pattern mask (when NULL full match) + RzRegex *regex; ///< Regex patterns of the bytes. Is optional. + size_t length; ///< Pattern & mask length +}; + +struct rz_search_opt_t { + RzSearchFindOpt *find_opts; + size_t max_hits; + ut64 chunk_size; + ut64 element_size; + RzThreadNCores max_threads; + + // cancel callback + void *cancel_usr; + RzSearchCancelCallback cancel_cb; +}; + +struct rz_search_find_opt_t { + bool match_inverse; ///< Set if the inverse of the given pattern should be matched. + bool match_overlap; ///< Set if hits can overlap. + size_t alignment; ///< The address alignment to start the search from. If >1, only `buffer + (alignment * x)` is searched. +}; + +RZ_IPI RZ_OWN RzSearchHit *rz_search_hit_new(const char *metadata, ut64 address, size_t size); +RZ_IPI void rz_search_hit_free(RZ_NULLABLE RzSearchHit *hit); + +RZ_IPI RZ_OWN RzSearchCollection *rz_search_collection_new_bytes_space(RZ_NONNULL RzSearchFindBytesCallback find, RZ_NONNULL RzSearchIsEmptyCallback is_empty, RZ_NULLABLE RzSearchFreeCallback free, RZ_NULLABLE void *user); +RZ_IPI RZ_OWN RzSearchCollection *rz_search_collection_new_graph_space(RZ_NONNULL RzSearchFindGraphCallback find, RZ_NONNULL RzSearchIsEmptyCallback is_empty, RZ_NULLABLE RzSearchFreeCallback free, RZ_NULLABLE void *user); +RZ_IPI bool rz_search_collection_has_find_callback(RZ_NONNULL RzSearchCollection *col, RZ_NONNULL void *expected); +RZ_IPI bool rz_search_collection_is_empty(RZ_NONNULL RzSearchCollection *col); +RZ_IPI static inline bool rz_search_collection_on_bytes_space(RZ_NONNULL RzSearchCollection *col) { + return col->space == RZ_SEARCH_SPACE_BYTES; +}; +RZ_IPI static inline bool rz_search_collection_on_graph_space(RZ_NONNULL RzSearchCollection *col) { + return col->space == RZ_SEARCH_SPACE_GRAPH; +}; + +#endif /* RZ_SEARCH_INTERNAL_H */ diff --git a/librz/util/str_search.c b/librz/util/str_search.c index 95ba54d36c..d6a63bd427 100644 --- a/librz/util/str_search.c +++ b/librz/util/str_search.c @@ -75,6 +75,7 @@ RZ_API void rz_detected_string_free(RzDetectedString *str) { return; } free(str->string); + rz_regex_free(str->regex); free(str); } @@ -113,7 +114,6 @@ static UTF8StringInfo calculate_utf8_string_info(ut8 *str, int size) { } static FalsePositiveResult reduce_false_positives(const RzUtilStrScanOptions *opt, ut8 *str, int size, RzStrEnc str_type) { - switch (str_type) { case RZ_STRING_ENC_8BIT: { for (int i = 0; i < size; i++) {