String search performance improvement (#5383)
* Add speed up search for unaligned buffers and string encodings. The new search mode adjusts the buffer's data (swap endianness, fix alignment) so the buffer can be searched with PCRE2 without scanning. It also enables RzRegexMulti opbjects to have their own JIT stack. This makes them usable in a thread safe manner. * Fix regex compilation on big endian systems * Update the scanning check (UTF needs no scanning anymore).
This commit is contained in:
Rot127
GitHub
parent
805600d184
commit
2d7b178c84
12 changed files with 400 additions and 131 deletions
|
|
@ -294,7 +294,7 @@ RZ_API RZ_OWN RzList /*<RzSearchHit *>*/ *rz_core_search_string(RZ_NONNULL RzCor
|
|||
}
|
||||
|
||||
size_t match_alignment = rz_search_find_opt_get_alignment(rz_search_opt_get_find_options(user_opts));
|
||||
collection = rz_search_collection_strings(&scan_opt);
|
||||
collection = rz_search_collection_strings(&scan_opt, rz_search_opt_get_max_threads(user_opts));
|
||||
if (!collection ||
|
||||
!rz_search_collection_string_add(collection, re_pattern, cflags, match_alignment, encoding)) {
|
||||
rz_search_collection_free(collection);
|
||||
|
|
|
|||
|
|
@ -211,6 +211,7 @@ RZ_API void rz_search_opt_free(RZ_NULLABLE RzSearchOpt *opt);
|
|||
RZ_API bool rz_search_opt_set_max_hits(RZ_NONNULL RzSearchOpt *opt, size_t max_hits);
|
||||
RZ_API bool rz_search_opt_set_chunk_size(RZ_NONNULL RzSearchOpt *opt, ut64 chunk_size);
|
||||
RZ_API bool rz_search_opt_set_max_threads(RZ_NONNULL RzSearchOpt *opt, RzThreadNCores max_threads);
|
||||
RZ_API size_t rz_search_opt_get_max_threads(RZ_NONNULL const RzSearchOpt *opt);
|
||||
RZ_API bool rz_search_opt_set_show_progress_from_str(RZ_NONNULL RzSearchOpt *opt, const char *show_progress);
|
||||
RZ_API RzSearchProgress rz_search_opt_get_show_progress(RZ_NONNULL RzSearchOpt *opt);
|
||||
RZ_API bool rz_search_opt_set_cancel_cb(RZ_NONNULL RzSearchOpt *opt, RzSearchCancelCallback callback, void *user);
|
||||
|
|
@ -264,7 +265,7 @@ RZ_API RZ_OWN RzSearchCollection *rz_search_collection_bytes();
|
|||
RZ_API bool rz_search_collection_bytes_add(RZ_NONNULL RzSearchCollection *col, RZ_NULLABLE const char *pattern_desc, RZ_NONNULL const ut8 *bytes, RZ_NULLABLE const ut8 *mask, size_t length);
|
||||
RZ_API bool rz_search_collection_bytes_add_pattern(RZ_NONNULL RzSearchCollection *col, RZ_NONNULL RZ_OWN RzSearchBytesPattern *bytes_pattern);
|
||||
|
||||
RZ_API RZ_OWN RzSearchCollection *rz_search_collection_strings(RZ_NULLABLE RZ_BORROW RzUtilStrScanOptions *scan_opts);
|
||||
RZ_API RZ_OWN RzSearchCollection *rz_search_collection_strings(RZ_NULLABLE RZ_BORROW RzUtilStrScanOptions *scan_opts, size_t n_threads);
|
||||
RZ_API bool rz_search_collection_string_add(
|
||||
RZ_NONNULL RzSearchCollection *col,
|
||||
RZ_NONNULL const char *regex_pattern,
|
||||
|
|
@ -286,6 +287,8 @@ RZ_API RZ_OWN RzSearchCollection *rz_search_collection_magic(RZ_NONNULL const ch
|
|||
RZ_API RZ_OWN RzList /*<RzSearchHit *>*/ *rz_search_on_io(RZ_BORROW RZ_NONNULL RzSearchOpt *opt, RZ_BORROW RZ_NONNULL RzSearchCollection *col, RZ_BORROW RZ_NONNULL RzIO *io, RZ_BORROW RZ_NONNULL RzList /*<RzIOMap *>*/ *search_in);
|
||||
RZ_API RZ_OWN RzList /*<RzSearchHit *>*/ *rz_search_on_buffer(RZ_BORROW RZ_NONNULL RzSearchOpt *opt, RZ_BORROW RZ_NONNULL RzSearchCollection *col, RZ_BORROW RZ_NONNULL RzBuffer *buffer);
|
||||
|
||||
RZ_API bool rz_search_str_enc_needs_scanning(RzStrEnc encoding);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -74,6 +74,7 @@ typedef struct {
|
|||
RzRegex16 *re16;
|
||||
RzRegex32 *re32;
|
||||
};
|
||||
void *jit_stack; ///< The JIT stack for this pattern. Must be used only by one thread at a time.
|
||||
} RzRegexMulti;
|
||||
|
||||
typedef struct {
|
||||
|
|
|
|||
|
|
@ -312,6 +312,7 @@ static inline bool rz_string_enc_same_char_width_as_utf8(RzStrEnc enc) {
|
|||
return enc == RZ_STRING_ENC_UTF8 || enc == RZ_STRING_ENC_8BIT;
|
||||
}
|
||||
|
||||
RZ_API bool rz_string_enc_is_utf(RzStrEnc enc);
|
||||
RZ_API bool rz_string_enc_is_utf_native_endian(RzStrEnc enc);
|
||||
RZ_API size_t rz_string_enc_code_point_width(RzStrEnc enc);
|
||||
|
||||
|
|
|
|||
|
|
@ -72,6 +72,14 @@ RZ_API bool rz_search_opt_set_max_threads(RZ_NONNULL RzSearchOpt *opt, RzThreadN
|
|||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Returns the number of threads defined in the options \p opt.
|
||||
*/
|
||||
RZ_API size_t rz_search_opt_get_max_threads(RZ_NONNULL const RzSearchOpt *opt) {
|
||||
rz_return_val_if_fail(opt, 0);
|
||||
return opt->max_threads;
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Enables the printing of progress during the search.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -24,6 +24,9 @@
|
|||
|
||||
/**
|
||||
* \brief Minimal buffer size for each find() thread in bytes.
|
||||
*
|
||||
* ATTENTION: This value must be aligned to at least 4bytes.
|
||||
* Otherwise the string search breaks.
|
||||
*/
|
||||
#define RZ_SEARCH_MIN_CHUNK_SIZE 32ull
|
||||
|
||||
|
|
@ -33,6 +36,9 @@
|
|||
*
|
||||
* ATTENTION: If you change this value, update the test
|
||||
* in cmd_search_x::"search over boundary"
|
||||
*
|
||||
* ATTENTION: This value must be aligned to at least 4bytes.
|
||||
* Otherwise the string search breaks.
|
||||
*/
|
||||
#define RZ_SEARCH_DEFAULT_CHUNK_SIZE 0x1000ull
|
||||
|
||||
|
|
|
|||
|
|
@ -8,14 +8,57 @@
|
|||
#include <rz_vector.h>
|
||||
#include <rz_util/ht_uu.h>
|
||||
#include <rz_util/rz_str_search.h>
|
||||
#include "rz_util/rz_str.h"
|
||||
#include <rz_th.h>
|
||||
#include "rz_util/rz_assert.h"
|
||||
#include "search_internal.h"
|
||||
|
||||
typedef struct string_search {
|
||||
RzUtilStrScanOptions options; ///< String scan options
|
||||
RzPVector /*<RzDetectedString *>*/ *strings; ///< Strings to search
|
||||
/**
|
||||
* \brief Strings to search.
|
||||
* Each thread gets its own thread safe copy of the patterns to search.
|
||||
* The outer vector is indexed by thread ID.
|
||||
*/
|
||||
RzPVector /*<RzPVector<RzDetectedString *> *>*/ *strings;
|
||||
/**
|
||||
* \brief This queue holds the thread ids. Each worker is draws a
|
||||
* one number and then pick its thread safe object for searching strings from
|
||||
* above.
|
||||
* On return it must push its ID into the queue again to release it.
|
||||
*/
|
||||
RzThreadQueue *thread_ids;
|
||||
} StringSearch;
|
||||
|
||||
/**
|
||||
* \brief Returns true if the string encoding requires scanning and conversion
|
||||
* to UTF-8 for searching.
|
||||
* Returns false if direct matching with PCRE2 is supported.
|
||||
*/
|
||||
RZ_API bool rz_search_str_enc_needs_scanning(RzStrEnc encoding) {
|
||||
switch (encoding) {
|
||||
case RZ_STRING_ENC_8BIT:
|
||||
case RZ_STRING_ENC_UTF8:
|
||||
case RZ_STRING_ENC_MUTF8:
|
||||
case RZ_STRING_ENC_UTF16LE:
|
||||
case RZ_STRING_ENC_UTF32LE:
|
||||
case RZ_STRING_ENC_UTF16BE:
|
||||
case RZ_STRING_ENC_UTF32BE:
|
||||
return false;
|
||||
case RZ_STRING_ENC_IBM037:
|
||||
case RZ_STRING_ENC_IBM290:
|
||||
case RZ_STRING_ENC_EBCDIC_UK:
|
||||
case RZ_STRING_ENC_EBCDIC_US:
|
||||
case RZ_STRING_ENC_EBCDIC_ES:
|
||||
case RZ_STRING_ENC_GUESS:
|
||||
return true;
|
||||
case RZ_STRING_ENC_SETTINGS:
|
||||
rz_warn_if_reached();
|
||||
return true;
|
||||
}
|
||||
rz_warn_if_reached();
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief UTF-8 and the encoding of the real string (in memory) must not match.
|
||||
* For example, if the real string is UTF-16 or UTF-32.
|
||||
|
|
@ -39,13 +82,11 @@ static bool native_string_find(RzSearchFindOpt *fopt, RzDetectedString *find, ut
|
|||
const ut8 *raw_buf = rz_buf_get_whole_hot_paths((RzBuffer *)buffer, &size);
|
||||
RzPVector *matches = NULL;
|
||||
|
||||
RzRegexMulti *re = rz_regex_multi_clone(find->regex, true);
|
||||
if (fopt->match_overlap) {
|
||||
matches = rz_regex_match_all_overlap_multi(re, raw_buf, size, 0, RZ_REGEX_DEFAULT);
|
||||
matches = rz_regex_match_all_overlap_multi(find->regex, raw_buf, size, 0, RZ_REGEX_DEFAULT);
|
||||
} else {
|
||||
matches = rz_regex_match_all_multi(re, raw_buf, size, 0, RZ_REGEX_DEFAULT);
|
||||
matches = rz_regex_match_all_multi(find->regex, raw_buf, size, 0, RZ_REGEX_DEFAULT);
|
||||
}
|
||||
rz_regex_free_multi_clone(re);
|
||||
if (!matches) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -60,10 +101,6 @@ static bool native_string_find(RzSearchFindOpt *fopt, RzDetectedString *find, ut
|
|||
}
|
||||
ut64 str_mem_len = group0->len * rz_string_enc_code_point_width(find->encoding);
|
||||
ut64 str_mem_offset = group0->start * rz_string_enc_code_point_width(find->encoding);
|
||||
if (fopt->alignment > 1 && rz_mem_align_padding(str_mem_offset, fopt->alignment) != 0) {
|
||||
// Match has not the correct alignment in memory.
|
||||
continue;
|
||||
}
|
||||
if (find->alignment > 1 && rz_mem_align_padding(str_mem_offset, find->alignment) != 0) {
|
||||
// Match has not the correct alignment in memory.
|
||||
continue;
|
||||
|
|
@ -82,24 +119,128 @@ static bool native_string_find(RzSearchFindOpt *fopt, RzDetectedString *find, ut
|
|||
return true;
|
||||
}
|
||||
|
||||
static inline int next_i(int i, RzStrEnc enc, size_t alignment) {
|
||||
if (i < 0 || alignment >= 4) {
|
||||
return -1;
|
||||
}
|
||||
switch (enc) {
|
||||
default:
|
||||
return -1;
|
||||
case RZ_STRING_ENC_UTF16BE:
|
||||
case RZ_STRING_ENC_UTF16LE:
|
||||
return i >= 1 || alignment != 1 ? -1 : i + 1;
|
||||
case RZ_STRING_ENC_UTF32BE:
|
||||
case RZ_STRING_ENC_UTF32LE:
|
||||
return i >= 3 || !RZ_BETWEEN(1, alignment, 2) ? -1 : i + 1;
|
||||
}
|
||||
}
|
||||
|
||||
static bool adjusted_buffer_string_find(RzSearchFindOpt *fopt, RzDetectedString *find, ut64 offset, const RzBuffer *buffer,
|
||||
RZ_OUT RzThreadQueue *hits, RZ_OUT size_t *n_hits) {
|
||||
if (fopt->alignment < 4 && !RZ_BETWEEN(1, fopt->alignment, 2)) {
|
||||
rz_warn_if_reached();
|
||||
return false;
|
||||
}
|
||||
ut64 ab_size;
|
||||
const ut8 *raw_buf = rz_buf_get_whole_hot_paths((RzBuffer *)buffer, &ab_size);
|
||||
if (!raw_buf || !ab_size || ab_size != rz_buf_size((RzBuffer *)buffer)) {
|
||||
rz_warn_if_reached();
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// PCRE2 can match UTF-8/16/32. But only if the buffer is aligned properly
|
||||
// to the code point width and is of host endianness.
|
||||
// This branch handles the cases where one or both are not true.
|
||||
size_t i = 0;
|
||||
do {
|
||||
ut8 *adjusted_buf = NULL;
|
||||
// Check if the alignment missmatches. If so, adjust it.
|
||||
if (!rz_string_code_points_align(find->encoding, find->alignment)) {
|
||||
adjusted_buf = rz_mem_copy_offset(raw_buf, ab_size, i);
|
||||
if (!adjusted_buf) {
|
||||
rz_warn_if_reached();
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
// Now swap the endianness if required.
|
||||
if (!rz_string_enc_is_utf_native_endian(find->encoding)) {
|
||||
switch (find->encoding) {
|
||||
default:
|
||||
rz_warn_if_reached();
|
||||
free(adjusted_buf);
|
||||
return NULL;
|
||||
case RZ_STRING_ENC_UTF32BE:
|
||||
case RZ_STRING_ENC_UTF32LE:
|
||||
adjusted_buf = adjusted_buf ? rz_mem_swap_bytes_4_inplace(adjusted_buf, ab_size) : rz_mem_swap_bytes_4(raw_buf, ab_size);
|
||||
break;
|
||||
case RZ_STRING_ENC_UTF16BE:
|
||||
case RZ_STRING_ENC_UTF16LE:
|
||||
adjusted_buf = adjusted_buf ? rz_mem_swap_bytes_2_inplace(adjusted_buf, ab_size) : rz_mem_swap_bytes_2(raw_buf, ab_size);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!adjusted_buf) {
|
||||
// Alignment is fine in this iteration and there is no endianness switch.
|
||||
// So use the default buffer.
|
||||
if (!native_string_find(fopt, find, offset, buffer, hits, n_hits)) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
RzBuffer *ab = rz_buf_new_from_bytes(adjusted_buf, ab_size);
|
||||
if (!native_string_find(fopt, find, offset + i, ab, hits, n_hits)) {
|
||||
rz_buf_free(ab);
|
||||
return false;
|
||||
}
|
||||
rz_buf_free(ab);
|
||||
}
|
||||
|
||||
i = next_i(i, find->encoding, find->alignment);
|
||||
if (i == -1) {
|
||||
break;
|
||||
}
|
||||
} while (true);
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline bool do_search_by_direct_matching(RzStrEnc encoding, size_t buf_alignment) {
|
||||
return rz_string_enc_is_utf_native_endian(encoding) &&
|
||||
rz_string_code_points_align(encoding, buf_alignment);
|
||||
}
|
||||
|
||||
static inline bool do_search_with_adjusted_buffer(RzStrEnc encoding, size_t alignment) {
|
||||
return rz_string_enc_is_utf(encoding) && !do_search_by_direct_matching(encoding, alignment);
|
||||
}
|
||||
|
||||
static bool string_find(RzSearchFindOpt *fopt, void *user, ut64 offset, const RzBuffer *buffer,
|
||||
RZ_OUT RzThreadQueue *hits, RZ_OUT size_t *n_hits) {
|
||||
rz_return_val_if_fail(fopt, false);
|
||||
|
||||
StringSearch *ss = (StringSearch *)user;
|
||||
size_t *thread_id = rz_th_queue_pop(ss->thread_ids, false);
|
||||
if (!thread_id) {
|
||||
return false;
|
||||
}
|
||||
RzPVector *strings = rz_pvector_at(ss->strings, *thread_id);
|
||||
|
||||
void **it_m = NULL;
|
||||
rz_pvector_foreach (ss->strings, it_m) {
|
||||
rz_pvector_foreach (strings, it_m) {
|
||||
RzDetectedString *find = *it_m;
|
||||
if (do_search_by_direct_matching(find->encoding, fopt->alignment)) {
|
||||
// The expected encoding is UTF with native endian.
|
||||
// Also, the alignment matches the code point width (offset is always aligned to 64bytes).
|
||||
// For those we can do simple regex matching, skipping the whole decoding stuff.
|
||||
if (!native_string_find(fopt, find, offset, buffer, hits, n_hits)) {
|
||||
return false;
|
||||
goto release_thread_id_ret_false;
|
||||
}
|
||||
continue;
|
||||
} else if (do_search_with_adjusted_buffer(find->encoding, fopt->alignment)) {
|
||||
// This string search is used for UTF encodings which:
|
||||
// - Have the opposite endianness of the host system.
|
||||
// - AND/OR have a code point width not congruent to fopt->alignment.
|
||||
if (!adjusted_buffer_string_find(fopt, find, offset, buffer, hits, n_hits)) {
|
||||
goto release_thread_id_ret_false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
|
@ -110,8 +251,6 @@ static bool string_find(RzSearchFindOpt *fopt, void *user, ut64 offset, const Rz
|
|||
// This costs a lot. So it is only done for strings with:
|
||||
// A) A funny encodig we can't match directly with RzRegex/PCRE2 (e.g. EBCDIC).
|
||||
// B) Encoding must be guessed.
|
||||
// C) Matches can be at misaligned memory addresses
|
||||
// (PCRE2 only matches strings aligned to their code point width).
|
||||
|
||||
RzDetectedString *detected = NULL;
|
||||
RzListIter *it_s = NULL;
|
||||
|
|
@ -119,7 +258,7 @@ static bool string_find(RzSearchFindOpt *fopt, void *user, ut64 offset, const Rz
|
|||
RzList *found = rz_list_newf((RzListFree)rz_detected_string_free);
|
||||
if (!found) {
|
||||
RZ_LOG_ERROR("search: failed to allocate found list for strings collection\n");
|
||||
return false;
|
||||
goto release_thread_id_ret_false;
|
||||
}
|
||||
|
||||
// Copy options here so we can set the hash table.
|
||||
|
|
@ -131,19 +270,17 @@ static bool string_find(RzSearchFindOpt *fopt, void *user, ut64 offset, const Rz
|
|||
if (n_str_in_buf < 0) {
|
||||
RZ_LOG_ERROR("Failed to scan buffer for strings.\n");
|
||||
rz_list_free(found);
|
||||
return false;
|
||||
goto release_thread_id_ret_false;
|
||||
}
|
||||
|
||||
*n_hits = 0;
|
||||
rz_list_foreach (found, it_s, detected) {
|
||||
RzRegexMulti *re = rz_regex_multi_clone(find->regex, true);
|
||||
RzPVector *matches = NULL;
|
||||
if (fopt->match_overlap) {
|
||||
matches = rz_regex_match_all_overlap(re->re8, detected->string, RZ_REGEX_ZERO_TERMINATED, 0, RZ_REGEX_DEFAULT);
|
||||
matches = rz_regex_match_all_overlap(find->regex->re8, detected->string, RZ_REGEX_ZERO_TERMINATED, 0, RZ_REGEX_DEFAULT);
|
||||
} else {
|
||||
matches = rz_regex_match_all(re->re8, detected->string, RZ_REGEX_ZERO_TERMINATED, 0, RZ_REGEX_DEFAULT);
|
||||
matches = rz_regex_match_all(find->regex->re8, detected->string, RZ_REGEX_ZERO_TERMINATED, 0, RZ_REGEX_DEFAULT);
|
||||
}
|
||||
rz_regex_free_multi_clone(re);
|
||||
void **it;
|
||||
rz_pvector_foreach (matches, it) {
|
||||
RzPVector *match = *it;
|
||||
|
|
@ -152,7 +289,7 @@ static bool string_find(RzSearchFindOpt *fopt, void *user, ut64 offset, const Rz
|
|||
RZ_LOG_ERROR("search: Failed to get group of match.\n");
|
||||
rz_list_free(found);
|
||||
rz_pvector_free(matches);
|
||||
return false;
|
||||
goto release_thread_id_ret_false;
|
||||
}
|
||||
ut64 str_mem_len;
|
||||
ut64 str_mem_offset;
|
||||
|
|
@ -172,7 +309,7 @@ static bool string_find(RzSearchFindOpt *fopt, void *user, ut64 offset, const Rz
|
|||
rz_search_hit_free(hit);
|
||||
rz_list_free(found);
|
||||
rz_pvector_free(matches);
|
||||
return false;
|
||||
goto release_thread_id_ret_false;
|
||||
}
|
||||
(*n_hits)++;
|
||||
}
|
||||
|
|
@ -180,13 +317,24 @@ static bool string_find(RzSearchFindOpt *fopt, void *user, ut64 offset, const Rz
|
|||
}
|
||||
rz_list_free(found);
|
||||
}
|
||||
|
||||
rz_th_queue_push(ss->thread_ids, thread_id, true);
|
||||
return true;
|
||||
|
||||
release_thread_id_ret_false:
|
||||
rz_th_queue_push(ss->thread_ids, thread_id, true);
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool string_is_empty(void *user) {
|
||||
StringSearch *ss = (StringSearch *)user;
|
||||
return rz_pvector_empty(ss->strings);
|
||||
void **it;
|
||||
rz_pvector_foreach (ss->strings, it) {
|
||||
RzPVector *pv = *it;
|
||||
if (!rz_pvector_empty(pv)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static void string_free(void *user) {
|
||||
|
|
@ -205,21 +353,37 @@ static void string_free(void *user) {
|
|||
* It can be NULL if string scanning is not required.
|
||||
* rz_search_collection_string_add() will refuse in this case to
|
||||
* add patterns to the collection which require it.
|
||||
* \param n_threads Number of threads for the search. Must be >0.
|
||||
*
|
||||
* \return On success returns a valid pointer, otherwise NULL
|
||||
*/
|
||||
RZ_API RZ_OWN RzSearchCollection *rz_search_collection_strings(RZ_NULLABLE RZ_BORROW RzUtilStrScanOptions *scan_opts) {
|
||||
RZ_API RZ_OWN RzSearchCollection *rz_search_collection_strings(RZ_NULLABLE RZ_BORROW RzUtilStrScanOptions *scan_opts, size_t n_threads) {
|
||||
rz_return_val_if_fail(n_threads, NULL);
|
||||
StringSearch *ss = RZ_NEW0(StringSearch);
|
||||
if (!ss) {
|
||||
RZ_LOG_ERROR("search: failed to allocate StringSearch\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ss->strings = rz_pvector_new((RzPVectorFree)rz_detected_string_free);
|
||||
if (!ss->strings) {
|
||||
RZ_LOG_ERROR("search: failed to initialize string collection\n");
|
||||
string_free(ss);
|
||||
return NULL;
|
||||
ss->strings = rz_pvector_new((RzPVectorFree)rz_pvector_free);
|
||||
ss->thread_ids = rz_th_queue_new(n_threads, NULL);
|
||||
if (!ss->strings || !ss->thread_ids) {
|
||||
goto error;
|
||||
}
|
||||
for (size_t t = 0; t < n_threads; ++t) {
|
||||
RzPVector *thread_unique_str = rz_pvector_new((RzPVectorFree)rz_detected_string_free);
|
||||
if (!thread_unique_str) {
|
||||
goto error;
|
||||
}
|
||||
size_t *t_ptr = RZ_NEW(size_t);
|
||||
if (!t_ptr) {
|
||||
goto error;
|
||||
}
|
||||
*t_ptr = t;
|
||||
if (!rz_th_queue_push(ss->thread_ids, t_ptr, true)) {
|
||||
goto error;
|
||||
}
|
||||
rz_pvector_push(ss->strings, thread_unique_str);
|
||||
}
|
||||
|
||||
if (scan_opts) {
|
||||
|
|
@ -227,6 +391,11 @@ RZ_API RZ_OWN RzSearchCollection *rz_search_collection_strings(RZ_NULLABLE RZ_BO
|
|||
}
|
||||
|
||||
return rz_search_collection_new_bytes_space(string_find, string_is_empty, string_free, ss);
|
||||
|
||||
error:
|
||||
RZ_LOG_ERROR("search: failed to initialize string collection\n");
|
||||
string_free(ss);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static RzDetectedString *setup_str_regex(const char *re_pattern, RzRegexFlags cflags, size_t match_alignment, RzStrEnc encoding) {
|
||||
|
|
@ -236,9 +405,8 @@ static RzDetectedString *setup_str_regex(const char *re_pattern, RzRegexFlags cf
|
|||
return NULL;
|
||||
}
|
||||
|
||||
bool code_point_matches_alignment = rz_string_code_points_align(encoding, match_alignment);
|
||||
RzRegexMulti *re;
|
||||
if (rz_string_enc_is_utf_native_endian(encoding) && code_point_matches_alignment) {
|
||||
if (rz_string_enc_is_utf(encoding)) {
|
||||
switch (encoding) {
|
||||
default:
|
||||
rz_warn_if_reached();
|
||||
|
|
@ -299,11 +467,10 @@ RZ_API bool rz_search_collection_string_add(
|
|||
RzStrEnc encoding) {
|
||||
rz_return_val_if_fail(col && regex_pattern, false);
|
||||
StringSearch *ss = (StringSearch *)col->user;
|
||||
if ((!rz_string_enc_is_utf_native_endian(encoding) ||
|
||||
!rz_string_code_points_align(encoding, match_alignment)) &&
|
||||
ss->options.max_str_length == 0) {
|
||||
if ((rz_search_str_enc_needs_scanning(encoding) &&
|
||||
ss->options.max_str_length == 0)) {
|
||||
RZ_LOG_ERROR("Cannot add pattern to collection: RzUtilStrScanOptions::max_str_length == 0."
|
||||
"This is not allowed if the serached encoding requires string scanning.\n");
|
||||
"This is not allowed if the searched encoding requires string scanning.\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -315,14 +482,18 @@ RZ_API bool rz_search_collection_string_add(
|
|||
return false;
|
||||
}
|
||||
|
||||
RzDetectedString *s = setup_str_regex(regex_pattern, cflags, match_alignment, encoding);
|
||||
if (!s) {
|
||||
return false;
|
||||
}
|
||||
if (!rz_pvector_push(ss->strings, s)) {
|
||||
RZ_LOG_ERROR("search: cannot add the string '%s'.\n", regex_pattern);
|
||||
rz_detected_string_free(s);
|
||||
return false;
|
||||
void **it;
|
||||
rz_pvector_foreach (ss->strings, it) {
|
||||
RzPVector *thread_strings = *it;
|
||||
RzDetectedString *s = setup_str_regex(regex_pattern, cflags, match_alignment, encoding);
|
||||
if (!s) {
|
||||
return false;
|
||||
}
|
||||
if (!rz_pvector_push(thread_strings, s)) {
|
||||
RZ_LOG_ERROR("search: cannot add the string '%s'.\n", regex_pattern);
|
||||
rz_detected_string_free(s);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
|
@ -350,23 +521,28 @@ RZ_API bool rz_search_collection_strings_check_config_improvements(
|
|||
StringSearch *ss = col->user;
|
||||
void **it;
|
||||
rz_pvector_foreach (ss->strings, it) {
|
||||
RzDetectedString *ds = *it;
|
||||
if (ds->encoding == RZ_STRING_ENC_GUESS) {
|
||||
if (log_suggestions) {
|
||||
RZ_LOG_WARN("The string encoding for the search is set to \"guess\".\n"
|
||||
"The search will consume vastly more resources and the guessing is unreliable.\n"
|
||||
"You can set a specific encoding with 'e str.encoding=<encoding>'.\n");
|
||||
void **itt;
|
||||
RzPVector *thread_unique = *it;
|
||||
rz_pvector_foreach (thread_unique, itt) {
|
||||
RzDetectedString *ds = *itt;
|
||||
if (ds->encoding == RZ_STRING_ENC_GUESS) {
|
||||
if (log_suggestions) {
|
||||
RZ_LOG_WARN("The string encoding for the search is set to \"guess\".\n"
|
||||
"The search will consume vastly more resources and the guessing is unreliable.\n"
|
||||
"You can set a specific encoding with 'e str.encoding=<encoding>'.\n");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (!rz_string_code_points_align(ds->encoding, search_options->find_opts->alignment)) {
|
||||
if (log_suggestions) {
|
||||
RZ_LOG_INFO("The string encoding has code points of more than 1 byte. But search.align is set to 1.\n"
|
||||
"The search will consume more resources, because alignment is not a multiple of the code point size.\n"
|
||||
"For larger binaries consider to change the encoding to a multiple of 2 (UTF-16) or 4 (UTF-32).\n");
|
||||
if (!rz_string_code_points_align(ds->encoding, search_options->find_opts->alignment)) {
|
||||
if (log_suggestions) {
|
||||
RZ_LOG_INFO("The string encoding has code points of more than 1 byte. But search.align is set to 1.\n"
|
||||
"The search will consume more resources, because alignment is not a multiple of the code point size.\n"
|
||||
"For larger binaries consider to change the encoding to a multiple of 2 (UTF-16) or 4 (UTF-32).\n");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,9 @@
|
|||
#include <rz_util.h>
|
||||
#include <rz_types.h>
|
||||
|
||||
#define RZ_REGEX_JIT_STACK_MIN (512 * 1024)
|
||||
#define RZ_REGEX_JIT_STACK_MAX (1024 * 1024)
|
||||
|
||||
typedef pcre2_general_context_8 RzRegexGeneralContext8; ///< General context.
|
||||
// typedef pcre2_compile_context RzRegexCompContext; ///< The context for compiling.
|
||||
typedef pcre2_match_context_8 RzRegexMatchContext8; ///< The context for matching.
|
||||
|
|
@ -90,12 +93,13 @@ static RZ_OWN void *regex_new(RZ_NONNULL const char *pattern, RzRegexFlags cflag
|
|||
}
|
||||
#ifdef SUPPORTS_PCRE2_JIT
|
||||
RzRegexStatus jit_err = pcre2_jit_compile_8(regex, jflags);
|
||||
|
||||
if (jit_err < 0) {
|
||||
print_pcre2_err(pat, jit_err, 0);
|
||||
}
|
||||
#endif
|
||||
} else if (pcre2_word_width == 16) {
|
||||
ut16 *utf16_pat = rz_str_utf8_to_utf16(pat, false);
|
||||
ut16 *utf16_pat = rz_str_utf8_to_utf16(pat, RZ_HOST_IS_BIG_ENDIAN);
|
||||
regex = pcre2_compile_16(
|
||||
(PCRE2_SPTR16)utf16_pat,
|
||||
PCRE2_ZERO_TERMINATED,
|
||||
|
|
@ -259,6 +263,27 @@ RZ_API RZ_OWN RzRegexMulti *rz_regex_new_multi(RZ_NONNULL const char *pattern, R
|
|||
free(re);
|
||||
return NULL;
|
||||
}
|
||||
#ifdef SUPPORTS_PCRE2_JIT
|
||||
switch (type) {
|
||||
default:
|
||||
rz_warn_if_reached();
|
||||
return NULL;
|
||||
case RZ_REGEX_UTF8:
|
||||
re->jit_stack = pcre2_jit_stack_create_8(RZ_REGEX_JIT_STACK_MIN, RZ_REGEX_JIT_STACK_MAX, NULL);
|
||||
break;
|
||||
case RZ_REGEX_UTF16:
|
||||
re->jit_stack = pcre2_jit_stack_create_16(RZ_REGEX_JIT_STACK_MIN, RZ_REGEX_JIT_STACK_MAX, NULL);
|
||||
break;
|
||||
case RZ_REGEX_UTF32:
|
||||
re->jit_stack = pcre2_jit_stack_create_32(RZ_REGEX_JIT_STACK_MIN, RZ_REGEX_JIT_STACK_MAX, NULL);
|
||||
break;
|
||||
}
|
||||
if (!re->jit_stack) {
|
||||
rz_warn_if_reached();
|
||||
rz_regex_free_multi(re);
|
||||
return NULL;
|
||||
}
|
||||
#endif
|
||||
return re;
|
||||
}
|
||||
|
||||
|
|
@ -434,6 +459,19 @@ RZ_API void rz_regex_free_multi(RZ_NULLABLE RZ_OWN RzRegexMulti *regex_multi) {
|
|||
rz_regex_free_32(regex_multi->re32);
|
||||
break;
|
||||
}
|
||||
#ifdef SUPPORTS_PCRE2_JIT
|
||||
switch (regex_multi->re_type) {
|
||||
case RZ_REGEX_UTF8:
|
||||
pcre2_jit_stack_free_8(regex_multi->jit_stack);
|
||||
break;
|
||||
case RZ_REGEX_UTF16:
|
||||
pcre2_jit_stack_free_16(regex_multi->jit_stack);
|
||||
break;
|
||||
case RZ_REGEX_UTF32:
|
||||
pcre2_jit_stack_free_32(regex_multi->jit_stack);
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
free(regex_multi);
|
||||
}
|
||||
|
||||
|
|
@ -613,14 +651,26 @@ static RZ_OWN RzPVector /*<RzRegexMatch *>*/ *match_first_8(
|
|||
RZ_NONNULL const ut8 *text,
|
||||
RzRegexSize text_size,
|
||||
RzRegexSize text_offset,
|
||||
RzRegexFlags mflags) {
|
||||
RzRegexFlags mflags,
|
||||
RZ_NULLABLE void *jit_stack) {
|
||||
rz_return_val_if_fail(regex && text, NULL);
|
||||
|
||||
RzPVector *matches = NULL;
|
||||
RzRegexMatchData *mdata = NULL;
|
||||
mdata = pcre2_match_data_create_from_pattern_8(regex, NULL);
|
||||
|
||||
pcre2_match_context_8 *mcontext = NULL;
|
||||
#ifdef SUPPORTS_PCRE2_JIT
|
||||
if (jit_stack) {
|
||||
mcontext = pcre2_match_context_create_8(NULL);
|
||||
if (!mcontext) {
|
||||
goto fini;
|
||||
}
|
||||
pcre2_jit_stack_assign_8(mcontext, NULL, jit_stack);
|
||||
}
|
||||
#endif
|
||||
RzRegexStatus rc = 0;
|
||||
rc = pcre2_match_8(regex, (PCRE2_SPTR8)text, text_size, text_offset, mflags | PCRE2_NO_UTF_CHECK, mdata, NULL);
|
||||
rc = pcre2_match_8(regex, (PCRE2_SPTR8)text, text_size, text_offset, mflags | PCRE2_NO_UTF_CHECK, mdata, mcontext);
|
||||
|
||||
if (rc == PCRE2_ERROR_NOMATCH) {
|
||||
// Nothing matched return empty vector.
|
||||
|
|
@ -680,14 +730,26 @@ static RZ_OWN RzPVector /*<RzRegexMatch *>*/ *match_first_16(
|
|||
RZ_NONNULL const ut16 *text,
|
||||
RzRegexSize text_size_code_units,
|
||||
RzRegexSize text_offset_code_units,
|
||||
RzRegexFlags mflags) {
|
||||
RzRegexFlags mflags,
|
||||
RZ_NULLABLE void *jit_stack) {
|
||||
rz_return_val_if_fail(regex && text, NULL);
|
||||
|
||||
RzPVector *matches = NULL;
|
||||
RzRegexMatchData *mdata = NULL;
|
||||
mdata = pcre2_match_data_create_from_pattern_16(regex, NULL);
|
||||
|
||||
pcre2_match_context_16 *mcontext = NULL;
|
||||
#ifdef SUPPORTS_PCRE2_JIT
|
||||
if (jit_stack) {
|
||||
mcontext = pcre2_match_context_create_16(NULL);
|
||||
if (!mcontext) {
|
||||
goto fini;
|
||||
}
|
||||
pcre2_jit_stack_assign_16(mcontext, NULL, jit_stack);
|
||||
}
|
||||
#endif
|
||||
RzRegexStatus rc = 0;
|
||||
rc = pcre2_match_16(regex, (PCRE2_SPTR16)text, text_size_code_units, text_offset_code_units, mflags | PCRE2_NO_UTF_CHECK, mdata, NULL);
|
||||
rc = pcre2_match_16(regex, (PCRE2_SPTR16)text, text_size_code_units, text_offset_code_units, mflags | PCRE2_NO_UTF_CHECK, mdata, mcontext);
|
||||
|
||||
if (rc == PCRE2_ERROR_NOMATCH) {
|
||||
// Nothing matched, return an empty vector
|
||||
|
|
@ -747,14 +809,26 @@ static RZ_OWN RzPVector /*<RzRegexMatch *>*/ *match_first_32(
|
|||
RZ_NONNULL const ut32 *text,
|
||||
RzRegexSize text_size_code_units,
|
||||
RzRegexSize text_offset_code_units,
|
||||
RzRegexFlags mflags) {
|
||||
RzRegexFlags mflags,
|
||||
RZ_NULLABLE void *jit_stack) {
|
||||
rz_return_val_if_fail(regex && text, NULL);
|
||||
|
||||
RzPVector *matches = NULL;
|
||||
RzRegexMatchData *mdata = NULL;
|
||||
mdata = pcre2_match_data_create_from_pattern_32(regex, NULL);
|
||||
|
||||
pcre2_match_context_32 *mcontext = NULL;
|
||||
#ifdef SUPPORTS_PCRE2_JIT
|
||||
if (jit_stack) {
|
||||
mcontext = pcre2_match_context_create_32(NULL);
|
||||
if (!mcontext) {
|
||||
goto fini;
|
||||
}
|
||||
pcre2_jit_stack_assign_32(mcontext, NULL, jit_stack);
|
||||
}
|
||||
#endif
|
||||
RzRegexStatus rc = 0;
|
||||
rc = pcre2_match_32(regex, (PCRE2_SPTR32)text, text_size_code_units, text_offset_code_units, mflags | PCRE2_NO_UTF_CHECK, mdata, NULL);
|
||||
rc = pcre2_match_32(regex, (PCRE2_SPTR32)text, text_size_code_units, text_offset_code_units, mflags | PCRE2_NO_UTF_CHECK, mdata, mcontext);
|
||||
|
||||
if (rc == PCRE2_ERROR_NOMATCH) {
|
||||
// Nothing matched return empty vector.
|
||||
|
|
@ -834,7 +908,7 @@ RZ_API RZ_OWN RzPVector /*<RzRegexMatch *>*/ *rz_regex_match_first(
|
|||
(ut8 *)text,
|
||||
text_size,
|
||||
text_offset,
|
||||
mflags);
|
||||
mflags, NULL);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -864,7 +938,7 @@ RZ_API RZ_OWN RzPVector /*<RzRegexMatch *>*/ *rz_regex_match_first_16(
|
|||
text,
|
||||
text_size,
|
||||
text_offset,
|
||||
mflags);
|
||||
mflags, NULL);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -894,7 +968,7 @@ RZ_API RZ_OWN RzPVector /*<RzRegexMatch *>*/ *rz_regex_match_first_32(
|
|||
text,
|
||||
text_size,
|
||||
text_offset,
|
||||
mflags);
|
||||
mflags, NULL);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -947,18 +1021,19 @@ static RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *match_all_internal_8(
|
|||
RzRegexSize text_size,
|
||||
RzRegexSize text_offset,
|
||||
RzRegexFlags mflags,
|
||||
bool allow_overlap) {
|
||||
bool allow_overlap,
|
||||
RZ_NULLABLE void *jit_stack) {
|
||||
rz_return_val_if_fail(regex && text, NULL);
|
||||
|
||||
RzPVector *all_matches = rz_pvector_new((RzPVectorFree)rz_pvector_free);
|
||||
RzPVector *matches = NULL;
|
||||
matches = match_first_8(regex, text, text_size, text_offset, mflags);
|
||||
matches = match_first_8(regex, text, text_size, text_offset, mflags, jit_stack);
|
||||
while (matches && rz_pvector_len(matches) > 0) {
|
||||
rz_pvector_push(all_matches, matches);
|
||||
RzRegexMatch *m = rz_pvector_head(matches);
|
||||
// Search again after the last match.
|
||||
text_offset = allow_overlap ? m->start + 1 : m->start + m->len;
|
||||
matches = match_first_8(regex, text, text_size, text_offset, mflags);
|
||||
matches = match_first_8(regex, text, text_size, text_offset, mflags, jit_stack);
|
||||
}
|
||||
|
||||
// Free last vector without matches.
|
||||
|
|
@ -972,18 +1047,19 @@ static RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *match_all_internal_16(
|
|||
RzRegexSize text_size_code_units,
|
||||
RzRegexSize text_offset_code_units,
|
||||
RzRegexFlags mflags,
|
||||
bool allow_overlap) {
|
||||
bool allow_overlap,
|
||||
RZ_NULLABLE void *jit_stack) {
|
||||
rz_return_val_if_fail(regex && text, NULL);
|
||||
|
||||
RzPVector *all_matches = rz_pvector_new((RzPVectorFree)rz_pvector_free);
|
||||
RzPVector *matches = NULL;
|
||||
matches = match_first_16(regex, text, text_size_code_units, text_offset_code_units, mflags);
|
||||
matches = match_first_16(regex, text, text_size_code_units, text_offset_code_units, mflags, jit_stack);
|
||||
while (matches && rz_pvector_len(matches) > 0) {
|
||||
rz_pvector_push(all_matches, matches);
|
||||
RzRegexMatch *m = rz_pvector_head(matches);
|
||||
// Search again after the last match.
|
||||
text_offset_code_units = allow_overlap ? m->start + 1 : m->start + m->len;
|
||||
matches = match_first_16(regex, text, text_size_code_units, text_offset_code_units, mflags);
|
||||
matches = match_first_16(regex, text, text_size_code_units, text_offset_code_units, mflags, jit_stack);
|
||||
}
|
||||
|
||||
// Free last vector without matches.
|
||||
|
|
@ -997,18 +1073,19 @@ static RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *match_all_internal_32(
|
|||
RzRegexSize text_size_code_units,
|
||||
RzRegexSize text_offset_code_units,
|
||||
RzRegexFlags mflags,
|
||||
bool allow_overlap) {
|
||||
bool allow_overlap,
|
||||
RZ_NULLABLE void *jit_stack) {
|
||||
rz_return_val_if_fail(regex && text, NULL);
|
||||
|
||||
RzPVector *all_matches = rz_pvector_new((RzPVectorFree)rz_pvector_free);
|
||||
RzPVector *matches = NULL;
|
||||
matches = match_first_32(regex, text, text_size_code_units, text_offset_code_units, mflags);
|
||||
matches = match_first_32(regex, text, text_size_code_units, text_offset_code_units, mflags, jit_stack);
|
||||
while (matches && rz_pvector_len(matches) > 0) {
|
||||
rz_pvector_push(all_matches, matches);
|
||||
RzRegexMatch *m = rz_pvector_head(matches);
|
||||
// Search again after the last match.
|
||||
text_offset_code_units = allow_overlap ? m->start + 1 : m->start + m->len;
|
||||
matches = match_first_32(regex, text, text_size_code_units, text_offset_code_units, mflags);
|
||||
matches = match_first_32(regex, text, text_size_code_units, text_offset_code_units, mflags, jit_stack);
|
||||
}
|
||||
|
||||
// Free last vector without matches.
|
||||
|
|
@ -1036,7 +1113,7 @@ RZ_API RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *rz_regex_match_all_ove
|
|||
RzRegexSize text_size,
|
||||
RzRegexSize text_offset,
|
||||
RzRegexFlags mflags) {
|
||||
return match_all_internal_8(regex, (ut8 *)text, text_size, text_offset, mflags, true);
|
||||
return match_all_internal_8(regex, (ut8 *)text, text_size, text_offset, mflags, true, NULL);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1058,7 +1135,7 @@ RZ_API RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *rz_regex_match_all(
|
|||
RzRegexSize text_size,
|
||||
RzRegexSize text_offset,
|
||||
RzRegexFlags mflags) {
|
||||
return match_all_internal_8(regex, (ut8 *)text, text_size, text_offset, mflags, false);
|
||||
return match_all_internal_8(regex, (ut8 *)text, text_size, text_offset, mflags, false, NULL);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1083,7 +1160,7 @@ RZ_API RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *rz_regex_match_all_ove
|
|||
RzRegexSize text_size,
|
||||
RzRegexSize text_offset,
|
||||
RzRegexFlags mflags) {
|
||||
return match_all_internal_16(regex, text, text_size, text_offset, mflags, true);
|
||||
return match_all_internal_16(regex, text, text_size, text_offset, mflags, true, NULL);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1107,7 +1184,7 @@ RZ_API RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *rz_regex_match_all_16(
|
|||
RzRegexSize text_size,
|
||||
RzRegexSize text_offset,
|
||||
RzRegexFlags mflags) {
|
||||
return match_all_internal_16(regex, text, text_size, text_offset, mflags, false);
|
||||
return match_all_internal_16(regex, text, text_size, text_offset, mflags, false, NULL);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1132,7 +1209,7 @@ RZ_API RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *rz_regex_match_all_ove
|
|||
RzRegexSize text_size,
|
||||
RzRegexSize text_offset,
|
||||
RzRegexFlags mflags) {
|
||||
return match_all_internal_32(regex, text, text_size, text_offset, mflags, true);
|
||||
return match_all_internal_32(regex, text, text_size, text_offset, mflags, true, NULL);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1146,11 +1223,11 @@ RZ_API RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *rz_regex_match_all_ove
|
|||
RzRegexFlags mflags) {
|
||||
switch (regex->re_type) {
|
||||
case RZ_REGEX_UTF8:
|
||||
return match_all_internal_8(regex->re8, (ut8 *)text, text_size, text_offset, mflags, true);
|
||||
return match_all_internal_8(regex->re8, (ut8 *)text, text_size, text_offset, mflags, true, regex->jit_stack);
|
||||
case RZ_REGEX_UTF16:
|
||||
return match_all_internal_16(regex->re16, (ut16 *)text, text_size / RZ_UTF16_CODE_POINT_WIDTH, text_offset / RZ_UTF16_CODE_POINT_WIDTH, mflags, true);
|
||||
return match_all_internal_16(regex->re16, (ut16 *)text, text_size / RZ_UTF16_CODE_POINT_WIDTH, text_offset / RZ_UTF16_CODE_POINT_WIDTH, mflags, true, regex->jit_stack);
|
||||
case RZ_REGEX_UTF32:
|
||||
return match_all_internal_32(regex->re32, (ut32 *)text, text_size / RZ_UTF32_CODE_POINT_WIDTH, text_offset / RZ_UTF32_CODE_POINT_WIDTH, mflags, true);
|
||||
return match_all_internal_32(regex->re32, (ut32 *)text, text_size / RZ_UTF32_CODE_POINT_WIDTH, text_offset / RZ_UTF32_CODE_POINT_WIDTH, mflags, true, regex->jit_stack);
|
||||
}
|
||||
rz_warn_if_reached();
|
||||
return NULL;
|
||||
|
|
@ -1164,11 +1241,11 @@ RZ_API RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *rz_regex_match_all_mul
|
|||
RzRegexFlags mflags) {
|
||||
switch (regex->re_type) {
|
||||
case RZ_REGEX_UTF8:
|
||||
return match_all_internal_8(regex->re8, (ut8 *)text, text_size, text_offset, mflags, false);
|
||||
return match_all_internal_8(regex->re8, (ut8 *)text, text_size, text_offset, mflags, false, regex->jit_stack);
|
||||
case RZ_REGEX_UTF16:
|
||||
return match_all_internal_16(regex->re16, (ut16 *)text, text_size / RZ_UTF16_CODE_POINT_WIDTH, text_offset / RZ_UTF16_CODE_POINT_WIDTH, mflags, false);
|
||||
return match_all_internal_16(regex->re16, (ut16 *)text, text_size / RZ_UTF16_CODE_POINT_WIDTH, text_offset / RZ_UTF16_CODE_POINT_WIDTH, mflags, false, regex->jit_stack);
|
||||
case RZ_REGEX_UTF32:
|
||||
return match_all_internal_32(regex->re32, (ut32 *)text, text_size / RZ_UTF32_CODE_POINT_WIDTH, text_offset / RZ_UTF32_CODE_POINT_WIDTH, mflags, false);
|
||||
return match_all_internal_32(regex->re32, (ut32 *)text, text_size / RZ_UTF32_CODE_POINT_WIDTH, text_offset / RZ_UTF32_CODE_POINT_WIDTH, mflags, false, regex->jit_stack);
|
||||
}
|
||||
rz_warn_if_reached();
|
||||
return NULL;
|
||||
|
|
@ -1195,7 +1272,7 @@ RZ_API RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *rz_regex_match_all_32(
|
|||
RzRegexSize text_size,
|
||||
RzRegexSize text_offset,
|
||||
RzRegexFlags mflags) {
|
||||
return match_all_internal_32(regex, text, text_size, text_offset, mflags, false);
|
||||
return match_all_internal_32(regex, text, text_size, text_offset, mflags, false, NULL);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -4472,6 +4472,28 @@ RZ_API bool rz_string_enc_is_utf_native_endian(RzStrEnc enc) {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Checks given encoding if it is UTF-8, UTF-16, or UTF-32.
|
||||
*
|
||||
* \return true For UTF-8/ASCII.
|
||||
* \return true For UTF-16-LE/UTF-32-LE.
|
||||
* \return true For UTF-16-BE/UTF-32-BE.
|
||||
* \return false Otherwise.
|
||||
*/
|
||||
RZ_API bool rz_string_enc_is_utf(RzStrEnc enc) {
|
||||
switch (enc) {
|
||||
default:
|
||||
return false;
|
||||
case RZ_STRING_ENC_8BIT:
|
||||
case RZ_STRING_ENC_UTF8:
|
||||
case RZ_STRING_ENC_UTF16LE:
|
||||
case RZ_STRING_ENC_UTF32LE:
|
||||
case RZ_STRING_ENC_UTF16BE:
|
||||
case RZ_STRING_ENC_UTF32BE:
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Returns the size of the code point in bytes.
|
||||
* UTF-8 = 1, UTF-16 = 2, UTF-32 = 4 etc.
|
||||
|
|
@ -4483,6 +4505,7 @@ RZ_API size_t rz_string_enc_code_point_width(RzStrEnc enc) {
|
|||
default:
|
||||
case RZ_STRING_ENC_GUESS:
|
||||
case RZ_STRING_ENC_SETTINGS:
|
||||
rz_warn_if_reached();
|
||||
return 0;
|
||||
case RZ_STRING_ENC_8BIT:
|
||||
case RZ_STRING_ENC_UTF8:
|
||||
|
|
|
|||
|
|
@ -1440,13 +1440,8 @@ EXPECT_ERR=
|
|||
RUN
|
||||
|
||||
NAME=String Search - Encoding: utf16be - Armenian - Fast on BE systems
|
||||
BROKEN=1
|
||||
FILE=bins/cmd/search/string_encodings/Armenian-Lipsum.utf16be
|
||||
CMDS=<<EOF
|
||||
# This test fails on S390x (big-endian) systems for whatever reason.
|
||||
* Marked as broken. Tracked in https://github.com/rizinorg/rizin/issues/5369
|
||||
#
|
||||
# Align to UTF16 code point width
|
||||
e search.align=2
|
||||
e str.encoding=utf16be
|
||||
/z "դեֆինիթիոնես ին վիս, ծասե պեռթինա" l utf16be
|
||||
|
|
|
|||
|
|
@ -38,10 +38,6 @@ static const char *files[] = {
|
|||
//
|
||||
// sha256: dc365472d8bbfdc3a3d47b5a0d8061c7d18233b131b0ed12fe599b53248629b2
|
||||
// "test/bins/test/bins/test_strings_zh.utf-32-le",
|
||||
//
|
||||
// Big endian strings to search. Search will significantly slower on little endian machines.
|
||||
// sha256: 315e96099d4c0ad7501e47f28a7781b8ed9fef0902bce5e962276a285362a684
|
||||
// "test/bins/test/bins/test_strings_zh.utf-16-be",
|
||||
};
|
||||
|
||||
// Patterns/strings to search in the files from above.
|
||||
|
|
@ -65,6 +61,8 @@ static const char *patterns[][3] = {
|
|||
// Always check RzSearchHit->size for the real byte length of a string.
|
||||
#define ELEMENT_SIZE 50
|
||||
|
||||
#define N_THREADS 4
|
||||
|
||||
/**
|
||||
* \brief Do a simple literal and regex search for strings in Hindi.
|
||||
*/
|
||||
|
|
@ -79,7 +77,7 @@ int test_rz_str_search_single_simple(void) {
|
|||
// Configuring specific values is optional.
|
||||
RzSearchOpt *search_opts = rz_search_opt_new();
|
||||
mu_assert_notnull(search_opts, "NULL check failed");
|
||||
rz_search_opt_set_max_threads(search_opts, 4);
|
||||
rz_search_opt_set_max_threads(search_opts, N_THREADS);
|
||||
rz_search_opt_set_max_hits(search_opts, 10);
|
||||
rz_search_opt_set_show_progress_from_str(search_opts, "no");
|
||||
rz_search_opt_set_chunk_size(search_opts, ELEMENT_SIZE);
|
||||
|
|
@ -100,7 +98,7 @@ int test_rz_str_search_single_simple(void) {
|
|||
// We can pass NULL here to the RzUtilStrScanOptions parameter,
|
||||
// because UTF-8 is endianness independent and can directly match the buffer with PCRE2.
|
||||
// No scanning for strings is required. Hence we don't need the options for it.
|
||||
RzSearchCollection *collection = rz_search_collection_strings(NULL);
|
||||
RzSearchCollection *collection = rz_search_collection_strings(NULL, N_THREADS);
|
||||
mu_assert_notnull(collection, "NULL check failed");
|
||||
|
||||
// Now add the two patterns we search for
|
||||
|
|
@ -142,7 +140,7 @@ int test_rz_str_search_io_simple(void) {
|
|||
// Configuring specific values is optional.
|
||||
RzSearchOpt *search_opts = rz_search_opt_new();
|
||||
mu_assert_notnull(search_opts, "NULL check failed");
|
||||
rz_search_opt_set_max_threads(search_opts, 4);
|
||||
rz_search_opt_set_max_threads(search_opts, N_THREADS);
|
||||
rz_search_opt_set_max_hits(search_opts, 10);
|
||||
rz_search_opt_set_show_progress_from_str(search_opts, "no");
|
||||
rz_search_opt_set_chunk_size(search_opts, ELEMENT_SIZE);
|
||||
|
|
@ -153,8 +151,6 @@ int test_rz_str_search_io_simple(void) {
|
|||
|
||||
// Set alignment to 2, because we search UTF-16 and its code points are aligned to 2.
|
||||
// It is possible to also set it to any other value of course.
|
||||
// But any value not aligned to the code point width of UTF-16 (anything not a multiple of 2)
|
||||
// will slow down the search.
|
||||
// For details see librz/search/README.md
|
||||
size_t match_alignment = 2;
|
||||
rz_search_find_opt_set_alignment(find_opts, match_alignment);
|
||||
|
|
@ -163,16 +159,7 @@ int test_rz_str_search_io_simple(void) {
|
|||
// Assign find options to the search options.
|
||||
rz_search_opt_set_find_options(search_opts, find_opts);
|
||||
|
||||
// Please refer to librz/search/README.md for an explanation why string scan options
|
||||
// are needed for an UTF-16 search.
|
||||
RzUtilStrScanOptions scan_opt = {
|
||||
.max_str_length = ELEMENT_SIZE,
|
||||
.min_str_length = 4,
|
||||
.prefer_big_endian = false,
|
||||
.check_ascii_freq = false,
|
||||
};
|
||||
|
||||
RzSearchCollection *collection = rz_search_collection_strings(&scan_opt);
|
||||
RzSearchCollection *collection = rz_search_collection_strings(NULL, N_THREADS);
|
||||
mu_assert_notnull(collection, "NULL check failed");
|
||||
|
||||
// Now add the pattern we search for.
|
||||
|
|
@ -221,7 +208,7 @@ int test_rz_str_search_multiple_enc(void) {
|
|||
// Configuring specific values is optional.
|
||||
RzSearchOpt *search_opts = rz_search_opt_new();
|
||||
mu_assert_notnull(search_opts, "NULL check failed");
|
||||
rz_search_opt_set_max_threads(search_opts, 4);
|
||||
rz_search_opt_set_max_threads(search_opts, N_THREADS);
|
||||
rz_search_opt_set_max_hits(search_opts, 10);
|
||||
rz_search_opt_set_show_progress_from_str(search_opts, "no");
|
||||
rz_search_opt_set_chunk_size(search_opts, ELEMENT_SIZE);
|
||||
|
|
@ -239,16 +226,7 @@ int test_rz_str_search_multiple_enc(void) {
|
|||
// Assign find options to the search options.
|
||||
rz_search_opt_set_find_options(search_opts, find_opts);
|
||||
|
||||
// Please refer to librz/search/README.md for an explanation why string scan options
|
||||
// are needed for an UTF-16 search.
|
||||
RzUtilStrScanOptions scan_opt = {
|
||||
.max_str_length = ELEMENT_SIZE,
|
||||
.min_str_length = 5,
|
||||
.prefer_big_endian = true,
|
||||
.check_ascii_freq = false,
|
||||
};
|
||||
|
||||
RzSearchCollection *collection = rz_search_collection_strings(&scan_opt);
|
||||
RzSearchCollection *collection = rz_search_collection_strings(NULL, N_THREADS);
|
||||
mu_assert_notnull(collection, "NULL check failed");
|
||||
|
||||
// Now add the patterns we search for. One for each encoding in the file.
|
||||
|
|
|
|||
|
|
@ -430,17 +430,18 @@ bool test_rz_scan_strings_detect_length_utf8(void) {
|
|||
* the search requires string scanning, but no scan optoins were given.
|
||||
*/
|
||||
bool test_rz_scan_strings_scan_options_error(void) {
|
||||
RzSearchCollection *collection = rz_search_collection_strings(NULL);
|
||||
mu_assert_false(rz_search_collection_string_add(collection, "", 0, 1, RZ_STRING_ENC_UTF16BE), "Should fail for this config");
|
||||
mu_assert_false(rz_search_collection_string_add(collection, "", 0, 1, RZ_STRING_ENC_UTF16LE), "Should fail for this config");
|
||||
mu_assert_false(rz_search_collection_string_add(collection, "", 0, 1, RZ_STRING_ENC_UTF32BE), "Should fail for this config");
|
||||
mu_assert_false(rz_search_collection_string_add(collection, "", 0, 1, RZ_STRING_ENC_UTF32LE), "Should fail for this config");
|
||||
RzSearchCollection *collection = rz_search_collection_strings(NULL, 4);
|
||||
mu_assert_false(rz_search_collection_string_add(collection, "", 0, 1, RZ_STRING_ENC_IBM037), "Should fail for this config");
|
||||
mu_assert_false(rz_search_collection_string_add(collection, "", 0, 1, RZ_STRING_ENC_IBM290), "Should fail for this config");
|
||||
mu_assert_false(rz_search_collection_string_add(collection, "", 0, 1, RZ_STRING_ENC_EBCDIC_UK), "Should fail for this config");
|
||||
mu_assert_false(rz_search_collection_string_add(collection, "", 0, 1, RZ_STRING_ENC_EBCDIC_US), "Should fail for this config");
|
||||
mu_assert_false(rz_search_collection_string_add(collection, "", 0, 1, RZ_STRING_ENC_EBCDIC_ES), "Should fail for this config");
|
||||
mu_assert_false(rz_search_collection_string_add(collection, "", 0, 1, RZ_STRING_ENC_GUESS), "Should fail for this config");
|
||||
mu_assert_false(rz_search_collection_string_add(collection, "", 0, 1, RZ_STRING_ENC_SETTINGS), "Should fail for this config");
|
||||
mu_assert_true(rz_search_collection_string_add(collection, "some_pattern", 0, 1, RZ_STRING_ENC_UTF16BE), "Should succeed for this config");
|
||||
mu_assert_true(rz_search_collection_string_add(collection, "some_pattern", 0, 1, RZ_STRING_ENC_UTF16LE), "Should succeed for this config");
|
||||
mu_assert_true(rz_search_collection_string_add(collection, "some_pattern", 0, 1, RZ_STRING_ENC_UTF32BE), "Should succeed for this config");
|
||||
mu_assert_true(rz_search_collection_string_add(collection, "some_pattern", 0, 1, RZ_STRING_ENC_UTF32LE), "Should succeed for this config");
|
||||
mu_assert_true(rz_search_collection_string_add(collection, "some_pattern", 0, 1, RZ_STRING_ENC_8BIT), "Should succeed for this config");
|
||||
mu_assert_true(rz_search_collection_string_add(collection, "some_pattern", 0, 1, RZ_STRING_ENC_UTF8), "Should succeed for this config");
|
||||
rz_search_collection_free(collection);
|
||||
|
|
|
|||
Loading…
Reference in a new issue