rizin/librz/search/keyword.c
Rot127 699c7d2dff
Improve performance of string search. (#5262)
* Make checks against defined Unicode points optional

* Allow to decode UTF-16 without writing the result.

* Remove PCRE2_NO_UTF_CHECK as default, since it can lead to undefined behavior.

* First refactor regex to support utf16 and utf32

* Add UTF-16-BE encoding function

* Add UTF-8 counting helper functions.

- One for counting the number of Unicode code points.
- The other to get the number of bytes required to represent the given UTF-8 string in UTF-16.

* Remove unused code

* Add UTF-8 to UTF-16 conversion function.

* Add type annotations

* Implement utf8 to utf32 string conversion

* Add UTF-16/32 versions of all other necessary regex functions for str search.

* Another regex refactor for utf16/32

* Add UTF16/32 regex matching tests.

* Implement still segfaulting (possibly JIT double usage) regex search.

* Duplicate match_first functions to reduce necessary branch predictions.

* Reduce number of required branches for encoding UTF16/32 to one.

* Duplicate match_all_internal functions to reduce necessary branch predictions.

* Fix too early free

* Only allocate match vector when needed.

* Fix: use code point size of buffer.

* Add missing return

* Normalize pointers to UTF16/32 strings to use proper code point with

* Also replace spaces with in utf16/32

* Add an additional host endian tests for UTF16/32 string encodings.

* Ensure thread savety.

JIT compiled patterns need to be owned by a single thread.
For the search we need to clone it.

JIT matching structures are optionally cloned as well.

* Enforce NO_UTF_CHECK in regex search.

This improves performance and currently is
default because we always match on binary data.

* Fix matching of UTF strings which are not suported by direct buffer matching.

PCRE2 only supports matching against memory which is aligned
to a code point width of the encoding.

These changes prevent taking the fast (direct matching with PCRE2) path
and use the slow string search path if the alignment doesn't match
the UTF string encoding.

To not complicate the change and additional alignment member
is added to each searched string in the search collection.

* Create search hit description on the stack

* Unset complete JIT matching if user provided custom jflags.

* Document what passing NULL to copy function pointers does.

* Replace the retarded idea of tracking offsets with a hashmap with a linear buffer.

This improves performance something like 10x.

* Remove const for the non-JIT builds.

* Remove additional flags for skip checking.

It is not needed because each decoded character is checked for printablity below anyways.

* Enfore no setup of IO mem with 0xff

* Fix: Set JIT complete flag for multi regex patterns

* Fix heap.

* Add note about worsed performance path.

* Fix unit test with string terminated by undefined code point.

* Run clang-format

* Fix order of arguments

* Add warning about string search with encoding=guess to tests.

* Fix string lengths, they no longer count the final invalid code point.

* Fix endian macro on Windows

* Fix NULL dereference

* Fix number tests

* Use endianness check not dependent on stdbit

* Fix type annotations.

* Reintroduce rz_str_len_utf8char

* Fix command description.

* Fix and unify RZ_SYS_ENDIAN macros.

- Don't allow unhandled architectures anymore.
- Check endianness for Sparc and PPC using non GCC/Clang compilers.
- Fix several endianness checks using the value instead of the macros.

* Fix tests

* Apply review comments.
2025-09-11 22:29:20 +08:00

268 lines
5.8 KiB
C

// SPDX-FileCopyrightText: 2010-2015 pancake <pancake@nopcode.org>
// SPDX-License-Identifier: LGPL-3.0-only
#include <rz_search.h>
static int ignoreMask(const ut8 *bm, int len) {
int i;
for (i = 0; i < len; i++) {
if (bm[i] != 0xff) {
return 0;
}
}
return 1;
}
/**
* \brief Initializes a RzSearchKeyword with keyword bytes, optional bitmask and optional data.
*
* \param kw_buf Byte buffer of the keyword.
* \param kw_buf_len Length of \p kw_buf in bytes.
* \param bm_buf A bitmask. It is applied the bytes before comparing it to the keyword bytes.
* \param bm_buf_len Length of the bitmask buffer \p bm_buf in bytes.
* \param data Pointer to the data to search in.
*
* \return The initialized RzSearchKeyword or NULL in case of failure.
*/
RZ_API RZ_OWN RzSearchKeyword *rz_search_keyword_new(const ut8 *kw_buf, int kw_len, RZ_NULLABLE const ut8 *bm_buf, int bm_buf_len, RZ_NULLABLE const char *data) {
RzSearchKeyword *kw;
if (kw_len < 1 || bm_buf_len < 0) {
return NULL;
}
kw = RZ_NEW0(RzSearchKeyword);
if (!kw) {
return NULL;
}
kw->type = RZ_SEARCH_KEYWORD_TYPE_BINARY;
kw->data = (void *)data;
kw->keyword_length = kw_len;
kw->bin_keyword = malloc(kw_len);
if (!kw->bin_keyword) {
rz_search_keyword_free(kw);
return NULL;
}
memcpy(kw->bin_keyword, kw_buf, kw_len);
if (bm_buf && bm_buf_len > 0 && !ignoreMask(bm_buf, bm_buf_len)) {
kw->bin_binmask = malloc(bm_buf_len);
if (!kw->bin_binmask) {
rz_search_keyword_free(kw);
return NULL;
}
memcpy(kw->bin_binmask, bm_buf, bm_buf_len);
kw->binmask_length = bm_buf_len;
} else {
kw->bin_binmask = NULL;
kw->binmask_length = 0;
}
return kw;
}
RZ_API void rz_search_keyword_free(RzSearchKeyword *kw) {
if (!kw) {
return;
}
free(kw->bin_binmask);
free(kw->bin_keyword);
free(kw);
}
RZ_API RzSearchKeyword *rz_search_keyword_new_str(const char *kwbuf, const char *bmstr, const char *data, int ignore_case) {
RzSearchKeyword *kw;
ut8 *bmbuf = NULL;
int bmlen = 0;
if (bmstr) {
bmbuf = malloc(strlen(bmstr) + 1);
if (!bmbuf) {
return NULL;
}
bmlen = rz_hex_str2bin(bmstr, bmbuf);
if (bmlen < 1) {
RZ_FREE(bmbuf);
}
}
kw = rz_search_keyword_new((ut8 *)kwbuf, strlen(kwbuf), bmbuf, bmlen, data);
if (kw) {
kw->icase = ignore_case;
kw->type = RZ_SEARCH_KEYWORD_TYPE_STRING;
}
free(bmbuf);
return kw;
}
RZ_API RzSearchKeyword *rz_search_keyword_new_wide(const char *kwbuf, const char *bmstr, const char *data, int ignore_case) {
RzSearchKeyword *kw;
int len;
const char *p2;
char *p, *str;
ut8 *bmbuf = NULL;
int bmlen = 0;
if (bmstr) {
bmbuf = malloc(strlen(bmstr) + 1);
if (!bmbuf) {
return NULL;
}
bmlen = rz_hex_str2bin(bmstr, bmbuf);
if (bmlen < 1) {
RZ_FREE(bmbuf);
}
}
len = strlen(kwbuf);
str = malloc((len + 1) * 2);
for (p2 = kwbuf, p = str; *p2;) {
RzCodePoint ch;
int num_utf8_bytes = rz_utf8_decode((const ut8 *)p2, kwbuf + len - p2, &ch, true);
if (num_utf8_bytes < 1) {
eprintf("WARNING: Malformed UTF8 at pos %td\n", p2 - kwbuf);
p[0] = *p2;
p[1] = 0;
p2++;
p += 2;
continue;
}
if (ignore_case && ch <= 0xff) {
ch = tolower(ch);
}
int num_wide_bytes = rz_utf16_encode((ut8 *)p, ch, false);
rz_warn_if_fail(num_wide_bytes != 0);
p2 += num_utf8_bytes;
p += num_wide_bytes;
}
kw = rz_search_keyword_new((ut8 *)str, p - str, bmbuf, bmlen, data);
free(str);
if (kw) {
kw->icase = ignore_case;
}
free(bmbuf);
return kw;
}
RZ_API RzSearchKeyword *rz_search_keyword_new_hex(const char *kwstr, const char *bmstr, const char *data) {
RzSearchKeyword *kw;
ut8 *kwbuf, *bmbuf;
int kwlen, bmlen = 0;
if (!kwstr) {
return NULL;
}
kwbuf = malloc(strlen(kwstr) + 1);
if (!kwbuf) {
return NULL;
}
kwlen = rz_hex_str2bin(kwstr, kwbuf);
if (kwlen < 1) {
free(kwbuf);
return NULL;
}
bmbuf = NULL;
if (bmstr && *bmstr) {
bmbuf = malloc(strlen(bmstr) + 1);
if (!bmbuf) {
free(kwbuf);
return NULL;
}
bmlen = rz_hex_str2bin(bmstr, bmbuf);
if (bmlen < 1) {
free(bmbuf);
free(kwbuf);
return NULL;
}
}
kw = rz_search_keyword_new(kwbuf, kwlen, bmbuf, bmlen, data);
free(kwbuf);
free(bmbuf);
return kw;
}
RZ_API RzSearchKeyword *rz_search_keyword_new_hexmask(const char *kwstr, const char *data) {
RzSearchKeyword *ks = NULL;
ut8 *kw, *bm;
if (kwstr != NULL) {
int len = strlen(kwstr);
kw = malloc(len + 4);
bm = malloc(len + 4);
if (kw != NULL && bm != NULL) {
len = rz_hex_str2bin_mask(kwstr, (ut8 *)kw, (ut8 *)bm, true);
if (len < 0) {
len = -len - 1;
}
if (len > 0) {
ks = rz_search_keyword_new(kw, len, bm, len, data);
}
}
free(kw);
free(bm);
}
return ks;
}
/* Validate a regexp in the canonical format /<regexp>/<options> */
RZ_API RzSearchKeyword *rz_search_keyword_new_regexp(const char *str, const char *data) {
RzSearchKeyword *kw;
int i = 0, start, length;
while (isspace((const unsigned char)str[i])) {
i++;
}
if (str[i++] != '/') {
return NULL;
}
/* Find the fist non backslash-escaped slash */
int specials = 0;
for (start = i; str[i]; i++) {
if (str[i] == '/' && str[i - 1] != '\\') {
break;
} else if (str[i - 1] == '\\' && isalpha(str[i])) {
specials++;
}
}
if (str[i++] != '/') {
return NULL;
}
length = i - start - 1;
if ((length > 128) || (length < 1)) {
return NULL;
}
kw = RZ_NEW0(RzSearchKeyword);
if (!kw) {
return NULL;
}
kw->bin_keyword = malloc(length + 1);
if (!kw->bin_keyword) {
rz_search_keyword_free(kw);
return NULL;
}
kw->bin_keyword[length] = 0;
memcpy(kw->bin_keyword, str + start, length);
kw->keyword_length = length - specials;
kw->type = RZ_SEARCH_KEYWORD_TYPE_STRING;
kw->data = (void *)data;
/* Parse the options */
for (; str[i]; i++) {
switch (str[i]) {
case 'i':
kw->icase = true;
break;
default:
rz_search_keyword_free(kw);
return NULL;
}
}
return kw;
}