portugalfuturista/rizin
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Improve performance of string search. (#5262)

Browse source 
* Make checks against defined Unicode points optional

* Allow to decode UTF-16 without writing the result.

* Remove PCRE2_NO_UTF_CHECK as default, since it can lead to undefined behavior.

* First refactor regex to support utf16 and utf32

* Add UTF-16-BE encoding function

* Add UTF-8 counting helper functions.

- One for counting the number of Unicode code points.
- The other to get the number of bytes required to represent the given UTF-8 string in UTF-16.

* Remove unused code

* Add UTF-8 to UTF-16 conversion function.

* Add type annotations

* Implement utf8 to utf32 string conversion

* Add UTF-16/32 versions of all other necessary regex functions for str search.

* Another regex refactor for utf16/32

* Add UTF16/32 regex matching tests.

* Implement still segfaulting (possibly JIT double usage) regex search.

* Duplicate match_first functions to reduce necessary branch predictions.

* Reduce number of required branches for encoding UTF16/32 to one.

* Duplicate match_all_internal functions to reduce necessary branch predictions.

* Fix too early free

* Only allocate match vector when needed.

* Fix: use code point size of buffer.

* Add missing return

* Normalize pointers to UTF16/32 strings to use proper code point with

* Also replace spaces with in utf16/32

* Add an additional host endian tests for UTF16/32 string encodings.

* Ensure thread savety.

JIT compiled patterns need to be owned by a single thread.
For the search we need to clone it.

JIT matching structures are optionally cloned as well.

* Enforce NO_UTF_CHECK in regex search.

This improves performance and currently is
default because we always match on binary data.

* Fix matching of UTF strings which are not suported by direct buffer matching.

PCRE2 only supports matching against memory which is aligned
to a code point width of the encoding.

These changes prevent taking the fast (direct matching with PCRE2) path
and use the slow string search path if the alignment doesn't match
the UTF string encoding.

To not complicate the change and additional alignment member
is added to each searched string in the search collection.

* Create search hit description on the stack

* Unset complete JIT matching if user provided custom jflags.

* Document what passing NULL to copy function pointers does.

* Replace the retarded idea of tracking offsets with a hashmap with a linear buffer.

This improves performance something like 10x.

* Remove const for the non-JIT builds.

* Remove additional flags for skip checking.

It is not needed because each decoded character is checked for printablity below anyways.

* Enfore no setup of IO mem with 0xff

* Fix: Set JIT complete flag for multi regex patterns

* Fix heap.

* Add note about worsed performance path.

* Fix unit test with string terminated by undefined code point.

* Run clang-format

* Fix order of arguments

* Add warning about string search with encoding=guess to tests.

* Fix string lengths, they no longer count the final invalid code point.

* Fix endian macro on Windows

* Fix NULL dereference

* Fix number tests

* Use endianness check not dependent on stdbit

* Fix type annotations.

* Reintroduce rz_str_len_utf8char

* Fix command description.

* Fix and unify RZ_SYS_ENDIAN macros.

- Don't allow unhandled architectures anymore.
- Check endianness for Sparc and PPC using non GCC/Clang compilers.
- Fix several endianness checks using the value instead of the macros.

* Fix tests

* Apply review comments.
This commit is contained in:
Rot127 2025-09-11 14:29:20 +00:00 committed by GitHub
parent 0c7db040d8
commit 699c7d2dff
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
39 changed files with 2377 additions and 553 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
librz/bin/pdb/cab_extract.c
View file

@ -111,7 +111,7 @@ static char *sanitize_cab_filename(struct mscabd_file *file, const char *output_
RzCodePoint code_point;
for (; input < endp;) {
code_point = 0;
int len = rz_utf8_decode(input, endp - input, &code_point);
int len = rz_utf8_decode(input, endp - input, &code_point, true);
if (!len) {
len = 1;
code_point = 0xFFFD;

7
librz/core/cmd/cmd_search.c
View file

@ -12,6 +12,7 @@
#include "cmd_search_rop.c"
#include "rz_cons.h"
#include <rz_config.h>
#include <rz_flag.h>
#include <rz_util/rz_file.h>
#include <rz_util/rz_log.h>
@ -2606,6 +2607,9 @@ static RzCmdStatus cmd_string_search_generic(RzCore *core, const char *string, c
if (!search_opts) {
return RZ_CMD_STATUS_ERROR;
}
bool memset_ff = rz_config_get_b(core->config, "io.ff");
// Disable memset of IO memory at read to be way faster.
rz_config_set_b(core->config, "io.ff", false);
CMD_SEARCH_BEGIN();
@ -2650,17 +2654,20 @@ static RzCmdStatus cmd_string_search_generic(RzCore *core, const char *string, c
RZ_LOG_ERROR("code: Failed to setup default search options.\n");
free(search_str);
rz_search_opt_free(search_opts);
rz_config_set_b(core->config, "io.ff", memset_ff);
CMD_SEARCH_END();
return RZ_CMD_STATUS_ERROR;
}
RzList *hits = rz_core_search_string(core, search_opts, search_str, search_str_len, flags, expected);
rz_config_set_b(core->config, "io.ff", memset_ff);
free(search_str);
rz_search_opt_free(search_opts);
CMD_SEARCH_END();
return cmd_core_handle_search_hits(core, state, hits);
invalid_args:
rz_config_set_b(core->config, "io.ff", memset_ff);
free(search_str);
rz_search_opt_free(search_opts);
CMD_SEARCH_END();

27
librz/core/csearch.c
View file

@ -161,8 +161,6 @@ RZ_API RZ_OWN RzList /*<RzSearchHit *>*/ *rz_core_search_bytes(RZ_NONNULL RzCore
}
}
// Don't pass the user provided search options.
// They were set up by the user and we respect them.
boundaries = rz_core_setup_io_search_parameters(core, user_opts);
if (!boundaries) {
RZ_LOG_ERROR("core: Setting up search from core failed.\n");
@ -219,8 +217,6 @@ RZ_API RZ_OWN RzList /*<RzSearchHit *>*/ *rz_core_search_values(RZ_NONNULL RzCor
}
}
// Don't pass the user provided search options.
// They were set up by the user and we respect them.
boundaries = rz_core_setup_io_search_parameters(core, user_opts);
if (!boundaries) {
RZ_LOG_ERROR("core: Setting up search from core failed.\n");
@ -253,7 +249,7 @@ quit:
*
* \return On success returns a valid pointer to a list of search hits, otherwise NULL.
*/
RZ_API RZ_OWN RzList /*<RzSearchHit *>*/ *rz_core_search_string(RZ_NONNULL RzCore *core, RZ_BORROW RZ_NONNULL RzSearchOpt *user_opts, RZ_NONNULL const char *re_pattern, size_t re_pattern_len, RzRegexFlags flags, RzStrEnc expected) {
RZ_API RZ_OWN RzList /*<RzSearchHit *>*/ *rz_core_search_string(RZ_NONNULL RzCore *core, RZ_BORROW RZ_NONNULL RzSearchOpt *user_opts, RZ_NONNULL const char *re_pattern, size_t re_pattern_len, RzRegexFlags cflags, RzStrEnc expected) {
rz_return_val_if_fail(core && user_opts && re_pattern, NULL);
if (RZ_STR_ISEMPTY(re_pattern)) {
@ -278,14 +274,7 @@ RZ_API RZ_OWN RzList /*<RzSearchHit *>*/ *rz_core_search_string(RZ_NONNULL RzCor
RzList *hits = NULL;
RzList *boundaries = NULL;
RzSearchOpt *search_opts = NULL;
RzSearchCollection *collection = rz_search_collection_strings(&scan_opt, expected, flags);
if (!collection ||
!rz_search_collection_string_add(collection, re_pattern, flags)) {
rz_search_collection_free(collection);
RZ_LOG_ERROR("core: Failed to initialize search collection.\n");
return NULL;
}
RzSearchCollection *collection = NULL;
if (!user_opts) {
// override user_opts with default one
@ -305,6 +294,18 @@ RZ_API RZ_OWN RzList /*<RzSearchHit *>*/ *rz_core_search_string(RZ_NONNULL RzCor
goto quit;
}
collection = rz_search_collection_strings(&scan_opt, expected);
size_t match_alignment = rz_search_find_opt_get_alignment(rz_search_opt_get_find_options(user_opts));
if (!collection ||
!rz_search_collection_string_add(collection, re_pattern, cflags, match_alignment)) {
rz_search_collection_free(collection);
RZ_LOG_ERROR("core: Failed to initialize search collection.\n");
return NULL;
}
rz_search_collection_strings_check_config_improvements(
collection, boundaries, user_opts, &scan_opt, true);
hits = perform_search_on_core_io(core, user_opts, boundaries, collection);
quit:

2
librz/core/disasm.c
View file

@ -3790,7 +3790,7 @@ static char *ds_esc_str(RzDisasmState *ds, const char *str, int len, const char
end = str + len - 1;
}
for (ptr = str; ptr < end; ptr += 4) {
if (rz_utf32le_decode((ut8 *)ptr, end - ptr, &ch) == 0) {
if (rz_utf32le_decode((ut8 *)ptr, end - ptr, &ch, true) == 0) {
enc = RZ_STRING_ENC_8BIT;
break;
}

12
librz/include/rz_search.h
View file

@ -215,12 +215,14 @@ RZ_API bool rz_search_opt_set_show_progress_from_str(RZ_NONNULL RzSearchOpt *opt
RZ_API RzSearchProgress rz_search_opt_get_show_progress(RZ_NONNULL RzSearchOpt *opt);
RZ_API bool rz_search_opt_set_cancel_cb(RZ_NONNULL RzSearchOpt *opt, RzSearchCancelCallback callback, void *user);
RZ_API bool rz_search_opt_set_find_options(RZ_NONNULL RzSearchOpt *opt, RZ_OWN RzSearchFindOpt *find_opts);
RZ_API const RzSearchFindOpt *rz_search_opt_get_find_options(RZ_NONNULL const RzSearchOpt *opt);
RZ_API RZ_OWN RzSearchFindOpt *rz_search_find_opt_new();
RZ_API void rz_search_find_opt_free(RZ_NULLABLE RzSearchFindOpt *opt);
RZ_API bool rz_search_find_opt_set_inverse_match(RZ_NONNULL RzSearchFindOpt *opt, bool inverse_match);
RZ_API bool rz_search_find_opt_set_overlap_match(RZ_NONNULL RzSearchFindOpt *opt, bool overlap_match);
RZ_API bool rz_search_find_opt_set_alignment(RZ_NONNULL RzSearchFindOpt *opt, size_t alignment);
RZ_API ut16 rz_search_find_opt_get_alignment(RZ_NONNULL const RzSearchFindOpt *opt);
typedef enum {
RZ_SEARCH_COLLECTION_CRYPTOGRAPHIC_AES_128 = 0,
@ -263,8 +265,14 @@ RZ_API RZ_OWN RzSearchCollection *rz_search_collection_bytes();
RZ_API bool rz_search_collection_bytes_add(RZ_NONNULL RzSearchCollection *col, RZ_NULLABLE const char *pattern_desc, RZ_NONNULL const ut8 *bytes, RZ_NULLABLE const ut8 *mask, size_t length);
RZ_API bool rz_search_collection_bytes_add_pattern(RZ_NONNULL RzSearchCollection *col, RZ_NONNULL RZ_OWN RzSearchBytesPattern *bytes_pattern);
RZ_API RZ_OWN RzSearchCollection *rz_search_collection_strings(RZ_NONNULL RzUtilStrScanOptions *opts, RzStrEnc expected, RzRegexFlags re_flags);
RZ_API bool rz_search_collection_string_add(RZ_NONNULL RzSearchCollection *col, RZ_NONNULL const char *regex_pattern, RzRegexFlags re_flags);
RZ_API RZ_OWN RzSearchCollection *rz_search_collection_strings(RZ_NONNULL RzUtilStrScanOptions *opts, RzStrEnc expected);
RZ_API bool rz_search_collection_string_add(RZ_NONNULL RzSearchCollection *col, RZ_NONNULL const char *regex_pattern, RzRegexFlags cflags, size_t match_alignment);
RZ_API bool rz_search_collection_strings_check_config_improvements(
RZ_NULLABLE const RzSearchCollection *col,
RZ_NULLABLE const RzList /*<RzIOMap *>*/ *boundaries,
RZ_NULLABLE const RzSearchOpt *search_options,
RZ_NULLABLE const RzUtilStrScanOptions *scan_opt,
bool print_msg);
RZ_API bool rz_search_collection_match_any(RZ_NULLABLE RzSearchCollection *sc, RZ_NONNULL const ut8 *buffer, size_t length);
RZ_API void rz_search_collection_free(RZ_NULLABLE RzSearchCollection *sc);

237
librz/include/rz_types.h
View file

@ -400,120 +400,151 @@ static inline void *rz_new_copy(int size, const void *data) {
#define O_BINARY 0
#endif
// clang-format off
#if __APPLE__
#if __i386__
#define RZ_SYS_BASE ((ut64)0x1000)
#elif __x86_64__
#define RZ_SYS_BASE ((ut64)0x100000000)
#else
#define RZ_SYS_BASE ((ut64)0x1000)
#endif
#if __i386__
#define RZ_SYS_BASE ((ut64)0x1000)
#elif __x86_64__
#define RZ_SYS_BASE ((ut64)0x100000000)
#else
#define RZ_SYS_BASE ((ut64)0x1000)
#endif
#elif __WINDOWS__
#define RZ_SYS_BASE ((ut64)0x01001000)
#define RZ_SYS_BASE ((ut64)0x01001000)
#else // linux, bsd, ...
#if __arm__ || __arm64__
#define RZ_SYS_BASE ((ut64)0x4000)
#else
#define RZ_SYS_BASE ((ut64)0x8048000)
#endif
#if __arm__ || __arm64__
#define RZ_SYS_BASE ((ut64)0x4000)
#else
#define RZ_SYS_BASE ((ut64)0x8048000)
#endif
#endif
/* arch */
#if __i386__
#define RZ_SYS_ARCH "x86"
#define RZ_SYS_BITS RZ_SYS_BITS_32
#define RZ_SYS_ENDIAN 0
#elif __EMSCRIPTEN__
#define RZ_SYS_ARCH "wasm"
#define RZ_SYS_BITS (RZ_SYS_BITS_32 | RZ_SYS_BITS_64)
#define RZ_SYS_ENDIAN 0
#elif __x86_64__
#define RZ_SYS_ARCH "x86"
#define RZ_SYS_BITS (RZ_SYS_BITS_32 | RZ_SYS_BITS_64)
#define RZ_SYS_ENDIAN 0
#elif __POWERPC__
#define RZ_SYS_ARCH "ppc"
#ifdef __powerpc64__
#define RZ_SYS_BITS (RZ_SYS_BITS_32 | RZ_SYS_BITS_64)
#else
#define RZ_SYS_BITS RZ_SYS_BITS_32
#endif
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define RZ_SYS_ENDIAN 0
#else
#define RZ_SYS_ENDIAN 1
#endif
#elif __arm__
#define RZ_SYS_ARCH "arm"
#define RZ_SYS_BITS RZ_SYS_BITS_32
#define RZ_SYS_ENDIAN 0
#elif __arm64__ || __aarch64__
#define RZ_SYS_ARCH "arm"
#define RZ_SYS_BITS (RZ_SYS_BITS_32 | RZ_SYS_BITS_64)
#define RZ_SYS_ENDIAN 0
#elif __arc__
#define RZ_SYS_ARCH "arc"
#define RZ_SYS_BITS RZ_SYS_BITS_32
#define RZ_SYS_ENDIAN 0
#elif __s390x__
#define RZ_SYS_ARCH "sysz"
#define RZ_SYS_BITS RZ_SYS_BITS_64
#define RZ_SYS_ENDIAN 1
#elif __sparc__
#define RZ_SYS_ARCH "sparc"
#define RZ_SYS_BITS RZ_SYS_BITS_32
#define RZ_SYS_ENDIAN 1
#elif __mips__
#define RZ_SYS_ARCH "mips"
#define RZ_SYS_BITS RZ_SYS_BITS_32
#define RZ_SYS_ENDIAN 1
#elif __EMSCRIPTEN__
/* we should default to wasm when ready */
#define RZ_SYS_ARCH "x86"
#define RZ_SYS_BITS RZ_SYS_BITS_32
#elif __riscv__ || __riscv
#define RZ_SYS_ARCH "riscv"
#define RZ_SYS_ENDIAN 0
#if __riscv_xlen == 32
#define RZ_SYS_BITS RZ_SYS_BITS_32
#else
#define RZ_SYS_BITS (RZ_SYS_BITS_32 | RZ_SYS_BITS_64)
#endif
#else
#ifdef _MSC_VER
#if defined(_M_X64) || defined(_M_AMD64)
#define RZ_SYS_ARCH "x86"
#define RZ_SYS_BITS (RZ_SYS_BITS_32 | RZ_SYS_BITS_64)
#define RZ_SYS_ENDIAN 0
#define __x86_64__ 1
#elif defined(_M_IX86)
#define RZ_SYS_ARCH "x86"
#define RZ_SYS_BITS (RZ_SYS_BITS_32)
#define RZ_SYS_ENDIAN 0
#define __i386__ 1
#elif defined(_M_ARM64)
#define RZ_SYS_ARCH "arm"
#define RZ_SYS_BITS (RZ_SYS_BITS_32 | RZ_SYS_BITS_64)
#define RZ_SYS_ENDIAN 0
#define __arm64__ 1
#elif defined(_M_ARM)
#define RZ_SYS_ARCH "arm"
#define RZ_SYS_BITS RZ_SYS_BITS_32
#define RZ_SYS_ENDIAN 0
#define __arm__ 1
#endif
#else
#define RZ_SYS_ARCH "unknown"
#define RZ_SYS_BITS RZ_SYS_BITS_32
#define RZ_SYS_ENDIAN 0
#endif
#endif
static const ut32 rz_endianness_one = 1;
#define RZ_SYS_ENDIAN_NONE 0
#define RZ_SYS_ENDIAN_LITTLE 1
#define RZ_SYS_ENDIAN_BIG 2
#define RZ_SYS_ENDIAN_BI 3
/* arch */
#if __i386__
#define RZ_SYS_ARCH "x86"
#define RZ_SYS_BITS RZ_SYS_BITS_32
#define RZ_SYS_ENDIAN RZ_SYS_ENDIAN_LITTLE
#elif __EMSCRIPTEN__
#define RZ_SYS_ARCH "wasm"
#define RZ_SYS_BITS (RZ_SYS_BITS_32 | RZ_SYS_BITS_64)
#define RZ_SYS_ENDIAN RZ_SYS_ENDIAN_LITTLE
#elif __x86_64__
#define RZ_SYS_ARCH "x86"
#define RZ_SYS_BITS (RZ_SYS_BITS_32 | RZ_SYS_BITS_64)
#define RZ_SYS_ENDIAN RZ_SYS_ENDIAN_LITTLE
#elif __POWERPC__
#define RZ_SYS_ARCH "ppc"
#ifdef __powerpc64__
#define RZ_SYS_BITS (RZ_SYS_BITS_32 | RZ_SYS_BITS_64)
#else
#define RZ_SYS_BITS RZ_SYS_BITS_32
#endif
#if defined(__BYTE_ORDER__)
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define RZ_SYS_ENDIAN RZ_SYS_ENDIAN_LITTLE
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#define RZ_SYS_ENDIAN RZ_SYS_ENDIAN_BIG
#else
#error "Unsupported endianness"
#endif
#else
#define RZ_SYS_ENDIAN ((*((char *)&(rz_endianness_one)) == 1) ? RZ_SYS_ENDIAN_LITTLE : RZ_SYS_ENDIAN_BIG)
#endif
#elif __arm__
#define RZ_SYS_ARCH "arm"
#define RZ_SYS_BITS RZ_SYS_BITS_32
#define RZ_SYS_ENDIAN RZ_SYS_ENDIAN_LITTLE
#elif __arm64__ || __aarch64__
#define RZ_SYS_ARCH "arm"
#define RZ_SYS_BITS (RZ_SYS_BITS_32 | RZ_SYS_BITS_64)
#define RZ_SYS_ENDIAN RZ_SYS_ENDIAN_LITTLE
#elif __arc__
#define RZ_SYS_ARCH "arc"
#define RZ_SYS_BITS RZ_SYS_BITS_32
#define RZ_SYS_ENDIAN RZ_SYS_ENDIAN_LITTLE
#elif __s390x__
#define RZ_SYS_ARCH "sysz"
#define RZ_SYS_BITS RZ_SYS_BITS_64
#define RZ_SYS_ENDIAN RZ_SYS_ENDIAN_BIG
#elif __sparc__
#define RZ_SYS_ARCH "sparc"
#define RZ_SYS_BITS RZ_SYS_BITS_32
#if defined(__BYTE_ORDER__)
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define RZ_SYS_ENDIAN RZ_SYS_ENDIAN_LITTLE
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#define RZ_SYS_ENDIAN RZ_SYS_ENDIAN_BIG
#else
#error "Unsupported endianness"
#endif
#else
#define RZ_SYS_ENDIAN ((*((char *)&(rz_endianness_one)) == 1) ? RZ_SYS_ENDIAN_LITTLE : RZ_SYS_ENDIAN_BIG)
#endif
#elif __mips__
#define RZ_SYS_ARCH "mips"
#define RZ_SYS_BITS RZ_SYS_BITS_32
#define RZ_SYS_ENDIAN RZ_SYS_ENDIAN_BIG
#elif __EMSCRIPTEN__
/* we should default to wasm when ready */
#define RZ_SYS_ARCH "x86"
#define RZ_SYS_BITS RZ_SYS_BITS_32
#elif __riscv__ || __riscv
#define RZ_SYS_ARCH "riscv"
#define RZ_SYS_ENDIAN RZ_SYS_ENDIAN_LITTLE
#if __riscv_xlen == 32
#define RZ_SYS_BITS RZ_SYS_BITS_32
#else
#define RZ_SYS_BITS (RZ_SYS_BITS_32 | RZ_SYS_BITS_64)
#endif
#else
#ifdef _MSC_VER
#if defined(_M_X64) || defined(_M_AMD64)
#define RZ_SYS_ARCH "x86"
#define RZ_SYS_BITS (RZ_SYS_BITS_32 | RZ_SYS_BITS_64)
#define RZ_SYS_ENDIAN RZ_SYS_ENDIAN_LITTLE
#define __x86_64__ 1
#elif defined(_M_IX86)
#define RZ_SYS_ARCH "x86"
#define RZ_SYS_BITS (RZ_SYS_BITS_32)
#define RZ_SYS_ENDIAN RZ_SYS_ENDIAN_LITTLE
#define __i386__ 1
#elif defined(_M_ARM64)
#define RZ_SYS_ARCH "arm"
#define RZ_SYS_BITS (RZ_SYS_BITS_32 | RZ_SYS_BITS_64)
#define RZ_SYS_ENDIAN RZ_SYS_ENDIAN_LITTLE
#define __arm64__ 1
#elif defined(_M_ARM)
#define RZ_SYS_ARCH "arm"
#define RZ_SYS_BITS RZ_SYS_BITS_32
#define RZ_SYS_ENDIAN RZ_SYS_ENDIAN_LITTLE
#define __arm__ 1
#else
#error "Unhandled Windows architecture."
#endif
#else
#error "Unhandled bits and edianness definitions for this architecture."
#endif
#endif
#ifndef RZ_SYS_ENDIAN
#error "Endianness for this architecture is not defined. This is no longer valid."
#elif (RZ_SYS_ENDIAN == RZ_SYS_ENDIAN_BI || RZ_SYS_ENDIAN == RZ_SYS_ENDIAN_NONE)
#error "RZ_SYS_ENDIAN_BI or RZ_SYS_ENDIAN_NONE are invalid values for the architecture endianness."
#endif
// clang-format on
#define RZ_HOST_IS_LITTLE_ENDIAN (RZ_SYS_ENDIAN == RZ_SYS_ENDIAN_LITTLE)
#define RZ_HOST_IS_BIG_ENDIAN (RZ_SYS_ENDIAN == RZ_SYS_ENDIAN_BIG)
typedef enum {
RZ_SYS_ARCH_NONE = 0,
RZ_SYS_ARCH_X86,

109
librz/include/rz_util/rz_regex.h
View file

@ -24,9 +24,18 @@
* M is inspected during rz_regex_match() execution
* D is inspected during pcre2_dfa_match() execution (not used).
*/
#define RZ_REGEX_DEFAULT 0
#define RZ_REGEX_LITERAL 0x02000000u /* PCRE2_LITERAL - C */
#define RZ_REGEX_CASELESS 0x00000008u /* PCRE2_CASELESS - C */
#define RZ_REGEX_DEFAULT 0
#define RZ_REGEX_LITERAL 0x02000000u /* PCRE2_LITERAL - C */
#define RZ_REGEX_CASELESS 0x00000008u /* PCRE2_CASELESS - C */
/**
* \brief If RZ_REGEX_EXTENDED is passed to rz_regex_new_16() or rz_regex_new_32()
* spaces in the pattern **will** be skipped! You need to replace them with \s.
* This is in accordance with the PCRE2 documentation.
*
* If RZ_REGEX_EXTENDED is passed to rz_regex_new() (the UTF-8 regular expressions)
* the spaces **will not** be skipped but interally be replaced with '\s'.
* This was done to keep our interal regex matching stable.
*/
#define RZ_REGEX_EXTENDED 0x00000080u /* PCRE2_EXTENDED - C */
#define RZ_REGEX_EXTENDED_MORE 0x01000000u /* PCRE2_EXTENDED_MORE - C */
#define RZ_REGEX_MULTILINE 0x00000400u /* PCRE2_MULTILINE - C */
@ -44,12 +53,41 @@ typedef int RzRegexStatus; ///< An status number returned by the regex API.
typedef size_t RzRegexSize; ///< Size of a text or regex. This is the size measured in code width. For UTF-8: bytes.
typedef ut32 RzRegexFlags; ///< Regex flag bits.
typedef uint8_t *RzRegexPattern; ///< A regex pattern string.
typedef void RzRegex; ///< A regex expression.
typedef void RzRegexCompContext; ///< A PCRE2 compile context.
typedef void RzRegex; ///< A regex expression for UTF-8 strings.
typedef void RzRegexCompContext; ///< A PCRE2 compile context for UTF-8 strings.
typedef void RzRegex16; ///< A regex expression for UTF-16 strings (host endianess).
typedef void RzRegexCompContext16; ///< A PCRE2 compile context for UTF-16 strings (host endianess).
typedef void RzRegex32; ///< A regex expression for UTF-32 strings (host endianess).
typedef void RzRegexCompContext32; ///< A PCRE2 compile context for UTF-32 strings (host endianess).
typedef enum {
RZ_REGEX_UTF8,
RZ_REGEX_UTF16,
RZ_REGEX_UTF32,
} RzRegexType;
typedef struct {
RzRegexType re_type;
RzRegexFlags compile_flags_jit;
union {
RzRegex *re8;
RzRegex16 *re16;
RzRegex32 *re32;
};
} RzRegexMulti;
typedef struct {
RzRegexSize group_idx; ///< Index of the group. Used to determine name if any was given.
RzRegexSize start; ///< Start offset into the text where the match starts.
/**
* \brief Start offset into the text where the match starts.
* The offset is in code units, not in characters!
* One code unit is 1 byte for UTF-8, 2 bytes for UTF-16, and 4 bytes for UTF-32.
*/
RzRegexSize start;
/**
* \brief The length of the match in number of code units.
* One code unit is 1 byte for UTF-8, 2 bytes for UTF-16, and 4 bytes for UTF-32.
*/
RzRegexSize len; ///< Length of match in bytes.
} RzRegexMatch;
@ -57,9 +95,20 @@ typedef void RzRegexMatchData; ///< PCRE2 internal match data type
RZ_API RZ_OWN RzRegex *rz_regex_new(RZ_NONNULL const char *pattern, RzRegexFlags cflags, RzRegexFlags jflags,
RzRegexCompContext *ccontext);
RZ_API RZ_OWN RzRegex16 *rz_regex_new_16(RZ_NONNULL const char *pattern, RzRegexFlags cflags, RzRegexFlags jflags,
RzRegexCompContext *ccontext);
RZ_API RZ_OWN RzRegex32 *rz_regex_new_32(RZ_NONNULL const char *pattern, RzRegexFlags cflags, RzRegexFlags jflags,
RzRegexCompContext *ccontext);
RZ_API RZ_OWN RzRegexMulti *rz_regex_new_multi(RZ_NONNULL const char *pattern, RzRegexFlags cflags, RzRegexFlags jflags,
RzRegexCompContext *ccontext, RzRegexType type);
RZ_API RZ_OWN RzRegex *rz_regex_new_bytes(RZ_NONNULL const ut8 *pattern, size_t pattern_len, RzRegexFlags cflags, RzRegexFlags jflags,
RzRegexCompContext *ccontext);
RZ_API void rz_regex_free(RZ_OWN RzRegex *regex);
RZ_API void rz_regex_free_16(RZ_OWN RzRegex16 *regex);
RZ_API void rz_regex_free_32(RZ_OWN RzRegex32 *regex);
RZ_API void rz_regex_free_multi(RZ_NULLABLE RZ_OWN RzRegexMulti *regex);
RZ_API RZ_OWN RzRegexMulti *rz_regex_multi_clone(RZ_NONNULL RzRegexMulti *regex, bool clone_jit);
RZ_API void rz_regex_free_multi_clone(RZ_NULLABLE RZ_OWN RzRegexMulti *regex);
RZ_API void rz_regex_error_msg(RzRegexStatus errcode, RZ_OUT char *errbuf, RzRegexSize errbuf_size);
RZ_API const ut8 *rz_regex_get_match_name(RZ_NONNULL const RzRegex *regex, ut32 name_idx);
RZ_API st32 rz_regex_get_group_idx_by_name(RZ_NONNULL const RzRegex *regex, const char *group);
@ -79,6 +128,18 @@ RZ_API RZ_OWN RzPVector /*<RzRegexMatch *>*/ *rz_regex_match_first(
RzRegexSize text_size,
RzRegexSize text_offset,
RzRegexFlags mflags);
RZ_API RZ_OWN RzPVector /*<RzRegexMatch *>*/ *rz_regex_match_first_16(
RZ_NONNULL const RzRegex16 *regex,
RZ_NONNULL const ut16 *text,
RzRegexSize text_size,
RzRegexSize text_offset,
RzRegexFlags mflags);
RZ_API RZ_OWN RzPVector /*<RzRegexMatch *>*/ *rz_regex_match_first_32(
RZ_NONNULL const RzRegex32 *regex,
RZ_NONNULL const ut32 *text,
RzRegexSize text_size,
RzRegexSize text_offset,
RzRegexFlags mflags);
RZ_API RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *rz_regex_match_all(
RZ_NONNULL const RzRegex *regex,
RZ_NONNULL const char *text,
@ -91,6 +152,42 @@ RZ_API RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *rz_regex_match_all_ove
RzRegexSize text_size,
RzRegexSize text_offset,
RzRegexFlags mflags);
RZ_API RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *rz_regex_match_all_16(
RZ_NONNULL const RzRegex16 *regex,
RZ_NONNULL const ut16 *text,
RzRegexSize text_size,
RzRegexSize text_offset,
RzRegexFlags mflags);
RZ_API RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *rz_regex_match_all_overlap_16(
RZ_NONNULL const RzRegex16 *regex,
RZ_NONNULL const ut16 *text,
RzRegexSize text_size,
RzRegexSize text_offset,
RzRegexFlags mflags);
RZ_API RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *rz_regex_match_all_32(
RZ_NONNULL const RzRegex32 *regex,
RZ_NONNULL const ut32 *text,
RzRegexSize text_size,
RzRegexSize text_offset,
RzRegexFlags mflags);
RZ_API RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *rz_regex_match_all_overlap_32(
RZ_NONNULL const RzRegex32 *regex,
RZ_NONNULL const ut32 *text,
RzRegexSize text_size,
RzRegexSize text_offset,
RzRegexFlags mflags);
RZ_API RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *rz_regex_match_all_multi(
RZ_NONNULL const RzRegexMulti *regex,
RZ_NONNULL const ut8 *text,
RzRegexSize text_size,
RzRegexSize text_offset,
RzRegexFlags mflags);
RZ_API RZ_OWN RzPVector /*<RzVector<RzRegexMatch *> *>*/ *rz_regex_match_all_overlap_multi(
RZ_NONNULL const RzRegexMulti *regex,
RZ_NONNULL const ut8 *text,
RzRegexSize text_size,
RzRegexSize text_offset,
RzRegexFlags mflags);
RZ_API bool rz_regex_contains(RZ_NONNULL const char *pattern, RZ_NONNULL const char *text,
RzRegexSize text_size,
RzRegexFlags cflags, RzRegexFlags mflags);

50
librz/include/rz_util/rz_str.h
View file

@ -2,6 +2,7 @@
#define RZ_STR_H
#include <wchar.h>
#include "rz_assert.h"
#include "rz_str_util.h"
#include "rz_list.h"
#include "rz_types.h"
@ -17,7 +18,11 @@ typedef enum {
} RzStrType;
typedef enum {
RZ_STRING_ENC_8BIT = 'b', // unknown 8bit encoding but with ASCII from 0 to 0x7f
/**
* \brief Unknown 8bit encoding but with ASCII from 0 to 0x7f.
* It is also used everywhere like it is ASCII.
*/
RZ_STRING_ENC_8BIT = 'b',
RZ_STRING_ENC_UTF8 = '8',
RZ_STRING_ENC_MUTF8 = 'm', // modified utf8
RZ_STRING_ENC_UTF16LE = 'u',
@ -85,6 +90,8 @@ RZ_API char *rz_str_crop(const char *str, unsigned int x, unsigned int y, unsign
RZ_API char *rz_str_scale(const char *r, int w, int h);
RZ_API bool rz_str_range_in(const char *r, ut64 addr);
RZ_API size_t rz_str_len_utf8(const char *s);
RZ_API size_t rz_str_utf8_num_ucp(RZ_NONNULL const char *str);
RZ_API size_t rz_str_utf8_get_width_utf16(RZ_NONNULL const char *str);
RZ_API size_t rz_str_len_utf8_ansi(const char *str);
RZ_API size_t rz_str_len_utf8char(const char *s, int left);
RZ_API size_t rz_str_utf8_charsize(const char *str);
@ -226,7 +233,9 @@ RZ_API void rz_str_uri_decode(char *buf);
RZ_API char *rz_str_uri_encode(const char *buf);
RZ_API char *rz_str_utf16_decode(const ut8 *s, int len);
RZ_API int rz_str_utf16_to_utf8(ut8 *dst, int len_dst, const ut8 *src, int len_src, bool little_endian);
RZ_API char *rz_str_utf16_encode(const char *s, int len);
RZ_DEPRECATE RZ_API char *rz_str_utf16_encode(const char *s, int len);
RZ_API RZ_OWN ut16 *rz_str_utf8_to_utf16(RZ_NONNULL const char *utf8_str, bool big_endian);
RZ_API RZ_OWN ut32 *rz_str_utf8_to_utf32(RZ_NONNULL const char *utf8_str, bool big_endian);
RZ_API char *rz_str_escape_utf8_for_json(const char *s, int len);
RZ_API char *rz_str_escape_mutf8_for_json(const char *s, int len);
RZ_API char *rz_str_home(const char *str);
@ -281,10 +290,45 @@ RZ_API RZ_OWN char *rz_str_stringify_raw_buffer(RzStrStringifyOpt *option, RZ_NU
RZ_API const char *rz_str_indent(int indent);
static inline bool rz_string_enc_is_utf8_compatible(RzStrEnc enc) {
/**
* \brief Returns true if the given encoding has the same byte character width as UTF-8.
* This is only true for UTF-8 and ASCII.
*
* Examples:
*
* ```c
* // IBM290 character width is always one byte, but the equivalent Japanese
* // characters in UTF-8 are 3 bytes.
* assert(rz_string_enc_same_char_width_as_utf8(RZ_STR_ENC_IBM290) == false);
*
* // ASCII character width is always one byte, and the equivalent
* // UTF-8 characters are also always 1 byte.
* assert(rz_string_enc_same_char_width_as_utf8(RZ_STR_ENC_8BIT) == true);
* ```
*/
static inline bool rz_string_enc_same_char_width_as_utf8(RzStrEnc enc) {
return enc == RZ_STRING_ENC_UTF8 || enc == RZ_STRING_ENC_8BIT;
}
RZ_API bool rz_string_enc_is_utf_native_endian(RzStrEnc enc);
RZ_API size_t rz_string_enc_code_point_width(RzStrEnc enc);
static inline bool rz_string_code_points_align(RzStrEnc enc, size_t memory_alignment) {
if (rz_string_enc_code_point_width(enc) == memory_alignment) {
return true;
}
switch (enc) {
case RZ_STRING_ENC_BASE64:
case RZ_STRING_ENC_SETTINGS:
rz_warn_if_reached();
return false;
case RZ_STRING_ENC_GUESS:
return false;
default:
return memory_alignment % rz_string_enc_code_point_width(enc) == 0;
}
}
#ifdef __cplusplus
}
#endif

37
librz/include/rz_util/rz_str_search.h
View file

@ -5,7 +5,6 @@
#include <rz_util/rz_assert.h>
#include <rz_util/rz_buf.h>
#include <rz_util/rz_regex.h>
#include <rz_util/ht_uu.h>
#include <rz_list.h>
#ifdef __cplusplus
@ -17,11 +16,21 @@ extern "C" {
*/
typedef struct {
char *string; ///< The detected string. Note that this one is always in UTF-8. No matter what the ecoding is in memory.
RzRegex *regex; ///< Regex matching the string. If set, the string member is the pattern.
RzRegexMulti *regex; ///< Regex matching the string. If set, the string member is the pattern.
ut64 addr; ///< Address/offset of the string in the RzBuffer
ut32 size; ///< Size of buffer containing the string in bytes
ut32 length; ///< Length of string in chars
RzStrEnc type; ///< String encoding in memory.
size_t alignment; ///< The address alignment a matched string must have. If search.align is set, both must match.
/**
* \brief Maps UTF-8 code point offsets to their memory offset.
* This is necessary if the string's character width in memory doesn't match UTF-8 character width.
* E.g. the in memory string is UTF-32 and has a character width of 4 bytes.
* But the decoded string above is always UTF-8 and has a character width of 1-4 bytes.
*
* It is NULL if the string encoding in memory is UTF-8 or ASCII.
*/
ut64 *byte_mem_map;
} RzDetectedString;
/**
@ -32,30 +41,6 @@ typedef struct {
size_t min_str_length; ///< Minimum string length
bool prefer_big_endian; ///< True if the preferred endianess for UTF strings is big-endian
bool check_ascii_freq; ///< If true, perform check on ASCII frequencies when looking for false positives
/**
* \brief Map UTF-8 byte offsets to memory offsets.
* The string scan function always returns UTF-8 strings.
* Independent what encoding the strings have in memory.
* Sometimes it is necessary to know the offsets of the real encoding.
* This maps an UTF-8 code point offset to the original code point offset in memory.
* The keys are ut64 values. With the upper 32bits holding the index into the
* "detected string list" returned by rz_scan_strings_whole_buf().
* The lower 32bits are the offset into the UTF-8 string.
* The value is the offset into the memory. Relevant to the buffer
* The string was found in.
*
* Example:
*
* Buffer (UTF-16): 0x00, 0x41, 0x00, 0x41, 0x00, 0x00, 0x00, 0x42, 0x00, 0x42
* Found strings (UTF-8): [ "AA", "BB" ]
* Map: {
* 0x0000000000000000: 0,
* 0x0000000000000001: 2,
* 0x0000000100000000: 6,
* 0x0000000100000001: 8
* }
*/
RZ_NULLABLE HtUU *utf8_to_mem_offset_map;
} RzUtilStrScanOptions;
RZ_API void rz_detected_string_free(RzDetectedString *str);

17
librz/include/rz_util/rz_utf16.h
View file

@ -4,10 +4,19 @@
/* For RzCodePoint definition */
#include "rz_utf8.h"
RZ_API size_t rz_utf16_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NONNULL RZ_OUT RzCodePoint *ch, bool bigendian);
RZ_API size_t rz_utf16le_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NONNULL RZ_OUT RzCodePoint *ch);
RZ_API size_t rz_utf16be_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NONNULL RZ_OUT RzCodePoint *ch);
RZ_API size_t rz_utf16le_encode(RZ_NONNULL RZ_OUT ut8 *buf, RzCodePoint ch);
/**
* \brief First Unicode code point which needs 4 bytes to be encoded.
*/
#define RZ_UTF16_FIRST_4BYTES_CODE_POINT 0x10000
/**
* \brief Width of an UTF16 character in bytes.
*/
#define RZ_UTF16_CODE_POINT_WIDTH 2
RZ_API size_t rz_utf16_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULLABLE RZ_OUT RzCodePoint *ch, bool check_is_def, bool bigendian);
RZ_API size_t rz_utf16le_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULLABLE RZ_OUT RzCodePoint *ch, bool check_is_def);
RZ_API size_t rz_utf16be_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULLABLE RZ_OUT RzCodePoint *ch, bool check_is_def);
RZ_API size_t rz_utf16_encode(RZ_NONNULL RZ_OUT ut8 *buf, RzCodePoint ch, bool big_endian);
RZ_API bool rz_utf16_is_printable_code_point(RZ_NONNULL const ut8 *buf, size_t buf_len, bool big_endian, size_t lookahead);
#endif // RZ_UTF16_H

13
librz/include/rz_util/rz_utf32.h
View file

@ -11,9 +11,16 @@
#define RZ_UTF32_UNICODE_BOM_LE 0xFFFE0000
#define RZ_UTF32_UNICODE_BOM_BE 0x0000FFFE
RZ_API size_t rz_utf32_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULLABLE RZ_OUT RzCodePoint *ch, bool big_endian);
RZ_API int rz_utf32le_decode(const ut8 *ptr, int ptrlen, RzCodePoint *ch);
RZ_API int rz_utf32be_decode(const ut8 *ptr, int ptrlen, RzCodePoint *ch);
/**
* \brief Width of an UTF32 character in bytes.
*/
#define RZ_UTF32_WIDTH_CHAR 4
#define RZ_UTF32_CODE_POINT_WIDTH 4
RZ_API size_t rz_utf32_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULLABLE RZ_OUT RzCodePoint *ch, bool check_is_def, bool big_endian);
RZ_API int rz_utf32le_decode(const ut8 *ptr, int ptrlen, RZ_NULLABLE RZ_OUT RzCodePoint *ch, bool check_validity);
RZ_API int rz_utf32be_decode(const ut8 *ptr, int ptrlen, RZ_NULLABLE RZ_OUT RzCodePoint *ch, bool check_validity);
RZ_API size_t rz_utf32_encode(RZ_NONNULL RZ_OUT ut8 *buf, RzCodePoint ch, bool big_endian);
RZ_API bool rz_utf32_valid_code_point(RZ_NONNULL const ut8 *buf, size_t buf_len, bool big_endian, size_t lookahead);
#endif // RZ_UTF32_H

7
librz/include/rz_util/rz_utf8.h
View file

@ -4,11 +4,16 @@
/* For RzStrEnc definition */
#include "rz_unicode.h"
/**
* \brief Width of an UTF32 character in bytes.
*/
#define RZ_UTF8_CODE_POINT_WIDTH 1
/**
* \brief An Unicode code point.
*/
RZ_API int rz_utf8_encode(ut8 *ptr, const RzCodePoint ch);
RZ_API size_t rz_utf8_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULLABLE RZ_OUT RzCodePoint *cp);
RZ_API size_t rz_utf8_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULLABLE RZ_OUT RzCodePoint *cp, bool check_is_def);
RZ_API int rz_mutf8_decode(const ut8 *ptr, int ptrlen, RzCodePoint *ch);
RZ_API int rz_utf8_encode_str(const RzCodePoint *str, ut8 *dst, const int dst_length);
RZ_API int rz_utf8_size(const ut8 *ptr);

2
librz/main/rz-asm.c
View file

@ -451,7 +451,7 @@ static int rasm_asm(RzAsmState *as, const char *buf, ut64 offset, ut64 len, int
if (hexwords) {
size_t i = 0;
for (i = 0; i < acode->len; i += sizeof(ut32)) {
ut32 dword = rz_read_ble32(acode->bytes + i, RZ_SYS_ENDIAN);
ut32 dword = rz_read_ble32(acode->bytes + i, RZ_HOST_IS_BIG_ENDIAN);
printf("0x%08x ", dword);
if ((i / 4) == 7) {
printf("\n");

2
librz/search/bytes_search.c
View file

@ -220,7 +220,9 @@ static bool bytes_find(RzSearchFindOpt *fopts, void *user, ut64 address, const R
rz_pvector_foreach (patterns, it) {
RzSearchBytesPattern *hp = (RzSearchBytesPattern *)*it;
if (hp->regex) {
RzRegexMulti *re = rz_regex_multi_clone(hp->regex, true);
RzPVector *matches = fopts->match_overlap ? rz_regex_match_all_overlap(hp->regex, (const char *)raw_buf, size, 0, RZ_REGEX_DEFAULT) : rz_regex_match_all(hp->regex, (const char *)raw_buf, size, 0, RZ_REGEX_DEFAULT);
rz_regex_free_multi_clone(re);
void **it;
RzPVector *match;
rz_pvector_foreach (matches, it) {

4
librz/search/keyword.c
View file

@ -113,7 +113,7 @@ RZ_API RzSearchKeyword *rz_search_keyword_new_wide(const char *kwbuf, const char
str = malloc((len + 1) * 2);
for (p2 = kwbuf, p = str; *p2;) {
RzCodePoint ch;
int num_utf8_bytes = rz_utf8_decode((const ut8 *)p2, kwbuf + len - p2, &ch);
int num_utf8_bytes = rz_utf8_decode((const ut8 *)p2, kwbuf + len - p2, &ch, true);
if (num_utf8_bytes < 1) {
eprintf("WARNING: Malformed UTF8 at pos %td\n", p2 - kwbuf);
p[0] = *p2;
@ -125,7 +125,7 @@ RZ_API RzSearchKeyword *rz_search_keyword_new_wide(const char *kwbuf, const char
if (ignore_case && ch <= 0xff) {
ch = tolower(ch);
}
int num_wide_bytes = rz_utf16le_encode((ut8 *)p, ch);
int num_wide_bytes = rz_utf16_encode((ut8 *)p, ch, false);
rz_warn_if_fail(num_wide_bytes != 0);
p2 += num_utf8_bytes;
p += num_wide_bytes;

7
librz/search/options.c
View file

@ -102,6 +102,11 @@ RZ_API bool rz_search_opt_set_find_options(RZ_NONNULL RzSearchOpt *opt, RZ_OWN R
return true;
}
RZ_API const RzSearchFindOpt *rz_search_opt_get_find_options(RZ_NONNULL const RzSearchOpt *opt) {
rz_return_val_if_fail(opt, NULL);
return opt->find_opts;
}
RZ_API RZ_OWN RzSearchFindOpt *rz_search_find_opt_new() {
return RZ_NEW0(RzSearchFindOpt);
}
@ -138,7 +143,7 @@ RZ_API bool rz_search_find_opt_set_alignment(RZ_NONNULL RzSearchFindOpt *opt, si
return true;
}
RZ_API ut16 rz_search_find_opt_get_alignment(RZ_NONNULL RzSearchFindOpt *opt) {
RZ_API ut16 rz_search_find_opt_get_alignment(RZ_NONNULL const RzSearchFindOpt *opt) {
rz_return_val_if_fail(opt, 0);
return opt->alignment;
}

207
librz/search/string_search.c
View file

@ -21,32 +21,71 @@ typedef struct string_search {
* For example, if the real string is UTF-16 or UTF-32.
* Here we set the real (in memory encoded) string offsets and string length.
*/
static void align_offsets(RzUtilStrScanOptions options, RzStrEnc encoding, RzDetectedString *detected, RzRegexMatch *group0, ut64 *str_mem_offset, ut64 *str_mem_len, ut64 found_idx) {
if (rz_string_enc_is_utf8_compatible(encoding)) {
static void align_offsets(RzUtilStrScanOptions options, RzStrEnc encoding, RzDetectedString *detected, RzRegexMatch *group0, ut64 *str_mem_offset, ut64 *str_mem_len) {
if (rz_string_enc_same_char_width_as_utf8(encoding) || !detected->byte_mem_map) {
*str_mem_offset = detected->addr + group0->start;
*str_mem_len = group0->len;
return;
}
bool offset_found = false;
bool len_found = false;
*str_mem_offset = detected->byte_mem_map[group0->start];
*str_mem_len = detected->byte_mem_map[group0->start + group0->len] - *str_mem_offset;
}
*str_mem_offset = ht_uu_find(options.utf8_to_mem_offset_map, found_idx | (group0->start), &offset_found);
if (!offset_found) {
RZ_LOG_WARN("Could not determine memory offset of %s string in search. String offset will be off for: %s\n",
rz_str_enc_as_string(detected->type), detected->string);
*str_mem_offset = detected->addr + group0->start;
}
*str_mem_len = ht_uu_find(options.utf8_to_mem_offset_map, found_idx | (group0->start + group0->len), &len_found) - *str_mem_offset;
if (!len_found) {
if (!offset_found) {
// If the previous offset was not found, we know something is broken.
// If it was found on the other hand, the string is exactly as long as the whole buffer.
// So `start + len` is OOB and hence not in the hash table.
RZ_LOG_WARN("Could not determine length of string in memory. String length will be off.\n");
static bool native_string_find(RzSearchFindOpt *fopt, StringSearch *ss, ut64 offset, const RzBuffer *buffer,
RZ_OUT RzThreadQueue *hits, RZ_OUT size_t *n_hits) {
RzStrEnc encoding = ss->encoding;
ut64 size;
const ut8 *raw_buf = rz_buf_get_whole_hot_paths((RzBuffer *)buffer, &size);
void **it_m;
rz_pvector_foreach (ss->strings, it_m) {
RzDetectedString *find = *it_m;
RzPVector *matches = NULL;
RzRegexMulti *re = rz_regex_multi_clone(find->regex, true);
if (fopt->match_overlap) {
matches = rz_regex_match_all_overlap_multi(re, raw_buf, size, 0, RZ_REGEX_DEFAULT);
} else {
matches = rz_regex_match_all_multi(re, raw_buf, size, 0, RZ_REGEX_DEFAULT);
}
*str_mem_len = group0->len;
rz_regex_free_multi_clone(re);
if (!matches) {
return false;
}
void **it;
rz_pvector_foreach (matches, it) {
RzPVector *match = *it;
RzRegexMatch *group0 = rz_pvector_at(match, 0);
if (!group0) {
RZ_LOG_ERROR("search: Failed to get group of match.\n");
rz_pvector_free(matches);
return false;
}
ut64 str_mem_len = group0->len * rz_string_enc_code_point_width(encoding);
ut64 str_mem_offset = group0->start * rz_string_enc_code_point_width(encoding);
if (fopt->alignment > 1 && rz_mem_align_padding(str_mem_offset, fopt->alignment) != 0) {
// Match has not the correct alignment in memory.
continue;
}
if (find->alignment > 1 && rz_mem_align_padding(str_mem_offset, find->alignment) != 0) {
// Match has not the correct alignment in memory.
continue;
}
char hit_type[64] = { 0 };
rz_strf(hit_type, "string.%s", rz_str_enc_as_string(encoding));
RzSearchHit *hit = rz_search_hit_new(hit_type, str_mem_offset + offset, str_mem_len, NULL);
if (!hit || !rz_th_queue_push(hits, hit, true)) {
rz_search_hit_free(hit);
rz_pvector_free(matches);
return false;
}
(*n_hits)++;
}
rz_pvector_free(matches);
}
return true;
}
static bool string_find(RzSearchFindOpt *fopt, void *user, ut64 offset, const RzBuffer *buffer,
@ -54,6 +93,23 @@ static bool string_find(RzSearchFindOpt *fopt, void *user, ut64 offset, const Rz
rz_return_val_if_fail(fopt, false);
StringSearch *ss = (StringSearch *)user;
bool code_point_matches_alignment = rz_string_code_points_align(ss->encoding, fopt->alignment);
if (rz_string_enc_is_utf_native_endian(ss->encoding) &&
code_point_matches_alignment) {
// The expected encoding is UTF with native endian.
// For those we can do simple regex matching, skipping the whole decoding stuff.
return native_string_find(fopt, ss, offset, buffer, hits, n_hits);
}
// Everything below is the slow and resource extensive route to search strings.
// It will scan the whole buffer for strings, decoding each one with the
// correct encoding and length and match them.
// This costs a lot. So it is only done for strings with:
// A) A funny encodig we can't match directly with RzRegex/PCRE2 (e.g. EBCDIC).
// B) Encoding must be guessed.
// C) Matches can be at misaligned memory addresses
// (PCRE2 only matches strings aligned to their code point width).
RzDetectedString *detected = NULL;
RzListIter *it_s = NULL;
@ -67,47 +123,53 @@ static bool string_find(RzSearchFindOpt *fopt, void *user, ut64 offset, const Rz
// The search options are a shared resource and we might get
// race-conditions editing and freeing it.
RzUtilStrScanOptions options = ss->options;
options.utf8_to_mem_offset_map = ht_uu_new();
int n_str_in_buf = rz_scan_strings_whole_buf(buffer, found, &options, ss->encoding);
if (n_str_in_buf < 0) {
RZ_LOG_ERROR("Failed to scan buffer for strings.\n");
ht_uu_free(options.utf8_to_mem_offset_map);
rz_list_free(found);
return false;
}
ut64 found_idx = 0;
*n_hits = 0;
rz_list_foreach (found, it_s, detected) {
void **it_m = NULL;
rz_pvector_foreach (ss->strings, it_m) {
RzDetectedString *find = *it_m;
RzPVector *matches = fopt->match_overlap ? rz_regex_match_all_overlap(find->regex, detected->string, RZ_REGEX_ZERO_TERMINATED, 0, RZ_REGEX_DEFAULT) : rz_regex_match_all(find->regex, detected->string, RZ_REGEX_ZERO_TERMINATED, 0, RZ_REGEX_DEFAULT);
RzRegexMulti *re = rz_regex_multi_clone(find->regex, true);
RzPVector *matches = NULL;
if (fopt->match_overlap) {
matches = rz_regex_match_all_overlap(re->re8, detected->string, RZ_REGEX_ZERO_TERMINATED, 0, RZ_REGEX_DEFAULT);
} else {
matches = rz_regex_match_all(re->re8, detected->string, RZ_REGEX_ZERO_TERMINATED, 0, RZ_REGEX_DEFAULT);
}
rz_regex_free_multi_clone(re);
void **it;
rz_pvector_foreach (matches, it) {
RzPVector *match = *it;
RzRegexMatch *group0 = rz_pvector_at(match, 0);
if (!group0) {
RZ_LOG_ERROR("search: Failed to get group of match.\n");
ht_uu_free(options.utf8_to_mem_offset_map);
rz_list_free(found);
rz_pvector_free(matches);
return false;
}
ut64 str_mem_len;
ut64 str_mem_offset;
align_offsets(options, detected->type, detected, group0, &str_mem_offset, &str_mem_len, found_idx << 32);
if (fopt->alignment > 1 && rz_mem_align_padding(str_mem_offset + group0->start, fopt->alignment) != 0) {
align_offsets(options, detected->type, detected, group0, &str_mem_offset, &str_mem_len);
if (fopt->alignment > 1 && rz_mem_align_padding(str_mem_offset, fopt->alignment) != 0) {
// Match has not the correct alignment in memory.
continue;
}
char *hit_type = rz_str_newf("string.%s", rz_str_enc_as_string(detected->type));
if (find->alignment > 1 && rz_mem_align_padding(str_mem_offset, find->alignment) != 0) {
// Match has not the correct alignment in memory.
continue;
}
char hit_type[64] = { 0 };
rz_strf(hit_type, "string.%s", rz_str_enc_as_string(detected->type));
RzSearchHit *hit = rz_search_hit_new(hit_type, str_mem_offset + offset, str_mem_len, NULL);
free(hit_type);
if (!hit || !rz_th_queue_push(hits, hit, true)) {
rz_search_hit_free(hit);
ht_uu_free(options.utf8_to_mem_offset_map);
rz_list_free(found);
rz_pvector_free(matches);
return false;
@ -116,10 +178,8 @@ static bool string_find(RzSearchFindOpt *fopt, void *user, ut64 offset, const Rz
}
rz_pvector_free(matches);
}
found_idx++;
}
ht_uu_free(options.utf8_to_mem_offset_map);
rz_list_free(found);
return true;
}
@ -143,11 +203,10 @@ static void string_free(void *user) {
*
* \param opts The RzUtilStrScanOptions options to use
* \param[in] expected The expected encoding
* \param[in] flags The regex flags to the \p re_pattern.
*
* \return On success returns a valid pointer, otherwise NULL
*/
RZ_API RZ_OWN RzSearchCollection *rz_search_collection_strings(RZ_NONNULL RzUtilStrScanOptions *opts, RzStrEnc expected, RzRegexFlags flags) {
RZ_API RZ_OWN RzSearchCollection *rz_search_collection_strings(RZ_NONNULL RzUtilStrScanOptions *opts, RzStrEnc expected) {
rz_return_val_if_fail(opts, NULL);
StringSearch *ss = RZ_NEW0(StringSearch);
@ -169,13 +228,35 @@ RZ_API RZ_OWN RzSearchCollection *rz_search_collection_strings(RZ_NONNULL RzUtil
return rz_search_collection_new_bytes_space(string_find, string_is_empty, string_free, ss);
}
static RzDetectedString *setup_str_regex(const char *re_pattern, RzRegexFlags flags) {
static RzDetectedString *setup_str_regex(const char *re_pattern, RzRegexFlags cflags, RzStrEnc encoding) {
char *re_pattern_clone = rz_str_dup(re_pattern);
if (!re_pattern_clone) {
RZ_LOG_ERROR("Failed to clone regex pattern\n");
return NULL;
}
RzRegex *re = rz_regex_new(re_pattern, flags, RZ_REGEX_DEFAULT, NULL);
RzRegexMulti *re;
if (rz_string_enc_is_utf_native_endian(encoding)) {
switch (encoding) {
default:
rz_warn_if_reached();
return NULL;
case RZ_STRING_ENC_UTF8:
case RZ_STRING_ENC_8BIT:
re = rz_regex_new_multi(re_pattern, cflags, RZ_REGEX_DEFAULT, NULL, RZ_REGEX_UTF8);
break;
case RZ_STRING_ENC_UTF16LE:
case RZ_STRING_ENC_UTF16BE:
re = rz_regex_new_multi(re_pattern, cflags, RZ_REGEX_DEFAULT, NULL, RZ_REGEX_UTF16);
break;
case RZ_STRING_ENC_UTF32LE:
case RZ_STRING_ENC_UTF32BE:
re = rz_regex_new_multi(re_pattern, cflags, RZ_REGEX_DEFAULT, NULL, RZ_REGEX_UTF32);
break;
}
} else {
re = rz_regex_new_multi(re_pattern, cflags, RZ_REGEX_DEFAULT, NULL, RZ_REGEX_UTF8);
}
if (!re) {
RZ_LOG_ERROR("Failed to compile regex pattern: '%s'\n", re_pattern);
free(re_pattern_clone);
@ -197,13 +278,14 @@ static RzDetectedString *setup_str_regex(const char *re_pattern, RzRegexFlags fl
/**
* \brief Adds a new regex pattern into a string RzSearchCollection.
*
* \param[in] col The RzSearchCollection to use.
* \param[in] regex_pattern The regular expression to add.
* \param[in] flags The regular expression flags.
* \param[in] col The RzSearchCollection to use.
* \param[in] regex_pattern The regular expression to add.
* \param[in] cflags The regular expression compile flags.
* \param[in] match_alignment The memory address alignment all matches must have.
*
* \return On success returns true, otherwise false.
*/
RZ_API bool rz_search_collection_string_add(RZ_NONNULL RzSearchCollection *col, RZ_NONNULL const char *regex_pattern, RzRegexFlags flags) {
RZ_API bool rz_search_collection_string_add(RZ_NONNULL RzSearchCollection *col, RZ_NONNULL const char *regex_pattern, RzRegexFlags cflags, size_t match_alignment) {
rz_return_val_if_fail(col && regex_pattern, false);
if (!rz_search_collection_has_find_callback(col, string_find)) {
@ -215,11 +297,56 @@ RZ_API bool rz_search_collection_string_add(RZ_NONNULL RzSearchCollection *col,
}
StringSearch *ss = (StringSearch *)col->user;
RzDetectedString *s = setup_str_regex(regex_pattern, flags);
if (!s || !rz_pvector_push(ss->strings, s)) {
bool code_point_matches_alignment = rz_string_code_points_align(ss->encoding, match_alignment);
RzDetectedString *s = setup_str_regex(regex_pattern, cflags, code_point_matches_alignment ? ss->encoding : RZ_STRING_ENC_UTF8);
if (!s) {
return false;
}
s->alignment = match_alignment;
if (!rz_pvector_push(ss->strings, s)) {
RZ_LOG_ERROR("search: cannot add the string '%s'.\n", regex_pattern);
rz_detected_string_free(s);
return false;
}
return true;
}
/**
* \brief Checks the elements of a string search and warns the user about possible optimizations.
*
* \param col The string search collection.
* \param boundaries The search boundaries.
* \param search_options The search options.
* \param scan_opts The string scan options.
* \param If true, it will print suggestions to improve the search performance as warning.
*
* \return Returns true if the config is optional. False otherwise.
*/
RZ_API bool rz_search_collection_strings_check_config_improvements(
RZ_NULLABLE const RzSearchCollection *col,
RZ_NULLABLE const RzList /*<RzIOMap *>*/ *boundaries,
RZ_NULLABLE const RzSearchOpt *search_options,
RZ_NULLABLE const RzUtilStrScanOptions *scan_opt,
bool log_suggestions) {
if (!search_options || !search_options->find_opts || !col) {
return true;
}
StringSearch *ss = col->user;
if (ss->encoding == RZ_STRING_ENC_GUESS) {
if (log_suggestions) {
RZ_LOG_WARN("The string encoding for the search is set to \"guess\".\n"
"The search will consume vastly more resources and the guessing is unreliable.\n"
"You can set a specific encoding with 'e str.encoding=<encoding>'.\n");
}
return false;
}
if (!rz_string_code_points_align(ss->encoding, search_options->find_opts->alignment)) {
if (log_suggestions) {
RZ_LOG_INFO("The string encoding has code points of more than 1 byte. But search.align is set to 1.\n"
"The search will consume more resources, because alignment is not a multiple of the code point size.\n"
"For larger binaries consider to change the encoding to a multiple of 2 (UTF-16) or 4 (UTF-32).\n");
}
return false;
}
return true;
}

8
librz/util/buf.c
View file

@ -205,23 +205,23 @@ static st64 buf_format(RzBuffer *dst, RzBuffer *src, const char *fmt, int n) {
goto err_exit;
}
if (tok->big_endian != RZ_SYS_ENDIAN && tok->type_size > 1) {
if ((RZ_HOST_IS_BIG_ENDIAN != (bool)tok->big_endian) && tok->type_size > 1) {
// just swap endianness if the host endianness
// is not the same and is not one byte
switch (tok->type_size) {
case 2: {
ut16 value = rz_read_ble16(tmp, tok->big_endian);
rz_write_ble16(tmp, value, RZ_SYS_ENDIAN);
rz_write_ble16(tmp, value, RZ_HOST_IS_BIG_ENDIAN);
break;
}
case 4: {
ut32 value = rz_read_ble32(tmp, tok->big_endian);
rz_write_ble32(tmp, value, RZ_SYS_ENDIAN);
rz_write_ble32(tmp, value, RZ_HOST_IS_BIG_ENDIAN);
break;
}
case 8: {
ut64 value = rz_read_ble64(tmp, tok->big_endian);
rz_write_ble64(tmp, value, RZ_SYS_ENDIAN);
rz_write_ble64(tmp, value, RZ_HOST_IS_BIG_ENDIAN);
break;
}
default:

3
librz/util/ht/ht_up.c
View file

@ -29,7 +29,9 @@ static void init_options(HT_(Options) *opt, HT_(DupValue) valdup, HT_(FreeValue)
/**
* \brief Create a new hash table that has ut64 as key and void* as value.
* \param valdup Function to making copy of a value when inserting
* If NULL simple assignment operator is used for copy.
* \param valfree Function to releasing a stored value
* If NULL data is not freed.
*/
RZ_API RZ_OWN HtName_(Ht) *Ht_(new)(RZ_NULLABLE HT_(DupValue) valdup, RZ_NULLABLE HT_(FreeValue) valfree) {
HT_(Options) opt;
@ -42,6 +44,7 @@ RZ_API RZ_OWN HtName_(Ht) *Ht_(new)(RZ_NULLABLE HT_(DupValue) valdup, RZ_NULLABL
* with preallocated buckets for \p initial_size entries.
* \param initial_size Initial size of the hash table
* \param valdup Function to making copy of a value when inserting
* If NULL simple assignment operator is used for copy.
* \param valfree Function to releasing a stored value
*/
RZ_API RZ_OWN HtName_(Ht) *Ht_(new_size)(ut32 initial_size, RZ_NULLABLE HT_(DupValue) valdup, RZ_NULLABLE HT_(FreeValue) valfree) {

966
librz/util/regex.c
View file

File diff suppressed because it is too large Load diff

192
librz/util/str.c
View file

@ -2,10 +2,12 @@
// SPDX-License-Identifier: LGPL-3.0-only
#include <rz_util/rz_regex.h>
#include <rz_platform.h>
#include "rz_list.h"
#include "rz_types.h"
#include <rz_util.h>
#include "rz_cons.h"
#include "rz_util/rz_assert.h"
#include "rz_util/rz_unicode.h"
#include <rz_vector.h>
#include <stdio.h>
@ -1727,15 +1729,15 @@ static char *rz_str_escape_utf(const char *buf, int buf_size, RzStrEnc enc, cons
case RZ_STRING_ENC_UTF32LE:
case RZ_STRING_ENC_UTF32BE:
if (enc == RZ_STRING_ENC_UTF16LE || enc == RZ_STRING_ENC_UTF16BE) {
ch_bytes = rz_utf16_decode((ut8 *)p, end - p, &ch, enc == RZ_STRING_ENC_UTF16BE);
ch_bytes = rz_utf16_decode((ut8 *)p, end - p, &ch, true, enc == RZ_STRING_ENC_UTF16BE);
min_char_width = 2;
} else {
ch_bytes = rz_utf32_decode((ut8 *)p, end - p, &ch, enc == RZ_STRING_ENC_UTF32BE);
ch_bytes = rz_utf32_decode((ut8 *)p, end - p, &ch, true, enc == RZ_STRING_ENC_UTF32BE);
min_char_width = 4;
}
break;
default:
ch_bytes = rz_utf8_decode((ut8 *)p, end - p, &ch);
ch_bytes = rz_utf8_decode((ut8 *)p, end - p, &ch, true);
min_char_width = 1;
}
if (!rz_str_escape_code_point(ch, ch_bytes, esc_opts)) {
@ -1844,7 +1846,7 @@ static char *escape_utf8_for_json(const char *buf, int buf_size, bool mutf8) {
q = new_buf;
while (p < end) {
ptrdiff_t bytes_left = end - p;
ch_bytes = mutf8 ? rz_mutf8_decode(p, bytes_left, &ch) : rz_utf8_decode(p, bytes_left, &ch);
ch_bytes = mutf8 ? rz_mutf8_decode(p, bytes_left, &ch) : rz_utf8_decode(p, bytes_left, &ch, true);
if (ch_bytes == 1) {
switch (*p) {
case '\n':
@ -2129,7 +2131,7 @@ RZ_API bool rz_str_is_utf8(RZ_NONNULL const char *str) {
const ut8 *ptr = (const ut8 *)str;
size_t len = strlen(str);
while (len) {
int bytes = rz_utf8_decode(ptr, len, NULL);
int bytes = rz_utf8_decode(ptr, len, NULL, true);
if (!bytes) {
return false;
}
@ -2141,7 +2143,7 @@ RZ_API bool rz_str_is_utf8(RZ_NONNULL const char *str) {
RZ_API bool rz_str_is_printable(const char *str) {
while (*str) {
int ulen = rz_utf8_decode((const ut8 *)str, strlen(str), NULL);
int ulen = rz_utf8_decode((const ut8 *)str, strlen(str), NULL, true);
if (ulen > 1) {
str += ulen;
continue;
@ -2156,7 +2158,7 @@ RZ_API bool rz_str_is_printable(const char *str) {
RZ_API bool rz_str_is_printable_limited(const char *str, int size) {
while (size > 0 && *str) {
int ulen = rz_utf8_decode((const ut8 *)str, strlen(str), NULL);
int ulen = rz_utf8_decode((const ut8 *)str, strlen(str), NULL, true);
if (ulen > 1) {
str += ulen;
continue;
@ -2172,7 +2174,7 @@ RZ_API bool rz_str_is_printable_limited(const char *str, int size) {
RZ_API bool rz_str_is_printable_incl_newlines(const char *str) {
while (*str) {
int ulen = rz_utf8_decode((const ut8 *)str, strlen(str), NULL);
int ulen = rz_utf8_decode((const ut8 *)str, strlen(str), NULL, true);
if (ulen > 1) {
str += ulen;
continue;
@ -2824,6 +2826,55 @@ RZ_API size_t rz_str_len_utf8(const char *s) {
return j + fullwidths;
}
/**
* \brief Counts the number of UTF-8 encoded
* Unicode code points in the given string.
*
* \return The number of Unicode code points *including* the final NUL.
*/
RZ_API size_t rz_str_utf8_num_ucp(RZ_NONNULL const char *str) {
rz_return_val_if_fail(str, 0);
size_t i = 0, char_cnt = 0;
while (str[i]) {
if ((str[i] & 0xc0) != 0x80) {
char_cnt++;
}
i++;
}
return char_cnt + 1;
}
/**
* \brief Determines the number of bytes required to encode the given UTF-8
* string into an UTF-16 string.
*
* \return The number of bytes required for an UTF16 string, *including* the final NUL.
*/
RZ_API size_t rz_str_utf8_get_width_utf16(RZ_NONNULL const char *str) {
rz_return_val_if_fail(str, 0);
size_t i = 0, byte_cnt = 0, extend_cnt = 0;
while (str[i]) {
if ((str[i] & 0xc0) != 0x80) {
extend_cnt = 0;
byte_cnt += 2;
i++;
continue;
}
// Check if code point is >= 0x10000
extend_cnt++;
if (extend_cnt == 3) {
RzCodePoint cp = 0;
rz_utf8_decode((ut8 *)str + (i - 3), 4, &cp, false);
if (cp >= RZ_UTF16_FIRST_4BYTES_CODE_POINT) {
byte_cnt += 2; // Add the additional two bytes needed.
}
extend_cnt = 0;
}
i++;
}
return byte_cnt + 2; // NUL terminator
}
RZ_API size_t rz_str_len_utf8_ansi(const char *str) {
int i = 0, len = 0, fullwidths = 0;
while (str[i]) {
@ -3092,8 +3143,65 @@ RZ_API char *rz_str_utf16_decode(const ut8 *s, int len) {
return result;
}
/**
* \brief Converts an UTF-8 string to an UTF-16 string of the
* requested endianess.
* If the \p utf8_str contains invalid Unicode code points, the new string
* will end at the first invalid one.
*
* \param utf8_str The UTF-8 encoded string.
* \param big_endian If true the returned UTF-16 string will be in big endian.
*
* \return The NUL terminated UTF-16 string or NULL in case of failure.
*/
RZ_API RZ_OWN ut16 *rz_str_utf8_to_utf16(RZ_NONNULL const char *utf8_str, bool big_endian) {
rz_return_val_if_fail(utf8_str, NULL);
size_t utf16_len = rz_str_utf8_get_width_utf16(utf8_str);
ut8 *utf16_str = RZ_NEWS0(ut8, utf16_len);
size_t utf16_idx = 0;
RzCodePoint ucp;
size_t char_width = 1;
size_t utf8_size = strlen(utf8_str) + 1;
for (size_t i = 0; i < utf8_size; i += char_width) {
if (!(char_width = rz_utf8_decode((ut8 *)utf8_str + i, utf8_size - i, &ucp, true))) {
break;
}
utf16_idx += rz_utf16_encode(utf16_str + utf16_idx, ucp, big_endian);
}
return (ut16 *)utf16_str;
}
/**
* \brief Converts an UTF-8 string to an UTF-32 string of the
* requested endianess.
* If the \p utf8_str contains invalid Unicode code points, the new string
* will end at the first invalid one.
*
* \param utf8_str The UTF-8 encoded string.
* \param big_endian If true the returned UTF-32 string will be in big endian.
*
* \return The NUL terminated UTF-32 string or NULL in case of failure.
*/
RZ_API RZ_OWN ut32 *rz_str_utf8_to_utf32(RZ_NONNULL const char *utf8_str, bool big_endian) {
rz_return_val_if_fail(utf8_str, NULL);
size_t utf32_len = rz_str_utf8_num_ucp(utf8_str) * RZ_UTF32_WIDTH_CHAR;
ut8 *utf32_str = RZ_NEWS0(ut8, utf32_len);
size_t utf32_idx = 0;
RzCodePoint ucp;
size_t char_width = 1;
size_t utf8_size = strlen(utf8_str) + 1;
for (size_t i = 0; i < utf8_size; i += char_width) {
if (!(char_width = rz_utf8_decode((ut8 *)utf8_str + i, utf8_size - i, &ucp, true))) {
break;
}
utf32_idx += rz_utf32_encode(utf32_str + utf32_idx, ucp, big_endian);
}
return (ut32 *)utf32_str;
}
// TODO: kill this completely, it makes no sense:
RZ_API char *rz_str_utf16_encode(const char *s, int len) {
// Even better, rewrite with the rz_utf16_encode() functions.
RZ_DEPRECATE RZ_API char *rz_str_utf16_encode(const char *s, int len) {
int i;
char ch[4], *d, *od, *tmp;
if (!s) {
@ -4155,22 +4263,22 @@ RZ_API RZ_OWN char *rz_str_stringify_raw_buffer(RzStrStringifyOpt *option, RZ_NU
rz_strbuf_init(&sb);
for (ut32 i = 0, line_runes = 0; i < buflen; i += rsize) {
if (enc == RZ_STRING_ENC_UTF32LE) {
rsize = rz_utf32le_decode(&buf[i], buflen - i, &code_point);
rsize = rz_utf32le_decode(&buf[i], buflen - i, &code_point, true);
if (rsize) {
rsize = 4;
}
} else if (enc == RZ_STRING_ENC_UTF16LE) {
rsize = rz_utf16le_decode(&buf[i], buflen - i, &code_point);
rsize = rz_utf16le_decode(&buf[i], buflen - i, &code_point, true);
if (rsize == 1) {
rsize = 2;
}
} else if (enc == RZ_STRING_ENC_UTF32BE) {
rsize = rz_utf32be_decode(&buf[i], buflen - i, &code_point);
rsize = rz_utf32be_decode(&buf[i], buflen - i, &code_point, true);
if (rsize) {
rsize = 4;
}
} else if (enc == RZ_STRING_ENC_UTF16BE) {
rsize = rz_utf16be_decode(&buf[i], buflen - i, &code_point);
rsize = rz_utf16be_decode(&buf[i], buflen - i, &code_point, true);
if (rsize == 1) {
rsize = 2;
}
@ -4188,7 +4296,7 @@ RZ_API RZ_OWN char *rz_str_stringify_raw_buffer(RzStrStringifyOpt *option, RZ_NU
code_point = buf[i];
rsize = code_point < 0x7F ? 1 : 0;
} else {
rsize = rz_utf8_decode(&buf[i], buflen - i, &code_point);
rsize = rz_utf8_decode(&buf[i], buflen - i, &code_point, true);
}
if (rsize == 0) {
@ -4331,3 +4439,59 @@ RZ_API const char *rz_str_indent(int indent) {
}
return indent_tbl[indent];
}
/**
* \brief Checks given encoding if it is UTF-8, UTF-16, or UTF-32
* of the host's endianness.
*
* \return true For UTF-8/ASCII.
* \return true For UTF-16-LE/UTF-32-LE if Rizin was built for a little endian architecture.
* \return true For UTF-16-BB/UTF-32-BB if Rizin was built for a big endian architecture.
* \return false Otherwise.
*/
RZ_API bool rz_string_enc_is_utf_native_endian(RzStrEnc enc) {
switch (enc) {
default:
return false;
case RZ_STRING_ENC_8BIT:
case RZ_STRING_ENC_UTF8:
return true;
case RZ_STRING_ENC_UTF16LE:
case RZ_STRING_ENC_UTF32LE:
return RZ_HOST_IS_LITTLE_ENDIAN;
case RZ_STRING_ENC_UTF16BE:
case RZ_STRING_ENC_UTF32BE:
return RZ_HOST_IS_BIG_ENDIAN;
}
}
/**
* \brief Returns the size of the code point in bytes.
* UTF-8 = 1, UTF-16 = 2, UTF-32 = 4 etc.
*
* \return Size of code point in bytes or 0 if given encoding is invalid.
*/
RZ_API size_t rz_string_enc_code_point_width(RzStrEnc enc) {
switch (enc) {
default:
case RZ_STRING_ENC_GUESS:
case RZ_STRING_ENC_BASE64:
case RZ_STRING_ENC_SETTINGS:
return 0;
case RZ_STRING_ENC_8BIT:
case RZ_STRING_ENC_UTF8:
case RZ_STRING_ENC_MUTF8:
case RZ_STRING_ENC_IBM037:
case RZ_STRING_ENC_IBM290:
case RZ_STRING_ENC_EBCDIC_UK:
case RZ_STRING_ENC_EBCDIC_US:
case RZ_STRING_ENC_EBCDIC_ES:
return 1;
case RZ_STRING_ENC_UTF16LE:
case RZ_STRING_ENC_UTF16BE:
return 2;
case RZ_STRING_ENC_UTF32LE:
case RZ_STRING_ENC_UTF32BE:
return 4;
}
}

74
librz/util/str_search.c
View file

@ -80,7 +80,8 @@ RZ_API void rz_detected_string_free(RzDetectedString *str) {
return;
}
free(str->string);
rz_regex_free(str->regex);
free(str->byte_mem_map);
rz_regex_free_multi(str->regex);
free(str);
}
@ -99,7 +100,7 @@ static UTF8StringInfo calculate_utf8_string_info(ut8 *str, int size) {
const ut8 *str_end = str + size;
RzCodePoint ch = 0;
while (str_ptr < str_end) {
int ch_bytes = rz_utf8_decode(str_ptr, str_end - str_ptr, &ch);
int ch_bytes = rz_utf8_decode(str_ptr, str_end - str_ptr, &ch, true);
if (!ch_bytes) {
break;
}
@ -230,8 +231,17 @@ static inline size_t buf_look_ahead(const RzUtilStrScanOptions *opt, RzStrEnc en
}
}
/**
* \brief Number of characters to store on the stack during scanning.
* If the scanned string has more characters than this or is valid
* it is copied to the heap.
* Used to save unnecessary memory allocations.
*/
#define SCANNING_STACK_BUF_CHARS 16
#define SCANNING_STACK_BUF_SIZE (RZ_UNICODE_MAX_BYTES_PER_CHAR * SCANNING_STACK_BUF_CHARS)
static RzDetectedString *process_one_string(const ut8 *buf, const ut64 from, ut64 needle, const ut64 to,
RzStrEnc str_type, bool ascii_only, const RzUtilStrScanOptions *opt, ut64 str_list_idx, bool test_false_positives) {
RzStrEnc str_type, bool ascii_only, const RzUtilStrScanOptions *opt, bool test_false_positives) {
rz_return_val_if_fail(str_type != RZ_STRING_ENC_GUESS, NULL);
size_t look_ahead = buf_look_ahead(opt, str_type);
if (look_ahead == 0) {
@ -241,10 +251,12 @@ static RzDetectedString *process_one_string(const ut8 *buf, const ut64 from, ut6
// Most calls to this function never produce a valid string (e.g. because they are too short).
// To save allocations and frees, we first decode the first few code points onto the stack.
// Then, if the stack buffer is full, we move it to the heap.
ut8 stack_alloc[RZ_UNICODE_MAX_BYTES_PER_CHAR * 5] = { 0 };
ut8 stack_alloc[SCANNING_STACK_BUF_SIZE] = { 0 };
// Gets only set if the stack buffer is full.
ut8 *heap_alloc = NULL;
ut8 *output_buf = stack_alloc;
ut64 *byte_mem_map = NULL;
size_t byte_mem_map_size = 0;
ut64 str_addr = needle;
// Bytes of a decoded/encoded character/code point.
@ -252,6 +264,7 @@ static RzDetectedString *process_one_string(const ut8 *buf, const ut64 from, ut6
// Counter of correctly decoded characters/code points.
int char_count = 0;
int i = 0;
bool stopped_with_undef_cp = false;
/* Eat a whole C string */
for (i = 0; i < opt->max_str_length - look_ahead && needle < to; i += char_bytes) {
@ -259,16 +272,16 @@ static RzDetectedString *process_one_string(const ut8 *buf, const ut64 from, ut6
switch (str_type) {
case RZ_STRING_ENC_UTF32LE:
char_bytes = rz_utf32le_decode(buf + needle - from, to - needle, &r);
char_bytes = rz_utf32le_decode(buf + needle - from, to - needle, &r, false);
break;
case RZ_STRING_ENC_UTF16LE:
char_bytes = rz_utf16le_decode(buf + needle - from, to - needle, &r);
char_bytes = rz_utf16le_decode(buf + needle - from, to - needle, &r, false);
break;
case RZ_STRING_ENC_UTF32BE:
char_bytes = rz_utf32be_decode(buf + needle - from, to - needle, &r);
char_bytes = rz_utf32be_decode(buf + needle - from, to - needle, &r, false);
break;
case RZ_STRING_ENC_UTF16BE:
char_bytes = rz_utf16be_decode(buf + needle - from, to - needle, &r);
char_bytes = rz_utf16be_decode(buf + needle - from, to - needle, &r, false);
break;
case RZ_STRING_ENC_IBM037:
char_bytes = rz_str_ibm037_to_unicode(*(buf + needle - from), &r);
@ -290,7 +303,7 @@ static RzDetectedString *process_one_string(const ut8 *buf, const ut64 from, ut6
RZ_LOG_ERROR("Illegal state reached. 'settings' encoding is not a valid value here.\n");
return NULL;
default:
char_bytes = rz_utf8_decode(buf + needle - from, to - needle, &r);
char_bytes = rz_utf8_decode(buf + needle - from, to - needle, &r, false);
if (char_bytes > 1) {
str_type = RZ_STRING_ENC_UTF8;
look_ahead = buf_look_ahead(opt, RZ_STRING_ENC_UTF8);
@ -304,9 +317,17 @@ static RzDetectedString *process_one_string(const ut8 *buf, const ut64 from, ut6
break;
}
if (opt->utf8_to_mem_offset_map && !rz_string_enc_is_utf8_compatible(str_type)) {
ut64 offset_id = ((str_list_idx) << 32) | i;
ht_uu_insert(opt->utf8_to_mem_offset_map, offset_id, needle);
if (!rz_string_enc_same_char_width_as_utf8(str_type)) {
size_t utf8_char_offset = i;
if (!byte_mem_map) {
byte_mem_map = RZ_NEWS0(ut64, SCANNING_STACK_BUF_SIZE);
byte_mem_map_size += SCANNING_STACK_BUF_SIZE;
} else if (utf8_char_offset >= byte_mem_map_size) {
byte_mem_map = realloc(byte_mem_map, (byte_mem_map_size + SCANNING_STACK_BUF_SIZE) * sizeof(ut64));
byte_mem_map_size += SCANNING_STACK_BUF_SIZE;
}
size_t mem_offset = needle;
byte_mem_map[utf8_char_offset] = mem_offset;
}
needle += char_bytes;
@ -334,7 +355,8 @@ static RzDetectedString *process_one_string(const ut8 *buf, const ut64 from, ut6
}
char_count++;
} else {
/* \0 marks the end of C-strings */
/* \0 or undefined code point marks the end of C-strings */
stopped_with_undef_cp = r != 0;
break;
}
}
@ -347,7 +369,7 @@ static RzDetectedString *process_one_string(const ut8 *buf, const ut64 from, ut6
goto error;
} else if (false_positive_result == RETRY_ASCII) {
free(heap_alloc);
return process_one_string(buf, from, str_addr, to, str_type, true, opt, str_list_idx, false);
return process_one_string(buf, from, str_addr, to, str_type, true, opt, false);
}
}
@ -358,7 +380,14 @@ static RzDetectedString *process_one_string(const ut8 *buf, const ut64 from, ut6
ds->type = str_type;
ds->length = char_count;
ds->size = needle - str_addr;
if (stopped_with_undef_cp) {
// The decoding stops if a byte sequence is an undefined unicode code point.
// This last undefined code point still increments needle by its code point width.
// Subtract it again, so we don't have it in the string length.
ds->size -= char_bytes;
}
ds->addr = str_addr;
ds->byte_mem_map = byte_mem_map;
ut64 off_adj = adjust_offset(str_type, buf, ds->addr - from);
ds->addr -= off_adj;
@ -369,12 +398,13 @@ static RzDetectedString *process_one_string(const ut8 *buf, const ut64 from, ut6
}
error:
free(byte_mem_map);
free(heap_alloc);
return NULL;
}
static inline bool can_be_utf16_le(const ut8 *buf, ut64 size) {
int rc = rz_utf8_decode(buf, size, NULL);
int rc = rz_utf8_decode(buf, size, NULL, true);
if (!rc || (size - rc) < 5) {
return false;
}
@ -390,7 +420,7 @@ static inline bool can_be_utf16_be(const ut8 *buf, ut64 size) {
}
static inline bool can_be_utf32_le(const ut8 *buf, ut64 size) {
int rc = rz_utf8_decode(buf, size, NULL);
int rc = rz_utf8_decode(buf, size, NULL, true);
if (!rc || (size - rc) < 5) {
return false;
}
@ -480,8 +510,8 @@ RZ_API int rz_scan_strings_raw(RZ_NONNULL const ut8 *buf, RZ_NONNULL RzList /*<R
} else if (can_be_utf32_be(ptr, size)) {
if (to - needle > 3 && can_be_utf32_le(ptr + 3, size - 3)) {
// The string can be either utf32-le or utf32-be
RzDetectedString *ds_le = process_one_string(buf, from, needle + 3, to, RZ_STRING_ENC_UTF32LE, false, opt, rz_list_length(list), false);
RzDetectedString *ds_be = process_one_string(buf, from, needle, to, RZ_STRING_ENC_UTF32BE, false, opt, rz_list_length(list), false);
RzDetectedString *ds_le = process_one_string(buf, from, needle + 3, to, RZ_STRING_ENC_UTF32LE, false, opt, false);
RzDetectedString *ds_be = process_one_string(buf, from, needle, to, RZ_STRING_ENC_UTF32BE, false, opt, false);
RzDetectedString *to_add = NULL;
RzDetectedString *to_delete = NULL;
@ -516,8 +546,8 @@ RZ_API int rz_scan_strings_raw(RZ_NONNULL const ut8 *buf, RZ_NONNULL RzList /*<R
} else if (can_be_utf16_be(ptr, size)) {
if (to - needle > 1 && can_be_utf16_le(ptr + 1, size - 1)) {
// The string can be either utf16-le or utf16-be
RzDetectedString *ds_le = process_one_string(buf, from, needle + 1, to, RZ_STRING_ENC_UTF16LE, false, opt, rz_list_length(list), false);
RzDetectedString *ds_be = process_one_string(buf, from, needle, to, RZ_STRING_ENC_UTF16BE, false, opt, rz_list_length(list), false);
RzDetectedString *ds_le = process_one_string(buf, from, needle + 1, to, RZ_STRING_ENC_UTF16LE, false, opt, false);
RzDetectedString *ds_be = process_one_string(buf, from, needle, to, RZ_STRING_ENC_UTF16BE, false, opt, false);
RzDetectedString *to_add = NULL;
RzDetectedString *to_delete = NULL;
@ -567,7 +597,7 @@ RZ_API int rz_scan_strings_raw(RZ_NONNULL const ut8 *buf, RZ_NONNULL RzList /*<R
continue;
}
} else {
int rc = rz_utf8_decode(ptr, size, NULL);
int rc = rz_utf8_decode(ptr, size, NULL, false);
if (!rc) {
needle++;
continue;
@ -580,7 +610,7 @@ RZ_API int rz_scan_strings_raw(RZ_NONNULL const ut8 *buf, RZ_NONNULL RzList /*<R
str_type = RZ_STRING_ENC_8BIT;
}
RzDetectedString *ds = process_one_string(buf, from, needle, to, str_type, false, opt, rz_list_length(list), test_false_positives);
RzDetectedString *ds = process_one_string(buf, from, needle, to, str_type, false, opt, test_false_positives);
if (!ds) {
needle++;
continue;

76
librz/util/utf16.c
View file

@ -25,14 +25,19 @@ static RzCodePoint utf16_surrogate_to_codepoint(ut16 high_surrogate, ut16 low_su
*
* \param buf The buffer to read the bytes from.
* \param buf_len The buffer length.
* \param ch The decoded code point. It is only written if a valid
* Unicode code point was decoded.
* \param codepoint (Optional) The decoded code point.
* \param check_is_def If true, checks the code point against the defined
* Unicode table. It will not write \p cp and return 0 if the decoded code
* point is undefined.
* If false, it won't perform any checks and just decode.
* Be aware, the check has a runtime of O(log n).
* Where n: number of undefined Unicode ranges.
* \param bigendian Flag if the \p buf holds UTF-16 bytes in big endian.
*
* \return Number of bytes decoded.
*/
RZ_API size_t rz_utf16_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NONNULL RZ_OUT RzCodePoint *ch, bool bigendian) {
rz_return_val_if_fail(buf && ch, 0);
RZ_API size_t rz_utf16_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULLABLE RZ_OUT RzCodePoint *codepoint, bool check_is_def, bool bigendian) {
rz_return_val_if_fail(buf, 0);
if (buf_len <= 1) {
return 0;
}
@ -54,10 +59,12 @@ RZ_API size_t rz_utf16_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NONN
bytes_used = 2;
check_assign:
if (!rz_unicode_code_point_is_legal_decode(cp)) {
if (rz_unicode_code_point_is_surrogate(cp) || (check_is_def && !rz_unicode_code_point_is_defined(cp))) {
return 0;
}
*ch = cp;
if (codepoint) {
*codepoint = cp;
}
return bytes_used;
}
@ -66,13 +73,19 @@ check_assign:
*
* \param buf The buffer to read the bytes from.
* \param buf_len The buffer length.
* \param codepoint The decoded code point.
* \param codepoint (Optional) The decoded code point.
* \param check_is_def If true, checks the code point against the defined
* Unicode table. It will not write \p cp and return 0 if the decoded code
* point is undefined.
* If false, it won't perform any checks and just decode.
* Be aware, the check has a runtime of O(log n).
* Where n: number of undefined Unicode ranges.
*
* \return Number of bytes decoded.
*/
RZ_API size_t rz_utf16le_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NONNULL RZ_OUT RzCodePoint *codepoint) {
rz_return_val_if_fail(buf && codepoint, 0);
return rz_utf16_decode(buf, buf_len, codepoint, false);
RZ_API size_t rz_utf16le_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULLABLE RZ_OUT RzCodePoint *codepoint, bool check_is_def) {
rz_return_val_if_fail(buf, 0);
return rz_utf16_decode(buf, buf_len, codepoint, check_is_def, false);
}
/**
@ -80,25 +93,50 @@ RZ_API size_t rz_utf16le_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NO
*
* \param buf The buffer to read the bytes from.
* \param buf_len The buffer length.
* \param codepoint The decoded code point.
* \param codepoint (Optional) The decoded code point.
* \param check_is_def If true, checks the code point against the defined
* Unicode table. It will not write \p cp and return 0 if the decoded code
* point is undefined.
* If false, it won't perform any checks and just decode.
* Be aware, the check has a runtime of O(log n).
* Where n: number of undefined Unicode ranges.
*
* \return Number of bytes decoded.
*/
RZ_API size_t rz_utf16be_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NONNULL RZ_OUT RzCodePoint *codepoint) {
rz_return_val_if_fail(buf && codepoint, 0);
return rz_utf16_decode(buf, buf_len, codepoint, true);
RZ_API size_t rz_utf16be_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULLABLE RZ_OUT RzCodePoint *codepoint, bool check_is_def) {
rz_return_val_if_fail(buf, 0);
return rz_utf16_decode(buf, buf_len, codepoint, check_is_def, true);
}
/**
* \brief Encodes a Unicode code point to little endian UTF16 bytes.
*
* \param buf The buffer to write the bytes to. Must be at least 4 bytes.
* \param codepoint The code point to encode.
* \param buf The buffer to write the bytes to. Must be at least 4 bytes.
* \param codepoint The code point to encode.
* \param big_endian Encodes in big endian order if set.
*
* \return Number of bytes encoded.
*/
RZ_API size_t rz_utf16le_encode(RZ_NONNULL RZ_OUT ut8 *buf, RzCodePoint codepoint) {
RZ_API size_t rz_utf16_encode(RZ_NONNULL RZ_OUT ut8 *buf, RzCodePoint codepoint, bool big_endian) {
rz_return_val_if_fail(buf, 0);
if (big_endian) {
if (codepoint < 0x10000) {
buf[1] = codepoint & 0xff;
buf[0] = codepoint >> 8 & 0xff;
return 2;
}
if (codepoint > 0x10FFFF) {
return 0;
}
codepoint -= 0x10000;
RzCodePoint high = 0xd800 + ((codepoint >> 10) & 0x3ff);
RzCodePoint low = 0xdc00 + (codepoint & 0x3ff);
buf[1] = high & 0xff;
buf[0] = high >> 8 & 0xff;
buf[3] = low & 0xff;
buf[2] = low >> 8 & 0xff;
return 4;
}
if (codepoint < 0x10000) {
buf[0] = codepoint & 0xff;
buf[1] = codepoint >> 8 & 0xff;
@ -130,7 +168,7 @@ RZ_API size_t rz_utf16le_encode(RZ_NONNULL RZ_OUT ut8 *buf, RzCodePoint codepoin
* \param buf_len The buffer length.
* \param big_endian Should be set if the bytes in the buffer are in big endian order.
* \param lookahead Number of code points to check.
* Note: if the buffer can't cover all \p lookahead code points, this returns false.
* NOTE: if the buffer can't cover all \p lookahead code points, this returns false.
*
* \return True if the buffer has \p lookahead number of printable UTF-16 characters.
* \return False otherwise.
@ -145,7 +183,7 @@ RZ_API bool rz_utf16_is_printable_code_point(RZ_NONNULL const ut8 *buf, size_t b
size_t offset = 0;
RzCodePoint cp = 0;
while (lookahead > 0) {
size_t dec_bytes = rz_utf16_decode(buf + offset, buf_len - offset, &cp, big_endian);
size_t dec_bytes = rz_utf16_decode(buf + offset, buf_len - offset, &cp, true, big_endian);
if (!rz_unicode_code_point_is_printable(cp) || dec_bytes == 0) {
return false;
}

51
librz/util/utf32.c
View file

@ -9,13 +9,18 @@
*
* \param buf The buffer to read from.
* \param buf_len The buffer size in bytes.
* \param ch The decoded code point. It is only written if a valid
* Unicode code point was decoded.
* \param cp The decoded code point.
* \param check_is_def If true, checks the code point against the defined
* Unicode table. It will not write \p cp and return 0 if the decoded code
* point is undefined.
* If false, it won't perform any checks and just decode.
* Be aware, the check has a runtime of O(log n).
* Where n: number of undefined Unicode ranges.
* \param big_endian If the buffer bytes have big endian order.
*
* \return The number of bytes converted. For UTF-32 this is always 0 or 4.
*/
RZ_API size_t rz_utf32_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULLABLE RZ_OUT RzCodePoint *ch, bool big_endian) {
RZ_API size_t rz_utf32_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULLABLE RZ_OUT RzCodePoint *ch, bool check_is_def, bool big_endian) {
rz_return_val_if_fail(buf, 0);
if (buf_len < 4) {
return 0;
@ -24,7 +29,7 @@ RZ_API size_t rz_utf32_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULL
return 4;
}
RzCodePoint cp = rz_read_ble32(buf, big_endian);
if (!rz_unicode_code_point_is_legal_decode(cp)) {
if (rz_unicode_code_point_is_surrogate(cp) || (check_is_def && !rz_unicode_code_point_is_defined(cp))) {
return 0;
}
*ch = cp;
@ -32,13 +37,13 @@ RZ_API size_t rz_utf32_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULL
}
/* Convert an UTF-32LE buf into a unicode RzCodePoint */
RZ_API int rz_utf32le_decode(const ut8 *ptr, int ptrlen, RzCodePoint *ch) {
return rz_utf32_decode(ptr, ptrlen, ch, false);
RZ_API int rz_utf32le_decode(const ut8 *ptr, int ptrlen, RZ_NULLABLE RZ_OUT RzCodePoint *ch, bool check_is_def) {
return rz_utf32_decode(ptr, ptrlen, ch, check_is_def, false);
}
/* Convert an UTF-32BE buf into a unicode RzCodePoint */
RZ_API int rz_utf32be_decode(const ut8 *ptr, int ptrlen, RzCodePoint *ch) {
return rz_utf32_decode(ptr, ptrlen, ch, true);
RZ_API int rz_utf32be_decode(const ut8 *ptr, int ptrlen, RZ_NULLABLE RZ_OUT RzCodePoint *ch, bool check_is_def) {
return rz_utf32_decode(ptr, ptrlen, ch, check_is_def, true);
}
/**
@ -50,7 +55,7 @@ RZ_API int rz_utf32be_decode(const ut8 *ptr, int ptrlen, RzCodePoint *ch) {
* \param buf_len The buffer length.
* \param big_endian Should be set if the bytes in the buffer are in big endian order.
* \param lookahead Number of code points to check.
* Note: if the buffer can't cover all \p lookahead code points, this returns false.
* NOTE: if the buffer can't cover all \p lookahead code points, this returns false.
*
* \return True if the buffer has \p lookahead valid UTF-32 code points.
* \return False otherwise.
@ -77,3 +82,31 @@ RZ_API bool rz_utf32_valid_code_point(RZ_NONNULL const ut8 *buf, size_t buf_len,
}
return true;
}
/**
* \brief Encodes the Unicode code point \p ucp into \p buf.
*
* \param buf The buffer to write the UTF-32 character into.
* The buffer must be at least 4 bytes in size.
* \param ucp The Unicode code point to encode.
* \param big_endian If true it will encode \p ucp as a big endian character. If false, as little endian.
*
* \return Number of bytes written into \p buf. 0 in case of failure, 4 otherwise.
*/
RZ_API size_t rz_utf32_encode(RZ_NONNULL RZ_OUT ut8 *buf, RzCodePoint ucp, bool big_endian) {
if (ucp > RZ_UNICODE_LAST_CODE_POINT || rz_unicode_code_point_is_surrogate(ucp)) {
return 0;
}
if (big_endian) {
buf[3] = ucp & 0xff;
buf[2] = (ucp >> 8) & 0xff;
buf[1] = (ucp >> 16) & 0xff;
buf[0] = (ucp >> 24) & 0xff;
return 4;
}
buf[0] = ucp & 0xff;
buf[1] = (ucp >> 8) & 0xff;
buf[2] = (ucp >> 16) & 0xff;
buf[3] = (ucp >> 24) & 0xff;
return 4;
}

17
librz/util/utf8.c
View file

@ -369,12 +369,17 @@ RZ_API const char *rz_utf_block_name(int idx) {
*
* \param buf The buffer to read from.
* \param The buffer length in bytes.
* \param cp The decoded code point. It is only written if a valid
* Unicode code point was decoded.
* \param cp The decoded code point.
* \param check_is_def If true, checks the code point against the defined
* Unicode table. It will not write \p cp and return 0 if the decoded code
* point is undefined.
* If false, it won't perform any checks and just decode.
* Be aware, the check has a runtime of O(log n).
* Where n: number of undefined Unicode ranges.
*
* \return The number of bytes decoded. Is always between 0-4.
*/
RZ_API size_t rz_utf8_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULLABLE RZ_OUT RzCodePoint *cp) {
RZ_API size_t rz_utf8_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULLABLE RZ_OUT RzCodePoint *cp, bool check_is_def) {
rz_return_val_if_fail(buf, 0);
if (buf_len < 1) {
return 0;
@ -403,7 +408,7 @@ RZ_API size_t rz_utf8_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULLA
}
bytes_used = 4;
}
if (!rz_unicode_code_point_is_legal_decode(code_point)) {
if (check_is_def && !rz_unicode_code_point_is_legal_decode(code_point)) {
return 0;
}
if (cp) {
@ -420,7 +425,7 @@ RZ_API int rz_mutf8_decode(const ut8 *ptr, int ptrlen, RzCodePoint *ch) {
}
return 2;
}
return rz_utf8_decode(ptr, ptrlen, ch);
return rz_utf8_decode(ptr, ptrlen, ch, false);
}
/* Convert a unicode RzCodePoint into an UTF-8 buf */
@ -638,7 +643,7 @@ RZ_API int *rz_utf_block_list(const ut8 *str, int len, int **freq_list) {
RzCodePoint ch = 0;
while (str_ptr < str_end) {
int block_idx;
int ch_bytes = rz_utf8_decode(str_ptr, str_end - str_ptr, &ch);
int ch_bytes = rz_utf8_decode(str_ptr, str_end - str_ptr, &ch, true);
if (!ch_bytes) {
block_idx = RZ_UNICODE_BLOCKS_COUNT - 1;
ch_bytes = 1;

20
subprojects/packagefiles/pcre2/meson.build
View file

@ -22,9 +22,7 @@ config_h = configure_file(input : 'src/config.h.generic',
libpcre2_c_args = [
'-DHAVE_CONFIG_H', # Default values from config.h
'-DPCRE2_CODE_UNIT_WIDTH=8',
'-DHAVE_MEMMOVE',
'-DSUPPORT_PCRE2_8',
'-DSUPPORT_UNICODE',
'-fvisibility=default',
]
@ -83,13 +81,25 @@ pcre2_includes = [
include_directories('src/'),
]
libpcre2 = static_library('pcre2', pcre2_files,
c_args: libpcre2_c_args,
libpcre2_8 = static_library('pcre2_8', pcre2_files,
c_args: libpcre2_c_args + ['-DPCRE2_CODE_UNIT_WIDTH=8', '-DSUPPORT_PCRE2_8'],
include_directories: pcre2_includes,
install: false,
)
libpcre2_16 = static_library('pcre2_16', pcre2_files,
c_args: libpcre2_c_args + ['-DPCRE2_CODE_UNIT_WIDTH=16', '-DSUPPORT_PCRE2_16'],
include_directories: pcre2_includes,
install: false,
)
libpcre2_32 = static_library('pcre2_32', pcre2_files,
c_args: libpcre2_c_args + ['-DPCRE2_CODE_UNIT_WIDTH=32', '-DSUPPORT_PCRE2_32'],
include_directories: pcre2_includes,
install: false,
)
pcre2_dep = declare_dependency(
link_with: libpcre2,
link_with: [ libpcre2_8, libpcre2_16, libpcre2_32 ],
include_directories: pcre2_includes
)

24
subprojects/packagefiles/pcre2_cross_native/meson.build
View file

@ -22,9 +22,7 @@ config_h = configure_file(input : 'src/config.h.generic',
libpcre2_c_args = [
'-DHAVE_CONFIG_H', # Default values from config.h
'-DPCRE2_CODE_UNIT_WIDTH=8',
'-DHAVE_MEMMOVE',
'-DSUPPORT_PCRE2_8',
'-DSUPPORT_UNICODE',
'-fvisibility=default',
]
@ -83,14 +81,28 @@ pcre2_includes = [
include_directories('src/'),
]
libpcre2_cross_native = static_library('pcre2_cross_native', pcre2_files,
c_args: libpcre2_c_args,
libpcre2_8_cross_native = static_library('pcre2_8_cross_native', pcre2_files,
c_args: libpcre2_c_args + ['-DPCRE2_CODE_UNIT_WIDTH=8', '-DSUPPORT_PCRE2_8'],
include_directories: pcre2_includes,
install: false,
native: true,
)
libpcre2_1_cross_native6 = static_library('pcre2_16_cross_native', pcre2_files,
c_args: libpcre2_c_args + ['-DPCRE2_CODE_UNIT_WIDTH=16', '-DSUPPORT_PCRE2_16'],
include_directories: pcre2_includes,
install: false,
native: true,
)
libpcre2_3_cross_native2 = static_library('pcre2_32_cross_native', pcre2_files,
c_args: libpcre2_c_args + ['-DPCRE2_CODE_UNIT_WIDTH=32', '-DSUPPORT_PCRE2_32'],
include_directories: pcre2_includes,
install: false,
native: true,
)
pcre2_cross_native_dep = declare_dependency(
link_with: libpcre2_cross_native,
include_directories: pcre2_includes,
link_with: [ libpcre2_8_cross_native, libpcre2_16_cross_native, libpcre2_32_cross_native ],
include_directories: pcre2_includes
)

14
test/db/analysis/golang
View file

@ -18,7 +18,7 @@ EXPECT=<<EOF
0x0063e49b str.version
0x0063e4ac str.HTTP_version
0x0063e566 str.
15389
15399
bufio
bytes
compress/flate
@ -153,7 +153,7 @@ EXPECT=<<EOF
;-- str.sort:
;-- str.sync:
;-- str.AAAA:
14534
14541
bufio
bytes
compress/flate
@ -273,7 +273,7 @@ EXPECT=<<EOF
0x0065de6d str.version
0x0065de8a str.HTTP_version
0x0065dfe6 str.
15100
15086
bufio
bytes
compress/flate
@ -437,7 +437,7 @@ EXPECT=<<EOF
0x0027401c str.btcctl.conf
0x00274090 str.rpc.cert
0x00274104 str.rpc.cert
16168
16188
bufio
bytes
compress/flate
@ -629,7 +629,7 @@ EXPECT=<<EOF
0x0026e4b8 str.btcctl.conf
0x0026e53c str.rpc.cert
0x0026e5c0 str.rpc.cert
16744
16769
EOF
RUN
@ -711,7 +711,7 @@ EXPECT=<<EOF
1788
0x004a699b str.hello__hacktivity
0x004a69d5 str.reflect.__funcTypeFixed64_.AssignableTo
9557
9570
compiler go1.15 Linker 03.00
errors
fmt
@ -963,7 +963,7 @@ EXPECT=<<EOF
0x00490692 str.name:
0x0049071e str.tail:
0x00490780 str.expected__foo__or__bar__subcommands
20682
20681
EOF
RUN

22
test/db/cmd/cmd_i
View file

@ -3778,19 +3778,19 @@ EOF
EXPECT=<<EOF
0x000002f6 0x000002f6 8 9 ibm037 SRSGENER
0x000004dc 0x000004dc 8 9 ibm037 SRSGENER
0x00000668 0x00000668 92 93 ibm037 SRSGENER_03/06/08_14.16 SRS Version 1.3.0_BASE COPYRIGHT 1998-2008 DAVID W DANNER ALL RIG
0x00000668 0x00000668 92 92 ibm037 SRSGENER_03/06/08_14.16 SRS Version 1.3.0_BASE COPYRIGHT 1998-2008 DAVID W DANNER ALL RIG
0x00001682 0x00001682 30 31 ibm037 SRSGENERNO *SRS> NONHELD
0x00002faa 0x00002faa 49 50 ibm037 SHUTDOWNSRSOPTS SRSTBLK SRSGENERALL STARTINGS
0x00002faa 0x00002faa 49 49 ibm037 SHUTDOWNSRSOPTS SRSTBLK SRSGENERALL STARTINGS
0x0000029a 0x0000029a 8 9 ibm037 SRSPRHEX
0x00000362 0x00000362 8 9 ibm037 SRSPRHEX
0x000016d4 0x000016d4 8 9 ibm037 SRSPRHEX
0x00001861 0x00001861 27 28 ibm037 SRSPRHEX_03/06/08_14.17°Ö}\f
0x00001861 0x00001861 27 27 ibm037 SRSPRHEX_03/06/08_14.17°Ö}\f
0x0000191f 0x0000191f 9 10 ibm037 ÏSRSPRHEX
0x0000315b 0x0000315b 27 28 ibm037 SRSPRHEX_03/06/08_14.17°Ö}\f
0x0000315b 0x0000315b 27 27 ibm037 SRSPRHEX_03/06/08_14.17°Ö}\f
0x0000323f 0x0000323f 9 10 ibm037 3SRSPRHEX
0x00004c0f 0x00004c0f 27 28 ibm037 SRSPRHEX_03/06/08_14.17°Ö}\f
0x00004c0f 0x00004c0f 27 27 ibm037 SRSPRHEX_03/06/08_14.17°Ö}\f
0x000003ba 0x000003ba 8 9 ibm037 SRSTASK
0x00002545 0x00002545 12 13 ibm037 «SRSTASK  \-
0x00002545 0x00002545 12 12 ibm037 «SRSTASK  \-
0x00003230 0x00003230 8 9 ibm037 SRSTASK
0x000033e4 0x000033e4 103 104 ibm037 SRSTASK_12/31/08_20.31 SRS Version 1.3.0_A13001 COPYRIGHT 1998-2008 DAVID W DANNER ALL RIGHTS RESERVED
EOF
@ -4651,7 +4651,7 @@ NAME=izz
FILE=bins/mach0/fatmach0-3true
CMDS=izz~http~codesigning
EXPECT=<<EOF
0x00003f3c 0x100002f3c 47 48 ascii ,http://www.apple.com/appleca/codesigning.crl0\r
0x00003f3c 0x100002f3c 47 47 ascii ,http://www.apple.com/appleca/codesigning.crl0\r
EOF
RUN
@ -4660,7 +4660,7 @@ FILE=bins/arm/elf/hello_world
CMDS=aaa;izxq
EXPECT=<<EOF
0x574 13 12 Hello world!
0x494 11 10 \bH\tKxD\tJ{D
0x494 10 10 \bH\tKxD\tJ{D
EOF
RUN
@ -4768,7 +4768,7 @@ EXPECT=<<EOF
0x00489d8a 26 52 .text utf16le http://pomf.cat/upload.php
0x00489edd 19 38 .text utf16le https://a.pomf.cat/
0x0048b9ed 26 52 .text utf16le http://pomf.cat/upload.php
0x0048ba22 19 40 .text utf16le https://a.pomf.cat/
0x0048ba22 19 38 .text utf16le https://a.pomf.cat/
EOF
RUN
@ -5167,7 +5167,7 @@ EXPECT=<<EOF
paddr vaddr flag len size section type xref-from string
------------------------------------------------------------------------------------------
0x00000574 0x00000574 str.Hello_world 12 13 .rodata ascii 0x00000512 Hello world!
0x00000494 0x00000494 register_tm_clones 10 11 .text ascii entry.init0 \bH\tKxD\tJ{D
0x00000494 0x00000494 register_tm_clones 10 10 .text ascii entry.init0 \bH\tKxD\tJ{D
EOF
RUN
@ -5178,6 +5178,6 @@ aaa
izxj
EOF
EXPECT=<<EOF
[{"vaddr":1396,"paddr":1396,"size":13,"length":12,"section":".rodata","type":"ascii","flag":"str.Hello_world","xref-from":"0x00000512","string":"Hello world!"},{"vaddr":1172,"paddr":1172,"size":11,"length":10,"section":".text","type":"ascii","flag":"register_tm_clones","xref-from":"entry.init0","string":"\bH\tKxD\tJ{D"}]
[{"vaddr":1396,"paddr":1396,"size":13,"length":12,"section":".rodata","type":"ascii","flag":"str.Hello_world","xref-from":"0x00000512","string":"Hello world!"},{"vaddr":1172,"paddr":1172,"size":10,"length":10,"section":".text","type":"ascii","flag":"register_tm_clones","xref-from":"entry.init0","string":"\bH\tKxD\tJ{D"}]
EOF
RUN

16
test/db/cmd/cmd_search_x
View file

@ -442,3 +442,19 @@ Alignment=16
0x00000040
EOF
RUN
NAME=/x on invalid UTF-8, check search works without PCRE2_NO_UTF_CHECK
FILE=bins/cmd/search/hex_bytes_invalid_utf
ARGS=
CMDS=<<EOF
# Byte search uses regex and it will fail with
# invalid UTF characters.
# So check here if we find bytes which would be invalid UTF-8.
/x e1e0ff~$[0]
/xr x99.?xff{2}~$[0]
EOF
EXPECT=<<EOF
0x0000001e
0x00000076
EOF
RUN

194
test/db/cmd/cmd_search_z
View file

@ -470,22 +470,6 @@ EOF
EXPECT_ERR=
RUN
NAME=String Search - Encoding: utf16be - Armenian
FILE=bins/cmd/search/string_encodings/Armenian-Lipsum.utf16be
CMDS=<<EOF
/z "դեֆինիթիոնես ին վիս, ծասե պեռթինա" l utf16be
# Requires increasing the block size because -.-
b 0x1000
# Check string is actually at offset of the hit.
ps utf16be unprintable @ hit.string.utf16be.0
EOF
EXPECT=<<EOF
0x0000020c 66 hit.string.utf16be.0
դեֆինիթիոնես ին վիս, ծասե պեռթինածիա նե վիս. ին մելիուս ֆածիլիս եոս, եսթ եռոս պոպուլո թիմեամ թե. իուս նո ծոնսուլաթու.
EOF
EXPECT_ERR=
RUN
NAME=String Search - Encoding: utf16be - Chinese
FILE=bins/cmd/search/string_encodings/Chinese-Lipsum.utf16be
CMDS=<<EOF
@ -629,22 +613,6 @@ EOF
EXPECT_ERR=
RUN
NAME=String Search - Encoding: utf16le - Armenian
FILE=bins/cmd/search/string_encodings/Armenian-Lipsum.utf16le
CMDS=<<EOF
/z "դեֆինիթիոնես ին վիս, ծասե պեռթինա" l utf16le
# Requires increasing the block size because -.-
b 0x1000
# Check string is actually at offset of the hit.
ps utf16le unprintable @ hit.string.utf16le.0
EOF
EXPECT=<<EOF
0x0000020c 66 hit.string.utf16le.0
դեֆինիթիոնես ին վիս, ծասե պեռթինածիա նե վիս. ին մելիուս ֆածիլիս եոս, եսթ եռոս պոպուլո թիմեամ թե. իուս նո ծոնսուլաթու.
EOF
EXPECT_ERR=
RUN
NAME=String Search - Encoding: utf16le - Chinese
FILE=bins/cmd/search/string_encodings/Chinese-Lipsum.utf16le
CMDS=<<EOF
@ -805,22 +773,6 @@ EOF
EXPECT_ERR=
RUN
NAME=String Search - Encoding: utf32be - Chinese
FILE=bins/cmd/search/string_encodings/Chinese-Lipsum.utf32be
CMDS=<<EOF
/z "京確年禁読応米新報書了号活修佐。" l utf32be
# Requires increasing the block size because -.-
b 0x1000
# Check string is actually at offset of the hit.
ps utf32be unprintable @ hit.string.utf32be.0
EOF
EXPECT=<<EOF
0x00000128 64 hit.string.utf32be.0
京確年禁読応米新報書了号活修佐。央良持著宣込警石寂益好独田接読仕。漂戦辺量食害恐人給景族缶掲価済教放。響城引真際通御芭情査男倍淺早住。金握崎利契事用鹿仮交体受寄終山率大。
EOF
EXPECT_ERR=
RUN
NAME=String Search - Encoding: utf32be - Georgian
FILE=bins/cmd/search/string_encodings/Georgian-Lipsum.utf32be
CMDS=<<EOF
@ -933,21 +885,6 @@ EOF
EXPECT_ERR=
RUN
NAME=String Search - Encoding: utf32le - Arabic
FILE=bins/cmd/search/string_encodings/Arabic-Lipsum.utf32le
CMDS=<<EOF
/z "ا الأوضاع, لم بوابة المب" l utf32le
# Requires increasing the block size because -.-
b 0x1000
# Check string is actually at offset of the hit.
# psutf3 unprintable @ hit.string.utf32le.0
EOF
EXPECT=<<EOF
0x00000270 96 hit.string.utf32le.0
EOF
EXPECT_ERR=
RUN
NAME=String Search - Encoding: utf32le - Armenian
FILE=bins/cmd/search/string_encodings/Armenian-Lipsum.utf32le
CMDS=<<EOF
@ -1393,16 +1330,28 @@ echo ----
/z ¢€𐍈 @e:search.str.min_length=3
EOF
EXPECT=<<EOF
WARNING: The string encoding for the search is set to "guess".
The search will consume vastly more resources and the guessing is unreliable.
You can set a specific encoding with 'e str.encoding=<encoding>'.
0x00000030 2 hit.string.ascii.0
WARNING: |ab| < search.str.min_length so some search hits may be hidden. Set search.str.min_length to 2 to see them.
----
WARNING: The string encoding for the search is set to "guess".
The search will consume vastly more resources and the guessing is unreliable.
You can set a specific encoding with 'e str.encoding=<encoding>'.
0x00000010 2 hit.string.ascii.0
0x00000020 2 hit.string.ascii.1
0x00000030 2 hit.string.ascii.2
WARNING: The string encoding for the search is set to "guess".
The search will consume vastly more resources and the guessing is unreliable.
You can set a specific encoding with 'e str.encoding=<encoding>'.
0x00000050 9 hit.string.utf8.0
WARNING: |¢€𐍈| < search.str.min_length so some search hits may be hidden. Set search.str.min_length to 3 to see them.
----
WARNING: The string encoding for the search is set to "guess".
The search will consume vastly more resources and the guessing is unreliable.
You can set a specific encoding with 'e str.encoding=<encoding>'.
0x00000040 9 hit.string.utf8.0
0x00000050 9 hit.string.utf8.1
EOF
@ -1414,6 +1363,9 @@ CMDS=!rizin -e cfg.fortunes=0 -e scr.color=0 -e search.show_progress=false -c "<
EXPECT=<<EOF
 [0x00000000]> [0x00000000]>  [0x00000000]>
 [0x00000000]> [0x00000000]>   [0x00000000]> / [0x00000000]> /  [0x00000000]> /z [0x00000000]> /z  [0x00000000]> /z [0x00000000]> /z   [0x00000000]> /z a [0x00000000]> /z a  [0x00000000]> /z ab [0x00000000]> /z ab [0x00000000]> /z ab
WARNING: The string encoding for the search is set to "guess".
The search will consume vastly more resources and the guessing is unreliable.
You can set a specific encoding with 'e str.encoding=<encoding>'.
WARNING: |ab| < search.str.min_length so some search hits may be hidden. Set search.str.min_length to 2 to see them.
 [0x00000000]> [0x00000000]>   [0x00000000]> q [0x00000000]> q [0x00000000]> q
EOF
@ -1423,6 +1375,122 @@ NAME=String search - log.show.sources
FILE=--
CMDS=!!rizin -1 -qc "e search.show_progress=0 search.str.min_length=4 log.show.sources=1; /z ¢€𐍈" = | tr 0-9 "#" | tr \134 / | sed "s,C:/projects/rizin,..,g"
EXPECT=<<EOF
WARNING: rz_search_collection_strings_check_config_improvements in ../librz/search/string_search.c:###: The string encoding for the search is set to "guess".
The search will consume vastly more resources and the guessing is unreliable.
You can set a specific encoding with 'e str.encoding=<encoding>'.
WARNING: rz_core_print_warnings_after in ../librz/core/core.c:###: cmd_string_search_generic in ../librz/core/cmd/cmd_search.c:####: |¢€𐍈| < search.str.min_length so some search hits may be hidden. Set search.str.min_length to # to see them.
EOF
RUN
#
# Test alignment of string encoding to code point widths
#
NAME=String Search - Encoding: utf32le - Arabic - fast on LE systems
FILE=bins/cmd/search/string_encodings/Arabic-Lipsum.utf32le
CMDS=<<EOF
# Align to UTF32 code point width
e search.align=4
e str.encoding=utf32le
/z "ا الأوضاع, لم بوابة المب" l utf32le
# Requires increasing the block size because -.-
b 0x1000
# Check string is actually at offset of the hit.
# psutf3 unprintable @ hit.string.utf32le.0
EOF
EXPECT=<<EOF
0x00000270 96 hit.string.utf32le.0
EOF
EXPECT_ERR=
RUN
NAME=String Search - Encoding: utf32be - Chinese - Fast on BE systems
FILE=bins/cmd/search/string_encodings/Chinese-Lipsum.utf32be
CMDS=<<EOF
# Align to UTF32 code point width
e search.align=4
e str.encoding=utf32be
/z "京確年禁読応米新報書了号活修佐。" l utf32be
# Requires increasing the block size because -.-
b 0x1000
# Check string is actually at offset of the hit.
ps utf32be unprintable @ hit.string.utf32be.0
EOF
EXPECT=<<EOF
0x00000128 64 hit.string.utf32be.0
京確年禁読応米新報書了号活修佐。央良持著宣込警石寂益好独田接読仕。漂戦辺量食害恐人給景族缶掲価済教放。響城引真際通御芭情査男倍淺早住。金握崎利契事用鹿仮交体受寄終山率大。
EOF
EXPECT_ERR=
RUN
NAME=String Search - Encoding: utf16be - Armenian - Fast on BE systems
FILE=bins/cmd/search/string_encodings/Armenian-Lipsum.utf16be
CMDS=<<EOF
# Align to UTF16 code point width
e search.align=2
e str.encoding=utf16be
/z "դեֆինիթիոնես ին վիս, ծասե պեռթինա" l utf16be
# Requires increasing the block size because -.-
b 0x1000
# Check string is actually at offset of the hit.
ps utf16be unprintable @ hit.string.utf16be.0
EOF
EXPECT=<<EOF
0x0000020c 66 hit.string.utf16be.0
դեֆինիթիոնես ին վիս, ծասե պեռթինածիա նե վիս. ին մելիուս ֆածիլիս եոս, եսթ եռոս պոպուլո թիմեամ թե. իուս նո ծոնսուլաթու.
EOF
EXPECT_ERR=
RUN
NAME=String Search - Encoding: utf16le - Armenian - Fast on LE systems
FILE=bins/cmd/search/string_encodings/Armenian-Lipsum.utf16le
CMDS=<<EOF
# Align to UTF16 code point width
e search.align=2
e str.encoding=utf16le
/z "դեֆինիթիոնես ին վիս, ծասե պեռթինա" l utf16le
# Requires increasing the block size because -.-
b 0x1000
# Check string is actually at offset of the hit.
ps utf16le unprintable @ hit.string.utf16le.0
EOF
EXPECT=<<EOF
0x0000020c 66 hit.string.utf16le.0
դեֆինիթիոնես ին վիս, ծասե պեռթինածիա նե վիս. ին մելիուս ֆածիլիս եոս, եսթ եռոս պոպուլո թիմեամ թե. իուս նո ծոնսուլաթու.
EOF
EXPECT_ERR=
RUN
NAME=String Search - Encoding: utf32le - Arabic - Fast search, aligned to code point width
FILE=bins/cmd/search/string_encodings/Arabic-Lipsum.utf32le
CMDS=<<EOF
e search.align=4
/z "ا الأوضاع, لم بوابة المب" l utf32le
# Requires increasing the block size lecause -.-
b 0x1000
# Check string is actually at offset of the hit.
ps utf32le unprintable @ hit.string.utf32le.0
EOF
EXPECT=<<EOF
0x00000270 96 hit.string.utf32le.0
ا الأوضاع, لم بوابة المبرمة عرض. إبّان اسبوعين البشريةً تعد في. كنقطة إيطاليا قام بل, أضف أن وبغطاء الباهضة.
EOF
EXPECT_ERR=
RUN
NAME=String Search - Encoding: utf16le - Armenian - Fast search, aligned to code point width
FILE=bins/cmd/search/string_encodings/Armenian-Lipsum.utf16le
CMDS=<<EOF
e search.align=2
/z "դեֆինիթիոնես ին վիս, ծասե պեռթինա" l utf16le
# Requires increasing the block size because -.-
b 0x1000
# Check string is actually at offset of the hit.
ps utf16le unprintable @ hit.string.utf16le.0
EOF
EXPECT=<<EOF
0x0000020c 66 hit.string.utf16le.0
դեֆինիթիոնես ին վիս, ծասե պեռթինածիա նե վիս. ին մելիուս ֆածիլիս եոս, եսթ եռոս պոպուլո թիմեամ թե. իուս նո ծոնսուլաթու.
EOF
EXPECT_ERR=
RUN

2
test/db/cmd/cmd_string
View file

@ -4,7 +4,7 @@ CMDS=izz
EXPECT=<<EOF
paddr vaddr len size section type string
----------------------------------------------------------------------------------------
0x00000034 0x00400034 4 10 utf16le @8\b@
0x00000034 0x00400034 4 8 utf16le @8\b@
0x00000200 0x00400200 25 26 .interp ascii /lib/ld-linux-x86-64.so.2
0x000002e1 0x004002e1 14 15 .dynstr ascii __gmon_start__
0x000002f0 0x004002f0 9 10 .dynstr ascii libc.so.6

27
test/db/cmd/regexp
View file

@ -95,7 +95,23 @@ bcd
0x00000010 5 hit.string.ascii.0
bcccde
EOF
EXPECT_ERR=
EXPECT_ERR=<<EOF
WARNING: The string encoding for the search is set to "guess".
The search will consume vastly more resources and the guessing is unreliable.
You can set a specific encoding with 'e str.encoding=<encoding>'.
WARNING: The string encoding for the search is set to "guess".
The search will consume vastly more resources and the guessing is unreliable.
You can set a specific encoding with 'e str.encoding=<encoding>'.
WARNING: The string encoding for the search is set to "guess".
The search will consume vastly more resources and the guessing is unreliable.
You can set a specific encoding with 'e str.encoding=<encoding>'.
WARNING: The string encoding for the search is set to "guess".
The search will consume vastly more resources and the guessing is unreliable.
You can set a specific encoding with 'e str.encoding=<encoding>'.
WARNING: The string encoding for the search is set to "guess".
The search will consume vastly more resources and the guessing is unreliable.
You can set a specific encoding with 'e str.encoding=<encoding>'.
EOF
RUN
NAME=/z at block end
@ -122,7 +138,14 @@ bd
0x000001fe 2 hit.string.ascii.0
bd
EOF
EXPECT_ERR=
EXPECT_ERR=<<EOF
WARNING: The string encoding for the search is set to "guess".
The search will consume vastly more resources and the guessing is unreliable.
You can set a specific encoding with 'e str.encoding=<encoding>'.
WARNING: The string encoding for the search is set to "guess".
The search will consume vastly more resources and the guessing is unreliable.
You can set a specific encoding with 'e str.encoding=<encoding>'.
EOF
RUN
NAME=consistency btw /z and /x

20
test/db/formats/elf/strings
View file

@ -21,12 +21,12 @@ aar
fl@F:strings
EOF
EXPECT=<<EOF
0x000005e8 0x004005e8 11 26 .rodata utf16le Hello World
0x004005e8 26 str.Hello_World
0x00400608 16 str.S
0x000005e8 0x004005e8 11 24 .rodata utf16le Hello World
0x004005e8 24 str.Hello_World
0x00400608 14 str.S
| 0x0040052e mov qword [var_10h], str.Hello_World ; 0x4005e8 ; u"\U0000feffHello World\U0000feff\n"
0x004005e8 26 str.Hello_World
0x00400608 16 str.S
0x004005e8 24 str.Hello_World
0x00400608 14 str.S
EOF
RUN
@ -42,12 +42,12 @@ aar
fl@F:strings
EOF
EXPECT=<<EOF
0x000005e8 0x004005e8 11 52 .rodata utf32le Hello World
0x004005e8 52 str.Hello_World
0x00400628 32 str.S
0x000005e8 0x004005e8 11 48 .rodata utf32le Hello World
0x004005e8 48 str.Hello_World
0x00400628 28 str.S
| 0x0040052e mov qword [var_10h], str.Hello_World ; 0x4005e8 ; U"\U0000feffHello World\U0000feff\n"
0x004005e8 52 str.Hello_World
0x00400628 32 str.S
0x004005e8 48 str.Hello_World
0x00400628 28 str.S
EOF
RUN

2
test/db/tools/rz_bin
View file

@ -1243,7 +1243,7 @@ FILE=bins/mz/broken.exe
CMDS=!rz-bin -zzj ${RZ_FILE} ; echo ""
EXPECT=<<EOF
{}
[{"vaddr":77,"paddr":77,"size":45,"length":44,"section":"","type":"ascii","string":"!This program cannot be run in DOS mode.\r\r\n$"},{"vaddr":155,"paddr":155,"size":6,"length":5,"section":"","type":"ascii","string":"Q\fb.Q"},{"vaddr":189,"paddr":189,"size":4,"length":4,"section":"","type":"ascii","string":"b.Q{"},{"vaddr":205,"paddr":205,"size":7,"length":7,"section":"","type":"ascii","string":"b.Q4<*P"},{"vaddr":213,"paddr":213,"size":5,"length":5,"section":"","type":"ascii","string":"b.Q4<"},{"vaddr":221,"paddr":221,"size":7,"length":7,"section":"","type":"ascii","string":"b.Q4<,P"},{"vaddr":229,"paddr":229,"size":7,"length":7,"section":"","type":"ascii","string":"b.QRich"},{"vaddr":241,"paddr":241,"size":30,"length":29,"section":"","type":"ascii","string":"https://malwarec2.com/drd.exe"}]
[{"vaddr":77,"paddr":77,"size":45,"length":44,"section":"","type":"ascii","string":"!This program cannot be run in DOS mode.\r\r\n$"},{"vaddr":155,"paddr":155,"size":5,"length":5,"section":"","type":"ascii","string":"Q\fb.Q"},{"vaddr":189,"paddr":189,"size":4,"length":4,"section":"","type":"ascii","string":"b.Q{"},{"vaddr":205,"paddr":205,"size":7,"length":7,"section":"","type":"ascii","string":"b.Q4<*P"},{"vaddr":213,"paddr":213,"size":5,"length":5,"section":"","type":"ascii","string":"b.Q4<"},{"vaddr":221,"paddr":221,"size":7,"length":7,"section":"","type":"ascii","string":"b.Q4<,P"},{"vaddr":229,"paddr":229,"size":7,"length":7,"section":"","type":"ascii","string":"b.QRich"},{"vaddr":241,"paddr":241,"size":30,"length":29,"section":"","type":"ascii","string":"https://malwarec2.com/drd.exe"}]
EOF
RUN

136
test/unit/test_encodings.c
View file

@ -19,40 +19,40 @@ bool test_rz_utf8_decode(void) {
const ut8 utf8_4b_valid_first[] = { 0xF0, 0x90, 0x80, 0x80 };
const ut8 utf8_4b_valid_last[] = { 0xF4, 0x8F, 0xBF, 0xBD };
mu_assert_eq(rz_utf8_decode(utf8_1b_valid_first, sizeof(utf8_1b_valid_first), &codepoint), 1, "Decode failed");
mu_assert_eq(rz_utf8_decode(utf8_1b_valid_first, sizeof(utf8_1b_valid_first), &codepoint, true), 1, "Decode failed");
mu_assert_eq(codepoint, 0, "Code point incorrect");
mu_assert_eq(rz_utf8_decode(utf8_1b_valid_last, sizeof(utf8_1b_valid_last), &codepoint), 1, "Decode failed");
mu_assert_eq(rz_utf8_decode(utf8_1b_valid_last, sizeof(utf8_1b_valid_last), &codepoint, true), 1, "Decode failed");
mu_assert_eq(codepoint, 0x7f, "Code point incorrect");
mu_assert_eq(rz_utf8_decode(utf8_2b_valid_first, sizeof(utf8_2b_valid_first), &codepoint), 2, "Decode failed");
mu_assert_eq(rz_utf8_decode(utf8_2b_valid_first, sizeof(utf8_2b_valid_first), &codepoint, true), 2, "Decode failed");
mu_assert_eq(codepoint, 0x80, "Code point incorrect");
mu_assert_eq(rz_utf8_decode(utf8_2b_valid_last, sizeof(utf8_2b_valid_last), &codepoint), 2, "Decode failed");
mu_assert_eq(rz_utf8_decode(utf8_2b_valid_last, sizeof(utf8_2b_valid_last), &codepoint, true), 2, "Decode failed");
mu_assert_eq(codepoint, 0x07FF, "Code point incorrect");
mu_assert_eq(rz_utf8_decode(utf8_3b_valid_first, sizeof(utf8_3b_valid_first), &codepoint), 3, "Decode failed");
mu_assert_eq(rz_utf8_decode(utf8_3b_valid_first, sizeof(utf8_3b_valid_first), &codepoint, true), 3, "Decode failed");
mu_assert_eq(codepoint, 0x800, "Code point incorrect");
mu_assert_eq(rz_utf8_decode(utf8_3b_valid_last, sizeof(utf8_3b_valid_last), &codepoint), 3, "Decode failed");
mu_assert_eq(rz_utf8_decode(utf8_3b_valid_last, sizeof(utf8_3b_valid_last), &codepoint, true), 3, "Decode failed");
mu_assert_eq(codepoint, 0xFFFD, "Code point incorrect");
mu_assert_eq(rz_utf8_decode(utf8_4b_valid_first, sizeof(utf8_4b_valid_first), &codepoint), 4, "Decode failed");
mu_assert_eq(rz_utf8_decode(utf8_4b_valid_first, sizeof(utf8_4b_valid_first), &codepoint, true), 4, "Decode failed");
mu_assert_eq(codepoint, 0x10000, "Code point incorrect");
mu_assert_eq(rz_utf8_decode(utf8_4b_valid_last, sizeof(utf8_4b_valid_last), &codepoint), 4, "Decode failed");
mu_assert_eq(rz_utf8_decode(utf8_4b_valid_last, sizeof(utf8_4b_valid_last), &codepoint, true), 4, "Decode failed");
mu_assert_eq(codepoint, 0x10FFFD, "Code point incorrect");
const ut8 utf8_1b_invalid_F[] = { 0xFF };
const ut8 utf8_2b_invalid_F[] = { 0xFF, 0x00 };
const ut8 utf8_3b_invalid_F[] = { 0xFF, 0x00, 0x00 };
mu_assert_eq(rz_utf8_decode(utf8_1b_invalid_F, sizeof(utf8_1b_invalid_F), &codepoint), 0, "Invalid decode, prefix bit false.");
mu_assert_eq(rz_utf8_decode(utf8_2b_invalid_F, sizeof(utf8_2b_invalid_F), &codepoint), 0, "Invalid decode, prefix bit false.");
mu_assert_eq(rz_utf8_decode(utf8_3b_invalid_F, sizeof(utf8_3b_invalid_F), &codepoint), 0, "Invalid decode, prefix bit false.");
mu_assert_eq(rz_utf8_decode(utf8_1b_invalid_F, sizeof(utf8_1b_invalid_F), &codepoint, true), 0, "Invalid decode, prefix bit false.");
mu_assert_eq(rz_utf8_decode(utf8_2b_invalid_F, sizeof(utf8_2b_invalid_F), &codepoint, true), 0, "Invalid decode, prefix bit false.");
mu_assert_eq(rz_utf8_decode(utf8_3b_invalid_F, sizeof(utf8_3b_invalid_F), &codepoint, true), 0, "Invalid decode, prefix bit false.");
const ut8 utf8_2b_invalid_small_code_point[] = { 0xC0, 0x80 };
const ut8 utf8_3b_invalid_small_code_point[] = { 0xE0, 0x80, 0x80 };
const ut8 utf8_4b_invalid_small_code_point[] = { 0xF0, 0x80, 0x80, 0x80 };
mu_assert_eq(rz_utf8_decode(utf8_2b_invalid_small_code_point, sizeof(utf8_2b_invalid_small_code_point), &codepoint), 0, "Invalid decode, code point is too small for encoding.");
mu_assert_eq(rz_utf8_decode(utf8_3b_invalid_small_code_point, sizeof(utf8_3b_invalid_small_code_point), &codepoint), 0, "Invalid decode, code point is too small for encoding.");
mu_assert_eq(rz_utf8_decode(utf8_4b_invalid_small_code_point, sizeof(utf8_4b_invalid_small_code_point), &codepoint), 0, "Invalid decode, code point is too small for encoding.");
mu_assert_eq(rz_utf8_decode(utf8_2b_invalid_small_code_point, sizeof(utf8_2b_invalid_small_code_point), &codepoint, true), 0, "Invalid decode, code point is too small for encoding.");
mu_assert_eq(rz_utf8_decode(utf8_3b_invalid_small_code_point, sizeof(utf8_3b_invalid_small_code_point), &codepoint, true), 0, "Invalid decode, code point is too small for encoding.");
mu_assert_eq(rz_utf8_decode(utf8_4b_invalid_small_code_point, sizeof(utf8_4b_invalid_small_code_point), &codepoint, true), 0, "Invalid decode, code point is too small for encoding.");
mu_end;
}
@ -63,20 +63,20 @@ bool test_rz_utf8_decode(void) {
bool test_rz_utf16_decode(void) {
RzCodePoint codepoint = 0;
const ut8 utf16le_surrogate[] = { 0xd8, 0x00 };
mu_assert_eq(rz_utf16_decode(utf16le_surrogate, 2, &codepoint, true), 0, "Invalid decode");
mu_assert_eq(rz_utf16_decode(utf16le_surrogate, 2, &codepoint, true, true), 0, "Invalid decode");
char utf8_out[5] = { 0 };
const ut8 utf16le_A[] = { 0x41, 0x00 };
const ut8 utf16be_A[] = { 0x00, 0x41 };
int nbytes = rz_utf16_decode(utf16le_A, 2, &codepoint, false);
int nbytes = rz_utf16_decode(utf16le_A, 2, &codepoint, true, false);
mu_assert_eq(nbytes, 2, "Decoded number of bytes mismatch.");
mu_assert_eq_fmt(codepoint, 0x0041, "Character decode failed.", "0x%" PFMT64x);
rz_utf8_encode((ut8 *)utf8_out, codepoint);
mu_assert_streq(utf8_out, "A", "Encode failed.");
memset(utf8_out, 0, sizeof(utf8_out));
nbytes = rz_utf16_decode(utf16be_A, 2, &codepoint, true);
nbytes = rz_utf16_decode(utf16be_A, 2, &codepoint, true, true);
mu_assert_eq(nbytes, 2, "Decoded number of bytes mismatch.");
mu_assert_eq_fmt(codepoint, 0x0041, "Character decode failed.", "0x%" PFMT64x);
rz_utf8_encode((ut8 *)utf8_out, codepoint);
@ -86,14 +86,14 @@ bool test_rz_utf16_decode(void) {
const ut8 utf16le[] = { 0xAC, 0x20 };
const ut8 utf16be[] = { 0x20, 0xAC };
nbytes = rz_utf16_decode(utf16le, 2, &codepoint, false);
nbytes = rz_utf16_decode(utf16le, 2, &codepoint, true, false);
mu_assert_eq(nbytes, 2, "Decoded number of bytes mismatch.");
mu_assert_eq_fmt(codepoint, 0x20AC, "Character decode failed.", "0x%" PFMT64x);
rz_utf8_encode((ut8 *)utf8_out, codepoint);
mu_assert_streq(utf8_out, "€", "Encode failed.");
memset(utf8_out, 0, sizeof(utf8_out));
nbytes = rz_utf16_decode(utf16be, 2, &codepoint, true);
nbytes = rz_utf16_decode(utf16be, 2, &codepoint, true, true);
mu_assert_eq(nbytes, 2, "Decoded number of bytes mismatch.");
mu_assert_eq_fmt(codepoint, 0x20AC, "Character decode failed.", "0x%" PFMT64x);
rz_utf8_encode((ut8 *)utf8_out, codepoint);
@ -104,14 +104,14 @@ bool test_rz_utf16_decode(void) {
const ut8 utf16le_surr[] = { 0x01, 0xD8, 0x37, 0xDC };
const ut8 utf16be_surr[] = { 0xD8, 0x01, 0xDC, 0x37 };
nbytes = rz_utf16_decode(utf16le_surr, 4, &codepoint, false);
nbytes = rz_utf16_decode(utf16le_surr, 4, &codepoint, true, false);
mu_assert_eq(nbytes, 4, "Decoded number of bytes mismatch.");
mu_assert_eq_fmt(codepoint, 0x10437, "Character decode failed.", "0x%" PFMT64x);
rz_utf8_encode((ut8 *)utf8_out, codepoint);
mu_assert_streq(utf8_out, "𐐷", "Encode failed.");
memset(utf8_out, 0, sizeof(utf8_out));
nbytes = rz_utf16_decode(utf16be_surr, 4, &codepoint, true);
nbytes = rz_utf16_decode(utf16be_surr, 4, &codepoint, true, true);
mu_assert_eq(nbytes, 4, "Decoded number of bytes mismatch.");
mu_assert_eq_fmt(codepoint, 0x10437, "Character decode failed.", "0x%" PFMT64x);
rz_utf8_encode((ut8 *)utf8_out, codepoint);
@ -121,14 +121,14 @@ bool test_rz_utf16_decode(void) {
const ut8 utf16le_first_surr[] = { 0x00, 0xD8, 0x00, 0xDC };
const ut8 utf16be_first_surr[] = { 0xD8, 0x00, 0xDC, 0x00 };
nbytes = rz_utf16_decode(utf16le_first_surr, 4, &codepoint, false);
nbytes = rz_utf16_decode(utf16le_first_surr, 4, &codepoint, true, false);
mu_assert_eq(nbytes, 4, "Decoded number of bytes mismatch.");
mu_assert_eq_fmt(codepoint, 0x10000, "Character decode failed.", "0x%" PFMT64x);
rz_utf8_encode((ut8 *)utf8_out, codepoint);
mu_assert_streq(utf8_out, "𐀀", "Encode failed.");
memset(utf8_out, 0, sizeof(utf8_out));
nbytes = rz_utf16_decode(utf16be_first_surr, 4, &codepoint, true);
nbytes = rz_utf16_decode(utf16be_first_surr, 4, &codepoint, true, true);
mu_assert_eq(nbytes, 4, "Decoded number of bytes mismatch.");
mu_assert_eq_fmt(codepoint, 0x10000, "Character decode failed.", "0x%" PFMT64x);
rz_utf8_encode((ut8 *)utf8_out, codepoint);
@ -138,38 +138,44 @@ bool test_rz_utf16_decode(void) {
const ut8 utf16le_last_surr[] = { 0xFF, 0xDB, 0xFF, 0xDF };
const ut8 utf16be_last_surr[] = { 0xDB, 0xFF, 0xDF, 0xFF };
nbytes = rz_utf16_decode(utf16le_last_surr, 4, &codepoint, false);
nbytes = rz_utf16_decode(utf16le_last_surr, 4, &codepoint, true, false);
mu_assert_eq(nbytes, 0, "Undefined code point.");
nbytes = rz_utf16_decode(utf16be_last_surr, 4, &codepoint, true);
nbytes = rz_utf16_decode(utf16be_last_surr, 4, &codepoint, true, true);
mu_assert_eq(nbytes, 0, "Undefined code point.");
nbytes = rz_utf16_decode(utf16be_last_surr, 2, &codepoint, false, true);
mu_assert_eq(nbytes, 0, "Surrogate should never be allowed.");
const ut8 utf16_undef[] = { 0xFF, 0xFF };
nbytes = rz_utf16_decode(utf16_undef, 2, &codepoint, false, true);
mu_assert_eq(nbytes, 2, "Undefined was allowed.");
const ut8 utf16le_invalid_small_surr[] = { 0x00, 0xD7, 0x00, 0xDB };
const ut8 utf16be_invalid_small_surr[] = { 0xD7, 0x00, 0xDB, 0x00 };
// Fails to decode 4, should decode 2 bytes.
nbytes = rz_utf16_decode(utf16le_invalid_small_surr, 4, &codepoint, false);
nbytes = rz_utf16_decode(utf16le_invalid_small_surr, 4, &codepoint, true, false);
mu_assert_eq(nbytes, 2, "Decoded number of bytes mismatch.");
mu_assert_eq_fmt(codepoint, 0xD700, "Character decode failed.", "0x%" PFMT64x);
nbytes = rz_utf16_decode(utf16be_invalid_small_surr, 4, &codepoint, true);
nbytes = rz_utf16_decode(utf16be_invalid_small_surr, 4, &codepoint, true, true);
mu_assert_eq(nbytes, 2, "Decoded number of bytes mismatch.");
mu_assert_eq_fmt(codepoint, 0xD700, "Character decode failed.", "0x%" PFMT64x);
const ut8 utf16le_invalid_big_surr[] = { 0x01, 0xDC, 0x37, 0xE0 };
const ut8 utf16be_invalid_big_surr[] = { 0xDC, 0x01, 0xE0, 0x37 };
nbytes = rz_utf16_decode(utf16le_invalid_big_surr, 4, &codepoint, false);
nbytes = rz_utf16_decode(utf16le_invalid_big_surr, 4, &codepoint, true, false);
mu_assert_eq(nbytes, 0, "Undefined code point.");
nbytes = rz_utf16_decode(utf16be_invalid_big_surr, 4, &codepoint, true);
nbytes = rz_utf16_decode(utf16be_invalid_big_surr, 4, &codepoint, true, true);
mu_assert_eq(nbytes, 0, "Undefined code point.");
const ut8 utf16le_last_non_surr[] = { 0xff, 0xff, 0xff, 0xff };
const ut8 utf16be_last_non_surr[] = { 0xff, 0xff, 0xff, 0xff };
nbytes = rz_utf16_decode(utf16le_last_non_surr, 4, &codepoint, false);
nbytes = rz_utf16_decode(utf16le_last_non_surr, 4, &codepoint, true, false);
mu_assert_eq(nbytes, 0, "Undefined code point.");
nbytes = rz_utf16_decode(utf16be_last_non_surr, 4, &codepoint, true);
nbytes = rz_utf16_decode(utf16be_last_non_surr, 4, &codepoint, true, true);
mu_assert_eq(nbytes, 0, "Undefined code point.");
mu_end;
@ -182,39 +188,58 @@ bool test_rz_utf16_encode(void) {
ut8 utf16_out[5] = { 0 };
const ut8 utf16le[] = { 0xAC, 0x20 };
const ut8 utf16be[] = { 0x20, 0xAC };
RzCodePoint codepoint = 0x20AC;
int nbytes = rz_utf16le_encode(utf16_out, codepoint);
int nbytes = rz_utf16_encode(utf16_out, codepoint, false);
mu_assert_eq(nbytes, 2, "Decoded number of bytes mismatch.");
mu_assert_memeq(utf16_out, utf16le, sizeof(utf16le), "Encode failed.");
nbytes = rz_utf16_encode(utf16_out, codepoint, true);
mu_assert_eq(nbytes, 2, "Decoded number of bytes mismatch.");
mu_assert_memeq(utf16_out, utf16be, sizeof(utf16be), "Encode failed.");
memset(utf16_out, 0, sizeof(utf16_out));
// With surrogate
const ut8 utf16le_surr[] = { 0x01, 0xD8, 0x37, 0xDC };
const ut8 utf16be_surr[] = { 0xD8, 0x01, 0xDC, 0x37 };
codepoint = 0x10437;
nbytes = rz_utf16le_encode(utf16_out, codepoint);
nbytes = rz_utf16_encode(utf16_out, codepoint, false);
mu_assert_eq(nbytes, 4, "Decoded number of bytes mismatch.");
mu_assert_memeq(utf16_out, utf16le_surr, sizeof(utf16le), "Encode failed.");
nbytes = rz_utf16_encode(utf16_out, codepoint, true);
mu_assert_eq(nbytes, 4, "Decoded number of bytes mismatch.");
mu_assert_memeq(utf16_out, utf16be_surr, sizeof(utf16be_surr), "Encode failed.");
memset(utf16_out, 0, sizeof(utf16_out));
const ut8 utf16le_first_surr[] = { 0x00, 0xD8, 0x00, 0xDC };
const ut8 utf16be_first_surr[] = { 0xD8, 0x00, 0xDC, 0x00 };
codepoint = 0x10000;
nbytes = rz_utf16le_encode(utf16_out, codepoint);
nbytes = rz_utf16_encode(utf16_out, codepoint, false);
mu_assert_eq(nbytes, 4, "Decoded number of bytes mismatch.");
mu_assert_memeq(utf16_out, utf16le_first_surr, sizeof(utf16le), "Encode failed.");
mu_assert_memeq(utf16_out, utf16le_first_surr, sizeof(utf16le_first_surr), "Encode failed.");
nbytes = rz_utf16_encode(utf16_out, codepoint, true);
mu_assert_eq(nbytes, 4, "Decoded number of bytes mismatch.");
mu_assert_memeq(utf16_out, utf16be_first_surr, sizeof(utf16be_first_surr), "Encode failed.");
memset(utf16_out, 0, sizeof(utf16_out));
const ut8 utf16le_last_surr[] = { 0xFF, 0xDB, 0xFF, 0xDF };
const ut8 utf16be_last_surr[] = { 0xDB, 0xFF, 0xDF, 0xFF };
codepoint = RZ_UNICODE_LAST_CODE_POINT;
nbytes = rz_utf16le_encode(utf16_out, codepoint);
nbytes = rz_utf16_encode(utf16_out, codepoint, false);
mu_assert_eq(nbytes, 4, "Decoded number of bytes mismatch.");
mu_assert_memeq(utf16_out, utf16le_last_surr, sizeof(utf16le), "Encode failed.");
mu_assert_memeq(utf16_out, utf16le_last_surr, sizeof(utf16le_last_surr), "Encode failed.");
nbytes = rz_utf16_encode(utf16_out, codepoint, true);
mu_assert_eq(nbytes, 4, "Decoded number of bytes mismatch.");
mu_assert_memeq(utf16_out, utf16be_last_surr, sizeof(utf16be_last_surr), "Encode failed.");
memset(utf16_out, 0, sizeof(utf16_out));
ut8 zero[5] = { 0 };
codepoint = 0x110000;
nbytes = rz_utf16le_encode(utf16_out, codepoint);
nbytes = rz_utf16_encode(utf16_out, codepoint, false);
mu_assert_eq(nbytes, 0, "Decoded number of bytes mismatch.");
mu_assert_memeq(utf16_out, zero, sizeof(utf16le), "Encode failed.");
nbytes = rz_utf16_encode(utf16_out, codepoint, true);
mu_assert_eq(nbytes, 0, "Decoded number of bytes mismatch.");
mu_assert_memeq(utf16_out, zero, sizeof(zero), "Encode failed.");
mu_end;
}
@ -242,36 +267,37 @@ bool test_rz_utf32_decode(void) {
const ut8 utf32le_red_general_black_tower[] = { 0x60, 0xFA, 0x01, 0x00, 0x41, 0xFA, 0x01, 0x00 };
RzCodePoint cp;
mu_assert_eq(rz_utf32_decode((ut8 *)INT_MIN, 0, &cp, false), 0, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32_size_1, sizeof(utf32_size_1), &cp, false), 0, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32_size_2, sizeof(utf32_size_2), &cp, false), 0, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32_size_3, sizeof(utf32_size_3), &cp, false), 0, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32_undefined_I, sizeof(utf32_undefined_I), &cp, false), 0, "Undefined");
mu_assert_eq(rz_utf32_decode(utf32_invalid_surrogate, sizeof(utf32_invalid_surrogate), &cp, false), 0, "Undefined");
mu_assert_eq(rz_utf32_decode((ut8 *)INT_MIN, 0, &cp, true, false), 0, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32_size_1, sizeof(utf32_size_1), &cp, true, false), 0, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32_size_2, sizeof(utf32_size_2), &cp, true, false), 0, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32_size_3, sizeof(utf32_size_3), &cp, true, false), 0, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32_undefined_I, sizeof(utf32_undefined_I), &cp, true, false), 0, "Undefined");
mu_assert_eq(rz_utf32_decode(utf32_undefined_I, sizeof(utf32_undefined_I), &cp, false, false), 4, "Undefined was allowed");
mu_assert_eq(rz_utf32_decode(utf32_invalid_surrogate, sizeof(utf32_invalid_surrogate), &cp, true, false), 0, "Undefined");
mu_assert_eq(rz_utf32_decode(utf32be_A, sizeof(utf32be_A), &cp, true), 4, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32be_A, sizeof(utf32be_A), &cp, true, true), 4, "Length check failed");
mu_assert_eq(cp, 0x41, "Incorrect decoding.");
mu_assert_eq(rz_utf32_decode(utf32le_A, sizeof(utf32le_A), &cp, false), 4, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32le_A, sizeof(utf32le_A), &cp, true, false), 4, "Length check failed");
mu_assert_eq(cp, 0x41, "Incorrect decoding.");
mu_assert_eq(rz_utf32_decode(utf32be_a, sizeof(utf32be_a), &cp, true), 4, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32be_a, sizeof(utf32be_a), &cp, true, true), 4, "Length check failed");
mu_assert_eq(cp, 0xff41, "Incorrect decoding.");
mu_assert_eq(rz_utf32_decode(utf32le_a, sizeof(utf32le_a), &cp, false), 4, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32le_a, sizeof(utf32le_a), &cp, true, false), 4, "Length check failed");
mu_assert_eq(cp, 0xff41, "Incorrect decoding.");
mu_assert_eq(rz_utf32_decode(utf32be_red_general, sizeof(utf32be_red_general), &cp, true), 4, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32be_red_general, sizeof(utf32be_red_general), &cp, true, true), 4, "Length check failed");
mu_assert_eq(cp, 0x01fa60, "Incorrect decoding.");
mu_assert_eq(rz_utf32_decode(utf32le_red_general, sizeof(utf32le_red_general), &cp, false), 4, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32le_red_general, sizeof(utf32le_red_general), &cp, true, false), 4, "Length check failed");
mu_assert_eq(cp, 0x01fa60, "Incorrect decoding.");
mu_assert_eq(rz_utf32_decode(utf32be_red_general_black_tower, sizeof(utf32be_red_general_black_tower), &cp, true), 4, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32be_red_general_black_tower, sizeof(utf32be_red_general_black_tower), &cp, true, true), 4, "Length check failed");
mu_assert_eq(cp, 0x01fa60, "Incorrect decoding.");
mu_assert_eq(rz_utf32_decode(utf32le_red_general_black_tower, sizeof(utf32le_red_general_black_tower), &cp, false), 4, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32le_red_general_black_tower, sizeof(utf32le_red_general_black_tower), &cp, true, false), 4, "Length check failed");
mu_assert_eq(cp, 0x01fa60, "Incorrect decoding.");
mu_assert_eq(rz_utf32_decode(utf32be_red_general_black_tower + 4, sizeof(utf32be_red_general_black_tower) - 4, &cp, true), 4, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32be_red_general_black_tower + 4, sizeof(utf32be_red_general_black_tower) - 4, &cp, true, true), 4, "Length check failed");
mu_assert_eq(cp, 0x01fa41, "Incorrect decoding.");
mu_assert_eq(rz_utf32_decode(utf32le_red_general_black_tower + 4, sizeof(utf32le_red_general_black_tower) - 4, &cp, false), 4, "Length check failed");
mu_assert_eq(rz_utf32_decode(utf32le_red_general_black_tower + 4, sizeof(utf32le_red_general_black_tower) - 4, &cp, true, false), 4, "Length check failed");
mu_assert_eq(cp, 0x01fa41, "Incorrect decoding.");
mu_end;

191
test/unit/test_regex.c
View file

@ -7,6 +7,7 @@
#include <rz_util/rz_strbuf.h>
#include <rz_util/rz_str.h>
#include <rz_vector.h>
#include <rz_platform.h>
bool exec_regex(RzRegex *regex, const char *str, RzRegexMatch **out) {
RzPVector *matches = rz_regex_match_all_not_grouped(regex, str, RZ_REGEX_ZERO_TERMINATED, 0, RZ_REGEX_DEFAULT);
@ -211,6 +212,193 @@ bool test_rz_regex_named_matches(void) {
mu_end;
}
bool test_rz_regex_match_all_native_utf8(void) {
RzPVector *match_groups = NULL;
RzRegexMatch *match = NULL;
const char *utf8 = "A salat with 🍇🍉🍍 Extra 🍍🍍🍍现代汉语常用字表 please.";
RzRegex *re = rz_regex_new("🍍..", RZ_REGEX_EXTENDED, 0, NULL);
mu_assert_notnull(re, "Regex was NULL");
RzPVector *matches = rz_regex_match_all(re, utf8, RZ_REGEX_ZERO_TERMINATED, 0, RZ_REGEX_DEFAULT);
mu_assert_notnull(matches, "matches was not set");
mu_assert_eq(rz_pvector_len(matches), 2, "matches len was wrong");
match_groups = rz_pvector_at(matches, 0);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 21, "match.start wrong");
mu_assert_eq(match->len, 6, "match.len wrong");
match_groups = rz_pvector_at(matches, 1);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 32, "match.start wrong");
mu_assert_eq(match->len, 12, "match.len wrong");
rz_pvector_free(matches);
// Overlap
matches = rz_regex_match_all_overlap(re, utf8, RZ_REGEX_ZERO_TERMINATED, 0, RZ_REGEX_DEFAULT);
mu_assert_notnull(matches, "matches was not set");
mu_assert_eq(rz_pvector_len(matches), 4, "matches len was wrong");
match_groups = rz_pvector_at(matches, 0);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 21, "match.start wrong");
mu_assert_eq(match->len, 6, "match.len wrong");
match_groups = rz_pvector_at(matches, 1);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 32, "match.start wrong");
mu_assert_eq(match->len, 12, "match.len wrong");
match_groups = rz_pvector_at(matches, 2);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 36, "match.start wrong");
mu_assert_eq(match->len, 11, "match.len wrong");
match_groups = rz_pvector_at(matches, 3);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 40, "match.start wrong");
mu_assert_eq(match->len, 10, "match.len wrong");
rz_pvector_free(matches);
rz_regex_free(re);
mu_end;
}
bool test_rz_regex_match_all_native_utf16(void) {
RzPVector *match_groups = NULL;
RzRegexMatch *match = NULL;
const char *utf8 = "A salat with 🍇🍉🍍 Extra 🍍🍍🍍现代汉语常用字表 please.";
// Encode to host endianess UTF-16/32
ut16 *utf16_he = rz_str_utf8_to_utf16(utf8, RZ_HOST_IS_BIG_ENDIAN);
RzRegex16 *re = rz_regex_new_16("🍍..", RZ_REGEX_EXTENDED, 0, NULL);
mu_assert_notnull(re, "Regex was NULL");
RzPVector *matches = rz_regex_match_all_16(re, utf16_he, RZ_REGEX_ZERO_TERMINATED, 0, RZ_REGEX_DEFAULT);
mu_assert_notnull(matches, "matches was not set");
mu_assert_eq(rz_pvector_len(matches), 2, "matches len was wrong");
match_groups = rz_pvector_at(matches, 0);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 17, "match.start wrong");
mu_assert_eq(match->len, 4, "match.len wrong");
match_groups = rz_pvector_at(matches, 1);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 26, "match.start wrong");
mu_assert_eq(match->len, 6, "match.len wrong");
rz_pvector_free(matches);
// Overlap
matches = rz_regex_match_all_overlap_16(re, utf16_he, RZ_REGEX_ZERO_TERMINATED, 0, RZ_REGEX_DEFAULT);
mu_assert_notnull(matches, "matches was not set");
mu_assert_eq(rz_pvector_len(matches), 4, "matches len was wrong");
match_groups = rz_pvector_at(matches, 0);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 17, "match.start wrong");
mu_assert_eq(match->len, 4, "match.len wrong");
match_groups = rz_pvector_at(matches, 1);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 26, "match.start wrong");
mu_assert_eq(match->len, 6, "match.len wrong");
match_groups = rz_pvector_at(matches, 2);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 28, "match.start wrong");
mu_assert_eq(match->len, 5, "match.len wrong");
match_groups = rz_pvector_at(matches, 3);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 30, "match.start wrong");
mu_assert_eq(match->len, 4, "match.len wrong");
rz_pvector_free(matches);
rz_regex_free(re);
mu_end;
}
bool test_rz_regex_match_all_native_utf32(void) {
RzPVector *match_groups = NULL;
RzRegexMatch *match = NULL;
const char *utf8 = "A salat with 🍇🍉🍍 Extra 🍍🍍🍍现代汉语常用字表 please.";
// Encode to host endianess UTF-32/32
ut32 *utf32_he = rz_str_utf8_to_utf32(utf8, RZ_HOST_IS_BIG_ENDIAN);
RzRegex32 *re = rz_regex_new_32("🍍..", RZ_REGEX_EXTENDED, 0, NULL);
mu_assert_notnull(re, "Regex was NULL");
RzPVector *matches = rz_regex_match_all_32(re, (ut32 *)utf32_he, RZ_REGEX_ZERO_TERMINATED, 0, RZ_REGEX_DEFAULT);
mu_assert_notnull(matches, "matches was not set");
mu_assert_eq(rz_pvector_len(matches), 2, "matches len was wrong");
match_groups = rz_pvector_at(matches, 0);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 15, "match.start wrong");
mu_assert_eq(match->len, 3, "match.len wrong");
match_groups = rz_pvector_at(matches, 1);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 23, "match.start wrong");
mu_assert_eq(match->len, 3, "match.len wrong");
rz_pvector_free(matches);
// Overlap
matches = rz_regex_match_all_overlap_32(re, utf32_he, RZ_REGEX_ZERO_TERMINATED, 0, RZ_REGEX_DEFAULT);
mu_assert_notnull(matches, "matches was not set");
mu_assert_eq(rz_pvector_len(matches), 4, "matches len was wrong");
match_groups = rz_pvector_at(matches, 0);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 15, "match.start wrong");
mu_assert_eq(match->len, 3, "match.len wrong");
match_groups = rz_pvector_at(matches, 1);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 23, "match.start wrong");
mu_assert_eq(match->len, 3, "match.len wrong");
match_groups = rz_pvector_at(matches, 2);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 24, "match.start wrong");
mu_assert_eq(match->len, 3, "match.len wrong");
match_groups = rz_pvector_at(matches, 3);
mu_assert_eq(rz_pvector_len(match_groups), 1, "num of match groups was wrong");
match = rz_pvector_at(match_groups, 0);
mu_assert_eq(match->start, 25, "match.start wrong");
mu_assert_eq(match->len, 3, "match.len wrong");
rz_pvector_free(matches);
rz_regex_free(re);
mu_end;
}
int main() {
mu_run_test(test_rz_regex_all_match);
mu_run_test(test_rz_regex_extend_space);
@ -220,4 +408,7 @@ int main() {
mu_run_test(test_rz_regex_named_matches);
mu_run_test(test_rz_regex_posix_blank);
mu_run_test(test_rz_regex_find);
mu_run_test(test_rz_regex_match_all_native_utf8);
mu_run_test(test_rz_regex_match_all_native_utf16);
mu_run_test(test_rz_regex_match_all_native_utf32);
}

131
test/unit/test_str.c
View file

@ -3,6 +3,7 @@
#include <rz_util.h>
#include "minunit.h"
#include <rz_types.h>
// TODO test rz_str_chop_path
@ -780,6 +781,132 @@ bool test_rz_str_isXutf8(void) {
mu_end;
}
bool test_rz_str_utf8_conversions(void) {
const char *needs_4 = "a";
const char *needs_6 = "🍍";
const char *needs_22 = "aa🍍🍍🍍aa";
const char *only_nul_needs_2 = "";
mu_assert_eq(rz_str_utf8_get_width_utf16(only_nul_needs_2), 2, "Should have been 0 + 2 = 2.");
mu_assert_eq(rz_str_utf8_get_width_utf16(needs_4), 4, "Should have been 2 + 2 = 4.");
mu_assert_eq(rz_str_utf8_get_width_utf16(needs_6), 6, "Should have been 4 + 2 = 6.");
mu_assert_eq(rz_str_utf8_get_width_utf16(needs_22), 22, "Should have been 20 + 2 = 22.");
mu_end;
}
bool test_rz_str_utf8_count_ucp(void) {
const char *a = "a";
const char *pine = "🍍";
const char *apine = "aa🍍🍍🍍aa";
const char *nul = "";
mu_assert_eq(rz_str_utf8_num_ucp(nul), 1, "Should have been 1 code point.");
mu_assert_eq(rz_str_utf8_num_ucp(a), 2, "Should have been 2 code points.");
mu_assert_eq(rz_str_utf8_num_ucp(pine), 2, "Should have been 2 code points.");
mu_assert_eq(rz_str_utf8_num_ucp(apine), 8, "Should have been 8 code points.");
mu_end;
}
bool test_rz_str_utf8_to_utf16(void) {
const char *a = "a";
const ut8 a16_le[] = { 0x61, 0x00, 0x00, 0x00 };
const ut8 a16_be[] = { 0x00, 0x61, 0x00, 0x00 };
const char *pine = "🍍";
const ut8 pine16_le[] = { 0x3c, 0xd8, 0x4d, 0xdf, 0x00, 0x00 };
const ut8 pine16_be[] = { 0xd8, 0x3c, 0xdf, 0x4d, 0x00, 0x00 };
const char *apine = "aa🍍🍍🍍aa";
const ut8 apine16_le[] = { 0x61, 0x00, 0x61, 0x00, 0x3c, 0xd8, 0x4d, 0xdf, 0x3c, 0xd8, 0x4d, 0xdf, 0x3c, 0xd8, 0x4d, 0xdf, 0x61, 0x00, 0x61, 0x00, 0x00, 0x00 };
const ut8 apine16_be[] = { 0x00, 0x61, 0x00, 0x61, 0xd8, 0x3c, 0xdf, 0x4d, 0xd8, 0x3c, 0xdf, 0x4d, 0xd8, 0x3c, 0xdf, 0x4d, 0x00, 0x61, 0x00, 0x61, 0x00, 0x00 };
const char *nul = "";
const ut8 nul16_le[] = { 0x0, 0x0 };
const ut8 nul16_be[] = { 0x0, 0x0 };
ut16 *out = rz_str_utf8_to_utf16(a, true);
mu_assert_memeq((ut8 *)out, a16_be, sizeof(a16_be), "string mismatch");
free(out);
out = rz_str_utf8_to_utf16(a, false);
mu_assert_memeq((ut8 *)out, a16_le, sizeof(a16_le), "string mismatch");
free(out);
out = rz_str_utf8_to_utf16(pine, true);
mu_assert_memeq((ut8 *)out, pine16_be, sizeof(pine16_be), "string mismatch");
free(out);
out = rz_str_utf8_to_utf16(pine, false);
mu_assert_memeq((ut8 *)out, pine16_le, sizeof(pine16_le), "string mismatch");
free(out);
out = rz_str_utf8_to_utf16(apine, true);
mu_assert_memeq((ut8 *)out, apine16_be, sizeof(apine16_be), "string mismatch");
free(out);
out = rz_str_utf8_to_utf16(apine, false);
mu_assert_memeq((ut8 *)out, apine16_le, sizeof(apine16_le), "string mismatch");
free(out);
out = rz_str_utf8_to_utf16(apine, RZ_HOST_IS_BIG_ENDIAN);
mu_assert_memeq((ut8 *)out, RZ_HOST_IS_BIG_ENDIAN ? apine16_be : apine16_le, RZ_HOST_IS_BIG_ENDIAN ? sizeof(apine16_be) : sizeof(apine16_le), "string with host endian mismatches");
free(out);
out = rz_str_utf8_to_utf16(nul, true);
mu_assert_memeq((ut8 *)out, nul16_be, sizeof(nul16_be), "string mismatch");
free(out);
out = rz_str_utf8_to_utf16(nul, false);
mu_assert_memeq((ut8 *)out, nul16_le, sizeof(nul16_le), "string mismatch");
free(out);
mu_end;
}
bool test_rz_str_utf8_to_utf32(void) {
const char *a = "a";
const ut8 a32_le[] = { 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
const ut8 a32_be[] = { 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x00 };
const char *pine = "🍍";
const ut8 pine32_le[] = { 0x4d, 0xf3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00 };
const ut8 pine32_be[] = { 0x00, 0x01, 0xf3, 0x4d, 0x00, 0x00, 0x00, 0x00 };
const char *apine = "aa🍍🍍🍍aa";
const ut8 apine32_le[] = { 0x61, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x4d, 0xf3, 0x01, 0x00, 0x4d, 0xf3, 0x01, 0x00, 0x4d, 0xf3, 0x01, 0x00, 0x61, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
const ut8 apine32_be[] = { 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x61, 0x00, 0x01, 0xf3, 0x4d, 0x00, 0x01, 0xf3, 0x4d, 0x00, 0x01, 0xf3, 0x4d, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x00 };
const char *nul = "";
const ut8 nul32_le[] = { 0x0, 0x00, 0x00, 0x0 };
const ut8 nul32_be[] = { 0x0, 0x00, 0x00, 0x0 };
ut32 *out = rz_str_utf8_to_utf32(a, true);
mu_assert_memeq((ut8 *)out, a32_be, sizeof(a32_be), "string mismatch");
free(out);
out = rz_str_utf8_to_utf32(a, false);
mu_assert_memeq((ut8 *)out, a32_le, sizeof(a32_le), "string mismatch");
free(out);
out = rz_str_utf8_to_utf32(pine, true);
mu_assert_memeq((ut8 *)out, pine32_be, sizeof(pine32_be), "string mismatch");
free(out);
out = rz_str_utf8_to_utf32(pine, false);
mu_assert_memeq((ut8 *)out, pine32_le, sizeof(pine32_le), "string mismatch");
free(out);
out = rz_str_utf8_to_utf32(apine, true);
mu_assert_memeq((ut8 *)out, apine32_be, sizeof(apine32_be), "string mismatch");
free(out);
out = rz_str_utf8_to_utf32(apine, false);
mu_assert_memeq((ut8 *)out, apine32_le, sizeof(apine32_le), "string mismatch");
free(out);
out = rz_str_utf8_to_utf32(apine, RZ_HOST_IS_BIG_ENDIAN);
mu_assert_memeq((ut8 *)out, RZ_HOST_IS_BIG_ENDIAN ? apine32_be : apine32_le, RZ_HOST_IS_BIG_ENDIAN ? sizeof(apine32_be) : sizeof(apine32_le), "string with host endian mismatches");
free(out);
out = rz_str_utf8_to_utf32(nul, true);
mu_assert_memeq((ut8 *)out, nul32_be, sizeof(nul32_be), "string mismatch");
free(out);
out = rz_str_utf8_to_utf32(nul, false);
mu_assert_memeq((ut8 *)out, nul32_le, sizeof(nul32_le), "string mismatch");
free(out);
mu_end;
}
bool all_tests() {
mu_run_test(test_rz_str_newf);
mu_run_test(test_rz_str_replace_char_once);
@ -823,6 +950,10 @@ bool all_tests() {
mu_run_test(test_rz_str_filter);
mu_run_test(test_rz_str_strchr);
mu_run_test(test_rz_str_isXutf8);
mu_run_test(test_rz_str_utf8_conversions);
mu_run_test(test_rz_str_utf8_count_ucp);
mu_run_test(test_rz_str_utf8_to_utf16);
mu_run_test(test_rz_str_utf8_to_utf32);
return tests_passed != tests_run;
}

2
test/unit/test_str_search.c
View file

@ -8,7 +8,7 @@ static RzUtilStrScanOptions g_opt = {
.max_str_length = 2048,
.min_str_length = 4,
.prefer_big_endian = false,
.check_ascii_freq = true
.check_ascii_freq = true,
};
bool test_rz_scan_strings_detect_ascii(void) {