portugalfuturista/rizin
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

librz/util: UTF-16/32 to UTF-8 conversion (#5431)

Browse source 
* Replace rz_str_utf16_to_utf8 with one matching the other string convert functions.
* Add rz_str_utf32_to_utf8
This commit is contained in:
Rot127 2025-10-05 11:16:53 +00:00 committed by GitHub
parent 1c5b3e6616
commit 92eff78276
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 185 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
librz/bin/format/dmp/dmp64.c
View file

@ -208,18 +208,19 @@ static bool rz_bin_dmp64_init_triage_drivers(struct rz_bin_dmp64_obj_t *obj) {
return false;
}
ut8 *file = calloc(str.count + 1, sizeof(ut16));
ut8 *file_utf8 = calloc(str.count + 1, sizeof(ut16));
if (!file || !file_utf8) {
if (!file) {
free(driver);
free(file);
free(file_utf8);
return false;
}
rz_buf_read(obj->b, file, str.count * sizeof(ut16));
const size_t size = (str.count + 1) * sizeof(ut16);
rz_str_utf16_to_utf8(file_utf8, size, file, size, true);
driver->file = (char *)file_utf8;
driver->file = (char *)rz_str_utf16_to_utf8(file, size, false);
free(file);
if (!driver) {
free(driver);
return false;
}
rz_list_push(obj->drivers, driver);
address += sizeof(dmp_driver_entry64);
}

3
librz/bin/p/bin_mdmp.c
View file

@ -269,12 +269,11 @@ static RzPVector /*<RzBinSection *>*/ *mdmp_sections(RzBinFile *bf) {
return ret;
}
ptr->name = RZ_NEWS0(char, ptr_name_len);
ptr->name = (char *)rz_str_utf16_to_utf8(str_buffer, str_length, false);
if (!ptr->name) {
free(ptr);
continue;
}
rz_str_utf16_to_utf8((ut8 *)ptr->name, str_length * 4, str_buffer, str_length, true);
ptr->vaddr = module->base_of_image;
ptr->vsize = module->size_of_image;
ptr->paddr = rz_bin_mdmp_get_paddr(obj, ptr->vaddr);

8
librz/core/cbin.c
View file

@ -4415,11 +4415,9 @@ static void bin_pe_versioninfo(RzCore *r, PJ *pj, int mode) {
int lenval = 0;
ut8 *key_utf16 = sdb_decode(sdb_const_get(sdb, "key"), &lenkey);
ut8 *val_utf16 = sdb_decode(sdb_const_get(sdb, "value"), &lenval);
ut8 *key_utf8 = calloc(lenkey * 2, 1);
ut8 *val_utf8 = calloc(lenval * 2, 1);
if (!key_utf8 || !val_utf8 ||
rz_str_utf16_to_utf8(key_utf8, lenkey * 2, key_utf16, lenkey, true) < 0 ||
rz_str_utf16_to_utf8(val_utf8, lenval * 2, val_utf16, lenval, true) < 0) {
ut8 *key_utf8 = rz_str_utf16_to_utf8(key_utf16, lenkey, false);
ut8 *val_utf8 = rz_str_utf16_to_utf8(val_utf16, lenval, false);
if (!key_utf8 || !val_utf8 || !key_utf16 || !val_utf16) {
RZ_LOG_WARN("core: cannot decode utf16 to utf8\n");
} else if (IS_MODE_JSON(mode)) {
pj_ks(pj, (char *)key_utf8, (char *)val_utf8);

1
librz/include/rz_util/rz_str.h
View file

@ -238,7 +238,6 @@ RZ_API RZ_OWN char *rz_str_format_msvc_argv(size_t argc, const char **argv);
RZ_API void rz_str_uri_decode(char *buf);
RZ_API char *rz_str_uri_encode(const char *buf);
RZ_API char *rz_str_utf16_decode(const ut8 *s, int len);
RZ_API int rz_str_utf16_to_utf8(ut8 *dst, int len_dst, const ut8 *src, int len_src, bool little_endian);
RZ_DEPRECATE RZ_API char *rz_str_utf16_encode(const char *s, int len);
RZ_API RZ_OWN ut16 *rz_str_utf8_to_utf16(RZ_NONNULL const char *utf8_str, bool big_endian);
RZ_API RZ_OWN ut32 *rz_str_utf8_to_utf32(RZ_NONNULL const char *utf8_str, bool big_endian);

1
librz/include/rz_util/rz_utf16.h
View file

@ -18,5 +18,6 @@ RZ_API size_t rz_utf16le_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NU
RZ_API size_t rz_utf16be_decode(RZ_NONNULL const ut8 *buf, size_t buf_len, RZ_NULLABLE RZ_OUT RzCodePoint *ch, bool check_is_def);
RZ_API size_t rz_utf16_encode(RZ_NONNULL RZ_OUT ut8 *buf, RzCodePoint ch, bool big_endian);
RZ_API bool rz_utf16_is_printable_code_point(RZ_NONNULL const ut8 *buf, size_t buf_len, bool big_endian, size_t lookahead);
RZ_API RZ_OWN ut8 *rz_str_utf16_to_utf8(RZ_NONNULL const ut8 *src, size_t len_src, bool little_endian);
#endif // RZ_UTF16_H

1
librz/include/rz_util/rz_utf32.h
View file

@ -22,5 +22,6 @@ RZ_API int rz_utf32le_decode(const ut8 *ptr, int ptrlen, RZ_NULLABLE RZ_OUT RzCo
RZ_API int rz_utf32be_decode(const ut8 *ptr, int ptrlen, RZ_NULLABLE RZ_OUT RzCodePoint *ch, bool check_validity);
RZ_API size_t rz_utf32_encode(RZ_NONNULL RZ_OUT ut8 *buf, RzCodePoint ch, bool big_endian);
RZ_API bool rz_utf32_valid_code_point(RZ_NONNULL const ut8 *buf, size_t buf_len, bool big_endian, size_t lookahead);
RZ_API RZ_OWN ut8 *rz_str_utf32_to_utf8(RZ_NONNULL const ut8 *src, size_t len_src, bool little_endian);
#endif // RZ_UTF32_H

7
librz/util/axml.c
View file

@ -172,16 +172,13 @@ static char *string_lookup(string_pool_t *pool, const ut8 *data, ut64 data_size,
// Size of UTF-16LE without NULL
n *= 2;
name = calloc(n * 2 + 1, 1);
if ((uintptr_t)start16 > (uintptr_t)data + data_size - sizeof(ut32) - n - 1) {
free(name);
return NULL;
}
name = (char *)rz_str_utf16_to_utf8((const ut8 *)start16, n, false);
// If UTF-16LE, decode to UTF-8 so we can print it to the screen
if (rz_str_utf16_to_utf8((ut8 *)name, n * 2, (const ut8 *)start16, n, true) < 0) {
free(name);
if (!name) {
RZ_LOG_ERROR("Failed to decode UTF16-LE\n");
return NULL;
}

61
librz/util/str.c
View file

@ -3048,67 +3048,6 @@ RZ_API char *rz_str_uri_encode(const char *s) {
return trimDown ? trimDown : od;
}
RZ_API int rz_str_utf16_to_utf8(ut8 *dst, int len_dst, const ut8 *src, int len_src, bool little_endian) {
ut8 *outstart = dst;
ut8 *outend = dst + len_dst;
ut16 *in = (ut16 *)src;
ut16 *inend;
ut32 c, d, inlen;
int bits;
if ((len_src % 2) == 1) {
len_src--;
}
inlen = len_src / 2;
inend = in + inlen;
while ((in < inend) && (dst - outstart + 5 < len_dst)) {
c = rz_read_ble16((const ut8 *)in, !little_endian);
in++;
if ((c & 0xFC00) == 0xD800) { /* surrogates */
if (in >= inend) { /* (in > inend) shouldn't happens */
break;
}
d = rz_read_ble16((const ut8 *)in, !little_endian);
in++;
if ((d & 0xFC00) == 0xDC00) {
c &= 0x03FF;
c <<= 10;
c |= d & 0x03FF;
c += 0x10000;
} else {
return -2;
}
}
/* assertion: c is a single UTF-4 value */
if (dst >= outend) {
break;
}
if (c < 0x80) {
*dst++ = c;
bits = -6;
} else if (c < 0x800) {
*dst++ = ((c >> 6) & 0x1F) | 0xC0;
bits = 0;
} else if (c < 0x10000) {
*dst++ = ((c >> 12) & 0x0F) | 0xE0;
bits = 6;
} else {
*dst++ = ((c >> 18) & 0x07) | 0xF0;
bits = 12;
}
for (; bits >= 0; bits -= 6) {
if (dst >= outend) {
break;
}
*dst++ = ((c >> bits) & 0x3F) | 0x80;
}
}
len_dst = dst - outstart;
return len_dst;
}
RZ_API char *rz_str_utf16_decode(const ut8 *s, int len) {
int i = 0;
int j = 0;

40
librz/util/utf16.c
View file

@ -195,3 +195,43 @@ RZ_API bool rz_utf16_is_printable_code_point(RZ_NONNULL const ut8 *buf, size_t b
}
return true;
}
/**
* \brief Converts the \p utf16_str into an UTF-8 string.
* The conversion will stop if there are any encoding or decoding issues
* (e.g. any code point is larger than RZ_UNICODE_LAST_CODE_POINT or a surrogate).
*
* \param utf16_str The UTF-16 string to convert to UTF-8. It must be at least
* 2 bytes long.
* \param len The len of \p utf16_str in bytes.
* \param big_endian Flag if \p ut16_str is encoded in big endian.
*
* \return The restulting UTF-8 string or NULL in case of failure.
*/
RZ_API RZ_OWN ut8 *rz_str_utf16_to_utf8(RZ_NONNULL const ut8 *utf16_str, size_t len, bool big_endian) {
rz_return_val_if_fail(utf16_str && len > 1, NULL);
// Worst case each 2 bytes UTF-16 character requires 3 bytes in UTF-8.
// Each 4 bytes UTF-16 character also needs 4 bytes in UTF-8.
size_t len_dst = len * 2;
ut8 *dst = RZ_NEWS0(ut8, len_dst);
size_t k = 0;
for (size_t i = 0; i < len;) {
RzCodePoint ucp;
size_t x = rz_utf16_decode(utf16_str + i, len - i, &ucp, false, big_endian);
if (!x) {
// Failed to decode.
goto return_dst;
}
i += x;
size_t utf8_bytes = rz_utf8_encode(dst + k, ucp);
if (!utf8_bytes) {
// Code point is larger than the last Unicode code point.
goto return_dst;
}
k += utf8_bytes;
}
return_dst:
return dst;
}

37
librz/util/utf32.c
View file

@ -110,3 +110,40 @@ RZ_API size_t rz_utf32_encode(RZ_NONNULL RZ_OUT ut8 *buf, RzCodePoint ucp, bool
buf[3] = (ucp >> 24) & 0xff;
return 4;
}
/**
* \brief Converts the \p utf32_str into an UTF-8 string.
* The conversion will stop if there are any encoding or decoding issues
* (e.g. any code point is larger than RZ_UNICODE_LAST_CODE_POINT or a surrogate).
*
* \param utf32_str The UTF-32 string to convert to UTF-8. It must be at least
* 4 bytes long.
* \param len The len of \p utf32_str in bytes.
* \param big_endian Flag if \p ut32_str is encoded in big endian.
*
* \return The resulting UTF-8 string or NULL in case of failure.
*/
RZ_API RZ_OWN ut8 *rz_str_utf32_to_utf8(RZ_NONNULL const ut8 *utf32_str, size_t len, bool big_endian) {
rz_return_val_if_fail(utf32_str && len > 3, NULL);
// Worst case each character is also 4 bytes in UTF-8.
size_t len_dst = len + 1;
ut8 *dst = RZ_NEWS0(ut8, len_dst);
size_t k = 0;
for (size_t i = 0; i < len; i += 4) {
RzCodePoint ucp;
if (!rz_utf32_decode(utf32_str + i, len - i, &ucp, false, big_endian)) {
// Failed to decode.
goto return_dst;
}
size_t utf8_bytes = rz_utf8_encode(dst + k, ucp);
if (!utf8_bytes) {
// Code point is larger than the last Unicode code point.
goto return_dst;
}
k += utf8_bytes;
}
return_dst:
return dst;
}

5
subprojects/rzwinkd/winkd.c
View file

@ -733,12 +733,11 @@ RzList /*<WindModule *>*/ *winkd_list_modules(RZ_BORROW RZ_NONNULL WindCtx *ctx)
}
read_at_uva_or_kernel(ctx, bufferaddr, unname, length);
mod->name = calloc((ut64)length + 1, 1);
mod->name = (char *)rz_str_utf16_to_utf8(unname, length + 2, false);
free(unname);
if (!mod->name) {
break;
}
rz_str_utf16_to_utf8((ut8 *)mod->name, length + 1, unname, length + 2, true);
free(unname);
rz_list_add_sorted(ret, mod, map_comparator, NULL);
ptr = next;

92
test/unit/test_str.c
View file

@ -907,6 +907,96 @@ bool test_rz_str_utf8_to_utf32(void) {
mu_end;
}
bool test_rz_str_utf16_to_utf8(void) {
const ut8 a[] = "a";
const ut8 a16_le[] = { 0x61, 0x00, 0x00, 0x00 };
const ut8 a16_be[] = { 0x00, 0x61, 0x00, 0x00 };
const ut8 pine[] = "🍍";
const ut8 pine16_le[] = { 0x3c, 0xd8, 0x4d, 0xdf, 0x00, 0x00 };
const ut8 pine16_be[] = { 0xd8, 0x3c, 0xdf, 0x4d, 0x00, 0x00 };
const ut8 apine[] = "aa🍍🍍🍍aa";
const ut8 apine16_le[] = { 0x61, 0x00, 0x61, 0x00, 0x3c, 0xd8, 0x4d, 0xdf, 0x3c, 0xd8, 0x4d, 0xdf, 0x3c, 0xd8, 0x4d, 0xdf, 0x61, 0x00, 0x61, 0x00, 0x00, 0x00 };
const ut8 apine16_be[] = { 0x00, 0x61, 0x00, 0x61, 0xd8, 0x3c, 0xdf, 0x4d, 0xd8, 0x3c, 0xdf, 0x4d, 0xd8, 0x3c, 0xdf, 0x4d, 0x00, 0x61, 0x00, 0x61, 0x00, 0x00 };
const ut8 nul[] = "";
const ut8 nul16_le[] = { 0x0, 0x0 };
const ut8 nul16_be[] = { 0x0, 0x0 };
ut8 *out = rz_str_utf16_to_utf8(a16_be, sizeof(a16_be), true);
mu_assert_memeq(out, a, sizeof(a), "string mismatch");
free(out);
out = rz_str_utf16_to_utf8(a16_le, sizeof(a16_le), false);
mu_assert_memeq(out, a, sizeof(a), "string mismatch");
free(out);
out = rz_str_utf16_to_utf8(pine16_be, sizeof(pine16_be), true);
mu_assert_memeq(out, pine, sizeof(pine), "string mismatch");
free(out);
out = rz_str_utf16_to_utf8(pine16_le, sizeof(pine16_le), false);
mu_assert_memeq(out, pine, sizeof(pine), "string mismatch");
free(out);
out = rz_str_utf16_to_utf8(apine16_be, sizeof(apine16_be), true);
mu_assert_memeq(out, apine, sizeof(apine), "string mismatch");
free(out);
out = rz_str_utf16_to_utf8(apine16_le, sizeof(apine16_le), false);
mu_assert_memeq(out, apine, sizeof(apine), "string mismatch");
free(out);
out = rz_str_utf16_to_utf8(nul16_be, sizeof(nul16_be), true);
mu_assert_memeq(out, nul, sizeof(nul), "string mismatch");
free(out);
out = rz_str_utf16_to_utf8(nul16_le, sizeof(nul16_le), false);
mu_assert_memeq(out, nul, sizeof(nul), "string mismatch");
free(out);
mu_end;
}
bool test_rz_str_utf32_to_utf8(void) {
const ut8 a[] = "a";
const ut8 a32_le[] = { 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
const ut8 a32_be[] = { 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x00 };
const ut8 pine[] = "🍍";
const ut8 pine32_le[] = { 0x4d, 0xf3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00 };
const ut8 pine32_be[] = { 0x00, 0x01, 0xf3, 0x4d, 0x00, 0x00, 0x00, 0x00 };
const ut8 apine[] = "aa🍍🍍🍍aa";
const ut8 apine32_le[] = { 0x61, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x4d, 0xf3, 0x01, 0x00, 0x4d, 0xf3, 0x01, 0x00, 0x4d, 0xf3, 0x01, 0x00, 0x61, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
const ut8 apine32_be[] = { 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x61, 0x00, 0x01, 0xf3, 0x4d, 0x00, 0x01, 0xf3, 0x4d, 0x00, 0x01, 0xf3, 0x4d, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x00 };
const ut8 nul[] = "";
const ut8 nul32_le[] = { 0x0, 0x00, 0x00, 0x0 };
const ut8 nul32_be[] = { 0x0, 0x00, 0x00, 0x0 };
ut8 *out = rz_str_utf32_to_utf8(a32_be, sizeof(a32_be), true);
mu_assert_memeq(out, a, sizeof(a), "string mismatch");
free(out);
out = rz_str_utf32_to_utf8(a32_le, sizeof(a32_le), false);
mu_assert_memeq(out, a, sizeof(a), "string mismatch");
free(out);
out = rz_str_utf32_to_utf8(pine32_be, sizeof(pine32_be), true);
mu_assert_memeq(out, pine, sizeof(pine), "string mismatch");
free(out);
out = rz_str_utf32_to_utf8(pine32_le, sizeof(pine32_le), false);
mu_assert_memeq(out, pine, sizeof(pine), "string mismatch");
free(out);
out = rz_str_utf32_to_utf8(apine32_be, sizeof(apine32_be), true);
mu_assert_memeq(out, apine, sizeof(apine), "string mismatch");
free(out);
out = rz_str_utf32_to_utf8(apine32_le, sizeof(apine32_le), false);
mu_assert_memeq(out, apine, sizeof(apine), "string mismatch");
free(out);
out = rz_str_utf32_to_utf8(nul32_be, sizeof(nul32_be), true);
mu_assert_memeq(out, nul, sizeof(nul), "string mismatch");
free(out);
out = rz_str_utf32_to_utf8(nul32_le, sizeof(nul32_le), false);
mu_assert_memeq(out, nul, sizeof(nul), "string mismatch");
free(out);
mu_end;
}
bool all_tests() {
mu_run_test(test_rz_str_newf);
mu_run_test(test_rz_str_replace_char_once);
@ -954,6 +1044,8 @@ bool all_tests() {
mu_run_test(test_rz_str_utf8_count_ucp);
mu_run_test(test_rz_str_utf8_to_utf16);
mu_run_test(test_rz_str_utf8_to_utf32);
mu_run_test(test_rz_str_utf16_to_utf8);
mu_run_test(test_rz_str_utf32_to_utf8);
return tests_passed != tests_run;
}