Add JSON projection grep (#6522)

librz/cons/cons.c

@@ -59,8 +59,14 @@ static void cons_stack_free(void *ptr) {
 	RzConsStack *s = (RzConsStack *)ptr;
 	free(s->buf);
 	if (s->grep) {
+		if (CTX(grep.str) == s->grep->str) {
+			CTX(grep.str) = NULL;
+		}
 		RZ_FREE(s->grep->str);
-		CTX(grep.str) = NULL;
+		if (CTX(grep.json_path) == s->grep->json_path) {
+			CTX(grep.json_path) = NULL;
+		}
+		RZ_FREE(s->grep->json_path);
 	}
 	free(s->grep);
 	free(s);
@@ -81,6 +87,9 @@ static RzConsStack *cons_stack_dump(bool recreate) {
 			if (CTX(grep).str) {
 				data->grep->str = rz_str_dup(CTX(grep).str);
 			}
+			if (CTX(grep).json_path) {
+				data->grep->json_path = rz_str_dup(CTX(grep).json_path);
+			}
 		}
 		if (recreate && CTX(buffer_sz) > 0) {
 			CTX(buffer) = malloc(CTX(buffer_sz));
@@ -108,6 +117,7 @@ static void cons_stack_load(RzConsStack *data, bool free_current) {
 	CTX(buffer_sz) = data->buf_size;
 	if (data->grep) {
 		free(CTX(grep).str);
+		free(CTX(grep).json_path);
 		memcpy(&CTX(grep), data->grep, sizeof(RzConsGrep));
 	}
 	CTX(noflush) = data->noflush;
@@ -843,6 +853,7 @@ RZ_API void rz_cons_clear(void) {
 
 static void cons_grep_reset(RzConsGrep *grep) {
 	RZ_FREE(grep->str);
+	RZ_FREE(grep->json_path);
 	RZ_FREE(grep->sorted_lines);
 	RZ_FREE(grep->unsorted_lines);
 	ZERO_FILL(*grep);

librz/cons/grep.c

@@ -2,9 +2,13 @@
 // SPDX-FileCopyrightText: 2009-2020 nibble <nibble.ds@gmail.com>
 // SPDX-License-Identifier: LGPL-3.0-only
 
+#include <rz_util/rz_regex.h>
+#include <rz_util/rz_str.h>
+#include <rz_util/rz_strbuf.h>
 #include <rz_cons.h>
 #include <rz_util/rz_print.h>
 #include <sdb.h>
+#include <string.h>
 
 #define I(x) rz_cons_singleton()->x
 
@@ -20,6 +24,133 @@ static char *strchr_ns(char *s, const char ch) {
 	return p;
 }
 
+/**
+ * \brief Duplicate the JSON path inside a grep `{...}` expression.
+ * \param str Expression starting at `{`.
+ * \return Path string, or NULL on missing delimiter.
+ */
+static RZ_OWN char *json_path_dup(RZ_BORROW const char *str, ut32 size) {
+	rz_return_val_if_fail(str && *str == '{', NULL);
+	const char *start = str + 1;
+	const char *end = rz_str_rchr(start, str + size - 1, '}');
+	return end ? rz_str_ndup(start, end - start) : NULL;
+}
+
+/**
+ * \brief Project selected keys from a JSON array of objects.
+ * \param json Parsed JSON input.
+ * \param projection Parsed projection path.
+ * \return Compact JSON string, or NULL on malformed projection/allocation failure.
+ */
+static RZ_OWN char *json_path_array_projection(RZ_BORROW const RzJson *json, const char *json_path) {
+	rz_return_val_if_fail(json && json_path, NULL);
+	RzStrBuf *match = rz_regex_full_match_str("^[[:space:]]*\\.[[:space:]]*\\[[[:space:]]*\\][[:space:]]*\\\\?\\|[[:space:]]*\\{[[:space:]]*\\K[^{}|]+?(?=[[:space:]]*\\}[[:space:]]*$)", json_path, RZ_REGEX_ZERO_TERMINATED, RZ_REGEX_DEFAULT, RZ_REGEX_DEFAULT, ",");
+	RzList *fields = rz_str_split_duplist(rz_strbuf_get(match), ",", true);
+	rz_strbuf_free(match);
+	if (!fields) {
+		return NULL;
+	}
+
+	RzListIter *it;
+	const char *field;
+	rz_list_foreach (fields, it, field) {
+		bool is_valid = !RZ_STR_ISEMPTY(field);
+		const char *field_ch;
+		for (field_ch = field; is_valid && *field_ch; field_ch++) {
+			switch (*field_ch) {
+			case '.':
+			case '[':
+			case ']':
+			case '{':
+			case '}':
+			case '|':
+			case ',':
+				is_valid = false;
+				break;
+			default: break;
+			}
+		}
+		if (!is_valid) {
+			rz_list_free(fields);
+			return NULL;
+		}
+	}
+
+	if (json->type != RZ_JSON_ARRAY) {
+		rz_list_free(fields);
+		return rz_str_dup("[]");
+	}
+
+	PJ *pj = pj_new();
+	if (!pj) {
+		rz_list_free(fields);
+		return NULL;
+	}
+	pj_a(pj);
+	const RzJson *item;
+	for (item = json->children.first; item; item = item->next) {
+		if (item->type != RZ_JSON_OBJECT) {
+			continue;
+		}
+		pj_o(pj);
+		rz_list_foreach (fields, it, field) {
+			const RzJson *value = NULL;
+			const RzJson *child;
+			for (child = item->children.first; child; child = child->next) {
+				if (child->key && RZ_STR_EQ(child->key, field)) {
+					value = child;
+					break;
+				}
+			}
+			if (value) {
+				char *value_json = rz_json_as_string(value, true);
+				if (!value_json) {
+					pj_free(pj);
+					rz_list_free(fields);
+					return NULL;
+				}
+				pj_j(pj, value_json);
+				free(value_json);
+			}
+		}
+		pj_end(pj);
+	}
+	pj_end(pj);
+	rz_list_free(fields);
+	return pj_drain(pj);
+}
+
+static char *json_path_eval(const char *json_string_in, const char *json_path) {
+	char *json_text = rz_str_dup(json_string_in);
+	if (!json_text) {
+		return NULL;
+	}
+	RzJson *json = rz_json_parse(json_text);
+	if (!json) {
+		free(json_text);
+		return NULL;
+	}
+	char *out = json_path_array_projection(json, json_path);
+	if (!out) {
+		const RzJson *out_json;
+		// To simplify grep syntax we omit brackets in `[0]` for JSON paths
+		if (*json_path != '[' && *json_path != '.') {
+			char *tmppath = rz_str_newf("[%s]", json_path);
+			out_json = rz_json_get_path(json, tmppath);
+			free(tmppath);
+		} else {
+			out_json = rz_json_get_path(json, json_path);
+		}
+		// When we receive the path, it's fetched with the key name
+		// We should get only the value
+		out = out_json ? rz_json_as_string(out_json, false) : NULL;
+	}
+
+	rz_json_free(json);
+	free(json_text);
+	return out;
+}
+
 static const char *help_detail_tilde[] = {
 	"Usage: [command]~[modifier][word,word][endmodifier][[column]][:line]\n"
 	"modifier:",
@@ -132,10 +263,8 @@ static void parse_grep_expression(const char *str) {
 					grep->less = 1;
 				}
 			} else {
-				char *jsonPath = rz_str_dup(str + 1);
+				char *jsonPath = json_path_dup(str, strlen(str));
-				char *jsonPathEnd = strchr(jsonPath, '}');
+				if (jsonPath) {
-				if (jsonPathEnd) {
-					*jsonPathEnd = 0;
 					free(grep->json_path);
 					grep->json_path = jsonPath;
 					grep->json = 1;
@@ -519,37 +648,15 @@ RZ_API void rz_cons_grepbuf(void) {
 	}
 	if (grep->json) {
 		if (grep->json_path) {
-			RzJson *json = rz_json_parse(cons->context->buffer);
+			char *out = json_path_eval(cons->context->buffer, grep->json_path);
-			if (!json) {
+			if (out) {
-				RZ_FREE(grep->json_path);
-				return;
-			}
-			const RzJson *excerpt;
-			// To simplify grep syntax we omit brackets in `[0]` for JSON paths
-			if (*grep->json_path != '[' && *grep->json_path != '.') {
-				char *tmppath = rz_str_newf("[%s]", grep->json_path);
-				excerpt = rz_json_get_path(json, tmppath);
-				free(tmppath);
-			} else {
-				excerpt = rz_json_get_path(json, grep->json_path);
-			}
-			if (excerpt) {
-				// When we receive the path, it's fetched with the key name
-				// We should get only the value
-				char *u = rz_json_as_string(excerpt, false);
-				if (!u) {
-					RZ_FREE(grep->json_path);
-					rz_json_free(json);
-					return;
-				}
 				free(cons->context->buffer);
-				cons->context->buffer = u;
+				cons->context->buffer = out;
-				cons->context->buffer_len = strlen(u);
+				cons->context->buffer_len = strlen(out);
 				cons->context->buffer_sz = cons->context->buffer_len + 1;
 				grep->json = 0;
 				rz_cons_newline();
 			}
-			rz_json_free(json);
 			RZ_FREE(grep->json_path);
 		} else {
 			const char *palette[] = {

librz/util/pj.c

@@ -183,6 +183,7 @@ RZ_API PJ *pj_kb(PJ *j, const char *k, bool v) {
 
 RZ_API PJ *pj_null(PJ *j) {
 	rz_return_val_if_fail(j, j);
+	pj_comma(j);
 	pj_raw(j, "null");
 	return j;
 }

test/db/cmd/cmd_json

@@ -30,6 +30,34 @@ EXPECT=<<EOF
 EOF
 RUN
 
+NAME=json path array object projection
+FILE==
+CMDS=<<EOF
+echo [{\"s\":\"alpha\",\"n\":1,\"b\":true,\"nil\":null,\"obj\":{\"x\":1},\"arr\":[1,2]},{\"s\":\"beta\",\"n\":2,\"b\":false,\"obj\":{\"y\":\"z\"}},{\"s\":\"gamma\",\"arr\":[]}]~{.[]\|{s,n,b,nil,obj,arr,missing}}
+echo [{\"k\":1},7,\"x\",null,true,[],{\"z\":2},{\"k\":3}]~{.[]\|{k}}
+echo []~{ .[] \| { key1, key2 } }
+echo [{\"key1\":1}]~{ . [ ] \| { key1 } }
+echo [{\"a b\":1},{\"a\":2}]~{.[]\|{a b}}
+echo [{\"arr\":[null,1]}]~{.[]\|{arr}}
+echo [{\"z\":1},{\"z\":2}]~{.[]\|{key1,key2}}
+echo [{\"offset2\":5},{\"offset\":7,\"offset2\":8}]~{.[]\|{offset}}
+echo {\"key1\":1}~{.[]\|{key1}}
+echo {\"a{b\":7}~{.a{b}
+EOF
+EXPECT=<<EOF
+[{"s":"alpha","n":1,"b":true,"nil":null,"obj":{"x":1},"arr":[1,2]},{"s":"beta","n":2,"b":false,"obj":{"y":"z"}},{"s":"gamma","arr":[]}]
+[{"k":1},{},{"k":3}]
+[]
+[{"key1":1}]
+[{"a b":1},{}]
+[{"arr":[null,1]}]
+[{},{}]
+[{},{"offset":7}]
+[]
+7
+EOF
+RUN
+
 NAME=/zj with backslash
 FILE=bins/pe/ConsoleApplication1.exe
 CMDS=/zj ConsoleApplication1.pdb~{}

test/unit/test_cons_grep.c

@@ -92,6 +92,115 @@ bool test_grep_line(void) {
 	mu_end;
 }
 
+bool test_grep_json_path(void) {
+	static const struct {
+		const char *json;
+		const char *grep;
+		const char *expected;
+		const char *message;
+	} cases[] = {
+		{
+			"[{\"offset\":1,\"bytes\":\"aa\",\"opcode\":\"nop\",\"family\":\"cpu\",\"type\":\"nop\",\"jump\":2},{\"offset\":2,\"bytes\":\"bb\",\"opcode\":\"ret\",\"family\":\"cpu\",\"type\":\"ret\"}]",
+			"{.[] | {offset, bytes, opcode, family, type, jump}}",
+			"[{\"offset\":1,\"bytes\":\"aa\",\"opcode\":\"nop\",\"family\":\"cpu\",\"type\":\"nop\",\"jump\":2},{\"offset\":2,\"bytes\":\"bb\",\"opcode\":\"ret\",\"family\":\"cpu\",\"type\":\"ret\"}]\n",
+			"json path array projection",
+		},
+		{
+			"[{\"s\":\"alpha\",\"n\":1,\"b\":true,\"nil\":null,\"obj\":{\"x\":1},\"arr\":[1,2]},{\"s\":\"beta\",\"n\":2,\"b\":false,\"obj\":{\"y\":\"z\"}},{\"s\":\"gamma\",\"arr\":[]}]",
+			"{.[]\\|{s,n,b,nil,obj,arr,missing}}",
+			"[{\"s\":\"alpha\",\"n\":1,\"b\":true,\"nil\":null,\"obj\":{\"x\":1},\"arr\":[1,2]},{\"s\":\"beta\",\"n\":2,\"b\":false,\"obj\":{\"y\":\"z\"}},{\"s\":\"gamma\",\"arr\":[]}]\n",
+			"json path array projection keeps existing typed values",
+		},
+		{
+			"[{\"key1\":1,\"key2\":\"aa\",\"key3\":true}]",
+			"{.[]\\|{key1, key2, key3}}",
+			"[{\"key1\":1,\"key2\":\"aa\",\"key3\":true}]\n",
+			"json path array projection with escaped pipe",
+		},
+		{
+			"[]",
+			"{ .[] \\| { key1, key2 } }",
+			"[]\n",
+			"json path array projection handles empty arrays",
+		},
+		{
+			"[{\"k\":1}]",
+			"{ . \t[ \t] \t| \t{ k } }",
+			"[{\"k\":1}]\n",
+			"json path array projection accepts whitespace between tokens",
+		},
+		{
+			"[{\"k\":1}]",
+			"{ . [ ] \t\\| \t{ k } }",
+			"[{\"k\":1}]\n",
+			"json path array projection accepts whitespace around escaped pipe",
+		},
+		{
+			"[{\"a b\":1},{\"a\":2}]",
+			"{.[]\\|{a b}}",
+			"[{\"a b\":1},{}]\n",
+			"json path array projection allows spaces in object keys",
+		},
+		{
+			"[{\"k\":1},7,\"x\",null,true,[],{\"z\":2},{\"k\":3}]",
+			"{.[] | {k}}",
+			"[{\"k\":1},{},{\"k\":3}]\n",
+			"json path array projection skips non-object array items and keeps empty objects",
+		},
+		{
+			"[{\"a\":1,\"nested\":{\"x\":1},\"arr\":[1,2],\"s\":\"hi\"},{\"b\":2},{\"z\":3}]",
+			"{.[] | {a, b, nested, arr}}",
+			"[{\"a\":1,\"nested\":{\"x\":1},\"arr\":[1,2]},{\"b\":2},{}]\n",
+			"json path array projection keeps partial objects and nested values",
+		},
+		{
+			"[{\"arr\":[null,1]}]",
+			"{.[]\\|{arr}}",
+			"[{\"arr\":[null,1]}]\n",
+			"json path array projection preserves null elements in nested arrays",
+		},
+		{
+			"[{\"z\":1},{\"z\":2}]",
+			"{.[]\\|{key1,key2}}",
+			"[{},{}]\n",
+			"json path array projection emits empty objects for missing keys",
+		},
+		{
+			"[{\"offset2\":5},{\"offset\":7,\"offset2\":8}]",
+			"{.[] | {offset}}",
+			"[{},{\"offset\":7}]\n",
+			"json path array projection matches object keys exactly",
+		},
+		{
+			"{\"key1\":1}",
+			"{.[]\\|{key1}}",
+			"[]\n",
+			"json path array projection returns empty array for non-array input",
+		},
+		{
+			"[{\"opcode\":\"nop\"}]",
+			"{[0].opcode}",
+			"nop\n",
+			"json path still extracts one node",
+		},
+		{
+			"{\"a{b\":7}",
+			"{.a{b}",
+			"7\n",
+			"json path preserves literal open brace in key",
+		},
+	};
+
+	for (size_t i = 0; i < RZ_ARRAY_SIZE(cases); i++) {
+		rz_cons_new();
+		rz_cons_print(cases[i].json);
+		rz_cons_grep(cases[i].grep);
+		mu_assert_streq(rz_cons_get_buffer(), cases[i].expected, cases[i].message);
+		rz_cons_free();
+	}
+	mu_end;
+}
+
 bool all_tests() {
 	mu_run_test(test_grep_parse_simple);
 	mu_run_test(test_grep_parse_multiple);
@@ -99,6 +208,7 @@ bool all_tests() {
 	mu_run_test(test_grep_parse_case_insensitive);
 	mu_run_test(test_grep_parse_line);
 	mu_run_test(test_grep_line);
+	mu_run_test(test_grep_json_path);
 	return tests_passed != tests_run;
 }