portugalfuturista/rizin
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update tree-sitter to 0.20 for rzshell and C parsers

Browse source
This commit is contained in:
Riccardo Schirone 2022-03-18 15:10:40 +01:00 committed by Riccardo Schirone
parent 1407c65c98
commit 88b0872dca
13 changed files with 47 additions and 409 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.reuse/dep5
View file

@ -240,11 +240,6 @@ Files: subprojects/mpc/*
Copyright: 2013-present Daniel Holden <contact@theorangeduck.com>
License: BSD-2-Clause
Files: subprojects/packagefiles/tree-sitter-0.19.5/lib/src/*
Copyright: 2018-2021 Max Brunsfeld
Copyright: 2021 Florian Märkl <info@florianmaerkl.de>
License: MIT
Files: librz/include/**/*.h librz/include/*.h librz/include/*.h.in
Copyright: 2021 RizinOrg <info@rizin.re>
License: LGPL-3.0-only

21
subprojects/packagefiles/tree-sitter-0.19.5/lib/src/host.h
View file

@ -1,21 +0,0 @@
// Determine endian and pointer size based on known defines.
// TS_BIG_ENDIAN and TS_PTR_SIZE can be set as -D compiler arguments
// to override this.
#if !defined(TS_BIG_ENDIAN)
#if (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) \
|| (defined( __APPLE_CC__) && (defined(__ppc__) || defined(__ppc64__)))
#define TS_BIG_ENDIAN 1
#else
#define TS_BIG_ENDIAN 0
#endif
#endif
#if !defined(TS_PTR_SIZE)
#if UINTPTR_MAX == 0xFFFFFFFF
#define TS_PTR_SIZE 32
#else
#define TS_PTR_SIZE 64
#endif
#endif

356
subprojects/packagefiles/tree-sitter-0.19.5/lib/src/subtree.h
View file

@ -1,356 +0,0 @@
#ifndef TREE_SITTER_SUBTREE_H_
#define TREE_SITTER_SUBTREE_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <limits.h>
#include <stdbool.h>
#include <stdio.h>
#include "./length.h"
#include "./array.h"
#include "./error_costs.h"
#include "./host.h"
#include "tree_sitter/api.h"
#include "tree_sitter/parser.h"
#define TS_TREE_STATE_NONE USHRT_MAX
#define NULL_SUBTREE ((Subtree) {.ptr = NULL})
// The serialized state of an external scanner.
//
// Every time an external token subtree is created after a call to an
// external scanner, the scanner's `serialize` function is called to
// retrieve a serialized copy of its state. The bytes are then copied
// onto the subtree itself so that the scanner's state can later be
// restored using its `deserialize` function.
//
// Small byte arrays are stored inline, and long ones are allocated
// separately on the heap.
typedef struct {
union {
char *long_data;
char short_data[24];
};
uint32_t length;
} ExternalScannerState;
// A compact representation of a subtree.
//
// This representation is used for small leaf nodes that are not
// errors, and were not created by an external scanner.
// The idea behind the layout of this struct is that the `is_inline`
// bit will fall exactly into the same location as the least significant
// bit of the pointer in `Subtree` or `M̀utableSubtree`, respectively.
// Because of alignment, for any valid pointer this will be 0, giving
// us the opportunity to make use of this bit to signify whether to use
// the pointer or the inline struct.
typedef struct {
#define SUBTREE_3_BYTES \
uint8_t symbol; \
uint8_t padding_bytes; \
uint8_t size_bytes; \
#define SUBTREE_6_BITS \
bool visible : 1; \
bool named : 1; \
bool extra : 1; \
bool has_changes : 1; \
bool is_missing : 1; \
bool is_keyword : 1;
#define SUBTREE_4_BYTES \
uint8_t padding_columns; \
uint8_t padding_rows : 4; \
uint8_t lookahead_bytes : 4; \
uint16_t parse_state;
#if TS_BIG_ENDIAN
#if TS_PTR_SIZE == 32
SUBTREE_3_BYTES
SUBTREE_6_BITS
bool unused : 1;
bool is_inline : 1;
SUBTREE_4_BYTES
#else
SUBTREE_4_BYTES
SUBTREE_3_BYTES
SUBTREE_6_BITS
bool unused : 1;
bool is_inline : 1;
#endif
#else
bool is_inline : 1;
SUBTREE_6_BITS
SUBTREE_3_BYTES
SUBTREE_4_BYTES
#endif
#undef SUBTREE_3_BYTES
#undef SUBTREE_6_BITS
#undef SUBTREE_4_BYTES
} SubtreeInlineData;
// A heap-allocated representation of a subtree.
//
// This representation is used for parent nodes, external tokens,
// errors, and other leaf nodes whose data is too large to fit into
// the inlinen representation.
typedef struct {
volatile uint32_t ref_count;
Length padding;
Length size;
uint32_t lookahead_bytes;
uint32_t error_cost;
uint32_t child_count;
TSSymbol symbol;
TSStateId parse_state;
bool visible : 1;
bool named : 1;
bool extra : 1;
bool fragile_left : 1;
bool fragile_right : 1;
bool has_changes : 1;
bool has_external_tokens : 1;
bool depends_on_column: 1;
bool is_missing : 1;
bool is_keyword : 1;
union {
// Non-terminal subtrees (`child_count > 0`)
struct {
uint32_t visible_child_count;
uint32_t named_child_count;
uint32_t node_count;
uint32_t repeat_depth;
int32_t dynamic_precedence;
uint16_t production_id;
struct {
TSSymbol symbol;
TSStateId parse_state;
} first_leaf;
};
// External terminal subtrees (`child_count == 0 && has_external_tokens`)
ExternalScannerState external_scanner_state;
// Error terminal subtrees (`child_count == 0 && symbol == ts_builtin_sym_error`)
int32_t lookahead_char;
};
} SubtreeHeapData;
// The fundamental building block of a syntax tree.
typedef union {
SubtreeInlineData data;
const SubtreeHeapData *ptr;
} Subtree;
// Like Subtree, but mutable.
typedef union {
SubtreeInlineData data;
SubtreeHeapData *ptr;
} MutableSubtree;
typedef Array(Subtree) SubtreeArray;
typedef Array(MutableSubtree) MutableSubtreeArray;
typedef struct {
MutableSubtreeArray free_trees;
MutableSubtreeArray tree_stack;
} SubtreePool;
void ts_external_scanner_state_init(ExternalScannerState *, const char *, unsigned);
const char *ts_external_scanner_state_data(const ExternalScannerState *);
void ts_subtree_array_copy(SubtreeArray, SubtreeArray *);
void ts_subtree_array_clear(SubtreePool *, SubtreeArray *);
void ts_subtree_array_delete(SubtreePool *, SubtreeArray *);
void ts_subtree_array_remove_trailing_extras(SubtreeArray *, SubtreeArray *);
void ts_subtree_array_reverse(SubtreeArray *);
SubtreePool ts_subtree_pool_new(uint32_t capacity);
void ts_subtree_pool_delete(SubtreePool *);
Subtree ts_subtree_new_leaf(
SubtreePool *, TSSymbol, Length, Length, uint32_t,
TSStateId, bool, bool, bool, const TSLanguage *
);
Subtree ts_subtree_new_error(
SubtreePool *, int32_t, Length, Length, uint32_t, TSStateId, const TSLanguage *
);
MutableSubtree ts_subtree_new_node(TSSymbol, SubtreeArray *, unsigned, const TSLanguage *);
Subtree ts_subtree_new_error_node(SubtreeArray *, bool, const TSLanguage *);
Subtree ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, Length, const TSLanguage *);
MutableSubtree ts_subtree_make_mut(SubtreePool *, Subtree);
void ts_subtree_retain(Subtree);
void ts_subtree_release(SubtreePool *, Subtree);
bool ts_subtree_eq(Subtree, Subtree);
int ts_subtree_compare(Subtree, Subtree);
void ts_subtree_set_symbol(MutableSubtree *, TSSymbol, const TSLanguage *);
void ts_subtree_summarize(MutableSubtree, const Subtree *, uint32_t, const TSLanguage *);
void ts_subtree_summarize_children(MutableSubtree, const TSLanguage *);
void ts_subtree_balance(Subtree, SubtreePool *, const TSLanguage *);
Subtree ts_subtree_edit(Subtree, const TSInputEdit *edit, SubtreePool *);
char *ts_subtree_string(Subtree, const TSLanguage *, bool include_all);
void ts_subtree_print_dot_graph(Subtree, const TSLanguage *, FILE *);
Subtree ts_subtree_last_external_token(Subtree);
bool ts_subtree_external_scanner_state_eq(Subtree, Subtree);
#define SUBTREE_GET(self, name) (self.data.is_inline ? self.data.name : self.ptr->name)
static inline TSSymbol ts_subtree_symbol(Subtree self) { return SUBTREE_GET(self, symbol); }
static inline bool ts_subtree_visible(Subtree self) { return SUBTREE_GET(self, visible); }
static inline bool ts_subtree_named(Subtree self) { return SUBTREE_GET(self, named); }
static inline bool ts_subtree_extra(Subtree self) { return SUBTREE_GET(self, extra); }
static inline bool ts_subtree_has_changes(Subtree self) { return SUBTREE_GET(self, has_changes); }
static inline bool ts_subtree_missing(Subtree self) { return SUBTREE_GET(self, is_missing); }
static inline bool ts_subtree_is_keyword(Subtree self) { return SUBTREE_GET(self, is_keyword); }
static inline TSStateId ts_subtree_parse_state(Subtree self) { return SUBTREE_GET(self, parse_state); }
static inline uint32_t ts_subtree_lookahead_bytes(Subtree self) { return SUBTREE_GET(self, lookahead_bytes); }
#undef SUBTREE_GET
// Get the size needed to store a heap-allocated subtree with the given
// number of children.
static inline size_t ts_subtree_alloc_size(uint32_t child_count) {
return child_count * sizeof(Subtree) + sizeof(SubtreeHeapData);
}
// Get a subtree's children, which are allocated immediately before the
// tree's own heap data.
#define ts_subtree_children(self) \
((self).data.is_inline ? NULL : (Subtree *)((self).ptr) - (self).ptr->child_count)
static inline void ts_subtree_set_extra(MutableSubtree *self) {
if (self->data.is_inline) {
self->data.extra = true;
} else {
self->ptr->extra = true;
}
}
static inline TSSymbol ts_subtree_leaf_symbol(Subtree self) {
if (self.data.is_inline) return self.data.symbol;
if (self.ptr->child_count == 0) return self.ptr->symbol;
return self.ptr->first_leaf.symbol;
}
static inline TSStateId ts_subtree_leaf_parse_state(Subtree self) {
if (self.data.is_inline) return self.data.parse_state;
if (self.ptr->child_count == 0) return self.ptr->parse_state;
return self.ptr->first_leaf.parse_state;
}
static inline Length ts_subtree_padding(Subtree self) {
if (self.data.is_inline) {
Length result = {self.data.padding_bytes, {self.data.padding_rows, self.data.padding_columns}};
return result;
} else {
return self.ptr->padding;
}
}
static inline Length ts_subtree_size(Subtree self) {
if (self.data.is_inline) {
Length result = {self.data.size_bytes, {0, self.data.size_bytes}};
return result;
} else {
return self.ptr->size;
}
}
static inline Length ts_subtree_total_size(Subtree self) {
return length_add(ts_subtree_padding(self), ts_subtree_size(self));
}
static inline uint32_t ts_subtree_total_bytes(Subtree self) {
return ts_subtree_total_size(self).bytes;
}
static inline uint32_t ts_subtree_child_count(Subtree self) {
return self.data.is_inline ? 0 : self.ptr->child_count;
}
static inline uint32_t ts_subtree_repeat_depth(Subtree self) {
return self.data.is_inline ? 0 : self.ptr->repeat_depth;
}
static inline uint32_t ts_subtree_node_count(Subtree self) {
return (self.data.is_inline || self.ptr->child_count == 0) ? 1 : self.ptr->node_count;
}
static inline uint32_t ts_subtree_visible_child_count(Subtree self) {
if (ts_subtree_child_count(self) > 0) {
return self.ptr->visible_child_count;
} else {
return 0;
}
}
static inline uint32_t ts_subtree_error_cost(Subtree self) {
if (ts_subtree_missing(self)) {
return ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY;
} else {
return self.data.is_inline ? 0 : self.ptr->error_cost;
}
}
static inline int32_t ts_subtree_dynamic_precedence(Subtree self) {
return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->dynamic_precedence;
}
static inline uint16_t ts_subtree_production_id(Subtree self) {
if (ts_subtree_child_count(self) > 0) {
return self.ptr->production_id;
} else {
return 0;
}
}
static inline bool ts_subtree_fragile_left(Subtree self) {
return self.data.is_inline ? false : self.ptr->fragile_left;
}
static inline bool ts_subtree_fragile_right(Subtree self) {
return self.data.is_inline ? false : self.ptr->fragile_right;
}
static inline bool ts_subtree_has_external_tokens(Subtree self) {
return self.data.is_inline ? false : self.ptr->has_external_tokens;
}
static inline bool ts_subtree_depends_on_column(Subtree self) {
return self.data.is_inline ? false : self.ptr->depends_on_column;
}
static inline bool ts_subtree_is_fragile(Subtree self) {
return self.data.is_inline ? false : (self.ptr->fragile_left || self.ptr->fragile_right);
}
static inline bool ts_subtree_is_error(Subtree self) {
return ts_subtree_symbol(self) == ts_builtin_sym_error;
}
static inline bool ts_subtree_is_eof(Subtree self) {
return ts_subtree_symbol(self) == ts_builtin_sym_end;
}
static inline Subtree ts_subtree_from_mut(MutableSubtree self) {
Subtree result;
result.data = self.data;
return result;
}
static inline MutableSubtree ts_subtree_to_mut_unsafe(Subtree self) {
MutableSubtree result;
result.data = self.data;
return result;
}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_SUBTREE_H_

0
subprojects/packagefiles/tree-sitter-0.19.5/meson.build → subprojects/packagefiles/tree-sitter-0.20.6/meson.build
View file

2
subprojects/packagefiles/tree-sitter-c/meson.build
View file

@ -1,4 +1,4 @@
project('tree-sitter-c', 'c', version: 'f05e279aedde06a25801c3f2b2cc8ac17fac52ae', default_options: ['werror=false'])
project('tree-sitter-c', 'c', default_options: ['werror=false'])
ts_c_files = [
'src/parser.c'

2
subprojects/rizin-shell-parser/Cargo.toml
View file

@ -20,7 +20,7 @@ include = [
path = "bindings/rust/lib.rs"
[dependencies]
tree-sitter = "0.17"
tree-sitter = "0.20"
[build-dependencies]
cc = "1.0"

4
subprojects/rizin-shell-parser/grammar.js
View file

@ -58,7 +58,7 @@ module.exports = grammar({
$._spec_sep,
],
inline: ($) => [$.stmt_delimiter, $.stmt_delimiter_singleline, $._comment],
inline: ($) => [$.stmt_delimiter, $.stmt_delimiter_singleline],
rules: {
statements: ($) =>
@ -438,7 +438,7 @@ module.exports = grammar({
concatenation: ($) => prec(-1, seq($._arg, repeat1(prec(-1, seq($._concat, $._arg))))),
_dec_number: ($) => choice(/[1-9][0-9]*/, /[0-9][0-9]+/),
_comment: ($) => token(choice(/#[^\r\n]*/)),
_comment: ($) => /#[^\r\n]*/,
stmt_delimiter: ($) => choice("\n", "\r", $.stmt_delimiter_singleline),
stmt_delimiter_singleline: ($) => choice(";"),

36
subprojects/rizin-shell-parser/package-lock.json generated
View file

@ -1,8 +1,36 @@
{
"name": "tree-sitter-rzcmd",
"version": "1.0.0",
"lockfileVersion": 1,
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "tree-sitter-rzcmd",
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"nan": "^2.14.0"
},
"devDependencies": {
"tree-sitter-cli": "^0.20.6"
}
},
"node_modules/nan": {
"version": "2.14.0",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz",
"integrity": "sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg=="
},
"node_modules/tree-sitter-cli": {
"version": "0.20.6",
"resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.20.6.tgz",
"integrity": "sha512-tjbAeuGSMhco/EnsThjWkQbDIYMDmdkWsTPsa/NJAW7bjaki9P7oM9TkLxfdlnm4LXd1wR5wVSM2/RTLtZbm6A==",
"dev": true,
"hasInstallScript": true,
"bin": {
"tree-sitter": "cli.js"
}
}
},
"dependencies": {
"nan": {
"version": "2.14.0",
@ -10,9 +38,9 @@
"integrity": "sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg=="
},
"tree-sitter-cli": {
"version": "0.19.4",
"resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.19.4.tgz",
"integrity": "sha512-p2kxjuoQeauXBu5eE+j7c5BMCRXmc17EiAswnnWn3ieUlHXBkA0Z7vRnaCSElDR34MhZnSgqgzuuzQk0cDqCjw==",
"version": "0.20.6",
"resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.20.6.tgz",
"integrity": "sha512-tjbAeuGSMhco/EnsThjWkQbDIYMDmdkWsTPsa/NJAW7bjaki9P7oM9TkLxfdlnm4LXd1wR5wVSM2/RTLtZbm6A==",
"dev": true
}
}

2
subprojects/rizin-shell-parser/package.json
View file

@ -12,6 +12,6 @@
"nan": "^2.14.0"
},
"devDependencies": {
"tree-sitter-cli": "^0.19.4"
"tree-sitter-cli": "^0.20.6"
}
}

15
subprojects/rizin-shell-parser/src/grammar.json generated
View file

@ -3556,16 +3556,8 @@
]
},
"_comment": {
"type": "TOKEN",
"content": {
"type": "CHOICE",
"members": [
{
"type": "PATTERN",
"value": "#[^\\r\\n]*"
}
]
}
"type": "PATTERN",
"value": "#[^\\r\\n]*"
},
"stmt_delimiter": {
"type": "CHOICE",
@ -3692,8 +3684,7 @@
],
"inline": [
"stmt_delimiter",
"stmt_delimiter_singleline",
"_comment"
"stmt_delimiter_singleline"
],
"supertypes": []
}

1
subprojects/rizin-shell-parser/src/tree_sitter/parser.h
View file

@ -123,6 +123,7 @@ struct TSLanguage {
unsigned (*serialize)(void *, char *);
void (*deserialize)(void *, const char *, unsigned);
} external_scanner;
const TSStateId *primary_state_ids;
};
/*

2
subprojects/tree-sitter-c.wrap
View file

@ -1,5 +1,5 @@
[wrap-git]
url = https://github.com/rizinorg/tree-sitter-c
revision = 3dd21d9440ca39d8c201335b2f3a393db41aab36
revision = 51ecadb131b5712457b543db8d15ab141ae9f117
patch_directory = tree-sitter-c
directory = tree-sitter-c

10
subprojects/tree-sitter.wrap
View file

@ -1,6 +1,6 @@
[wrap-file]
source_url = https://github.com/tree-sitter/tree-sitter/archive/v0.19.5.tar.gz
source_filename = v0.19.5.tar.gz
source_hash = 953edda5a21423cbfd916dbb1b1552a12f7ab649728846b5a1fb49483672e82b
patch_directory = tree-sitter-0.19.5
directory = tree-sitter-0.19.5
source_url = https://github.com/tree-sitter/tree-sitter/archive/v0.20.6.tar.gz
source_filename = v0.20.6.tar.gz
source_hash = 4d37eaef8a402a385998ff9aca3e1043b4a3bba899bceeff27a7178e1165b9de
patch_directory = tree-sitter-0.20.6
directory = tree-sitter-0.20.6