Use Perl non-greedy regex in tokenizer

We can now exclude C++ style comments and #includes,
in addition to C-style comments and strings.
This commit is contained in:
Mikaël Bouillot 2017-05-01 22:27:24 +02:00
parent 7db38c24fd
commit 9c2cf2cf86
2 changed files with 3 additions and 16 deletions

9
lib.py
View file

@ -39,15 +39,8 @@ def scriptLines (*args):
def unescape (bstr):
subs = (
('<','\033[32m/*'),
('>','*/\033[0m'),
('{','\033[33m"'),
('}','"\033[0m'),
('\1','\n'),
('\3','<'),
('\4','>'),
('\5','{'),
('\6','}'))
)
for a,b in subs:
a = a.encode()
b = b.encode()

View file

@ -148,14 +148,8 @@ case $cmd in
fi
git cat-file blob $ref 2>/dev/null |
tr '\n%<>{}' '\1\2\3\4\5\6' |
sed 's/\/\*/</g' |
sed 's/\*\//>/g' |
sed 's/\\"/%/g' |
sed 's/"\([^"]*\)"/{\1}/g' |
sed 's/%/\\"/g' |
tr '\2' '%' |
sed -r 's/(\W*)(<[^>]*>)?(\{[^}]*\})?(\w*)/\1\2\3\n\4\n/g' |
tr '\n' '\1' |
perl -pe 's%((/\*.*?\*/|//.*?\001|".*?"|#include *<.*?>|\W)+)(\w+)?%\1\n\3\n%g' |
head -n -1
;;