Merge pull request #130 from MaximeChretien/database_update
Database update
This commit is contained in:
Michael Opdenacker
GitHub
commit
4781abde05
26 changed files with 563 additions and 200 deletions
|
|
@ -10,8 +10,10 @@ python:
|
|||
- "3.6"
|
||||
|
||||
before_install:
|
||||
- sudo apt-get -y install libdb-dev exuberant-ctags python3-pytest
|
||||
- sudo apt-get -y install libdb-dev python3-pytest
|
||||
- pip install jinja2 pygments bsddb3 falcon
|
||||
- wget https://bootlin.com/pub/elixir/universal-ctags_0+git20221222-0ubuntu1_amd64.deb
|
||||
- sudo dpkg -i universal-ctags_0+git20221222-0ubuntu1_amd64.deb
|
||||
|
||||
script:
|
||||
- prove
|
||||
- prove
|
||||
|
|
|
|||
24
README.adoc
24
README.adoc
|
|
@ -79,7 +79,7 @@ For Debian
|
|||
____
|
||||
|
||||
----
|
||||
sudo apt install python3 python3-jinja2 python3-pygments python3-bsddb3 python3-falcon python3-pytest exuberant-ctags perl git apache2 libapache2-mod-wsgi-py3
|
||||
sudo apt install python3 python3-jinja2 python3-pygments python3-bsddb3 python3-falcon python3-pytest universal-ctags perl git apache2 libapache2-mod-wsgi-py3
|
||||
----
|
||||
|
||||
To enable the REST API, follow the installation instructions on https://github.com/GrahamDumpleton/mod_wsgi[`mod_wsgi`]
|
||||
|
|
@ -106,6 +106,22 @@ and install it as follows:
|
|||
sudo pip3 install Pygments-2.6.1.elixir-py3-none-any.whl
|
||||
----
|
||||
|
||||
=== Kconfig identifiers support
|
||||
|
||||
The service on https://elixir.bootlin.com relies on a modified version of https://ctags.io/[Universal-ctags],
|
||||
to enable indexation of Kconfig identifiers.
|
||||
|
||||
The changes have been sent upstream and should appear in future distributions.
|
||||
|
||||
In the meantime, you can either rebuild this package from its source on
|
||||
https://github.com/MaximeChretien/ctags/tree/kconfig-parser[GitHub], or download this
|
||||
https://bootlin.com/pub/elixir/universal-ctags_0+git20221222-0ubuntu1_amd64.deb[binary module]
|
||||
and install it as follows:
|
||||
|
||||
----
|
||||
sudo dpkg -i universal-ctags_0+git20221222-0ubuntu1_amd64.deb
|
||||
----
|
||||
|
||||
== Download Elixir Project
|
||||
|
||||
----
|
||||
|
|
@ -176,7 +192,7 @@ ____
|
|||
|
||||
Verify that the queries work:
|
||||
|
||||
$ ./query.py v4.10 ident raw_spin_unlock_irq
|
||||
$ ./query.py v4.10 ident raw_spin_unlock_irq C
|
||||
$ ./query.py v4.10 file /kernel/sched/clock.c
|
||||
|
||||
NOTE: `v4.10` can be replaced with any other tag.
|
||||
|
|
@ -272,10 +288,10 @@ After configuring httpd, you can test the API usage:
|
|||
|
||||
== ident query
|
||||
|
||||
Send a get request to `/api/ident/<Project>/<Ident>?version=<version>`.
|
||||
Send a get request to `/api/ident/<Project>/<Ident>?version=<version>&family=<family>`.
|
||||
For example:
|
||||
|
||||
curl http://127.0.0.1/api/ident/barebox/cdev?version=latest
|
||||
curl http://127.0.0.1/api/ident/barebox/cdev?version=latest&family=C
|
||||
|
||||
The response body is of the following structure:
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,12 @@ class IdentGetter:
|
|||
if version == 'latest':
|
||||
version = query('latest')
|
||||
|
||||
symbol_definitions, symbol_references, symbol_doccomments_UNUSED = query('ident', version, ident)
|
||||
if 'family' in req.params:
|
||||
family = req.params['family']
|
||||
else:
|
||||
family = 'C'
|
||||
|
||||
symbol_definitions, symbol_references, symbol_doccomments_UNUSED = query('ident', version, ident, family)
|
||||
resp.body = json.dumps(
|
||||
{
|
||||
'definitions': [sym.__dict__ for sym in symbol_definitions],
|
||||
|
|
|
|||
47
data.py
47
data.py
|
|
@ -29,6 +29,7 @@ import errno
|
|||
##################################################################################
|
||||
|
||||
defTypeR = {
|
||||
'c': 'config',
|
||||
'd': 'define',
|
||||
'e': 'enum',
|
||||
'E': 'enumerator',
|
||||
|
|
@ -50,31 +51,43 @@ maxId = 999999999
|
|||
|
||||
class DefList:
|
||||
'''Stores associations between a blob ID, a type (e.g., "function"),
|
||||
and a line number.'''
|
||||
def __init__(self, data=b''):
|
||||
self.data = data
|
||||
a line number and a file family.
|
||||
Also stores in which families the ident exists for faster tests.'''
|
||||
def __init__(self, data=b'#'):
|
||||
self.data, self.families = data.split(b'#')
|
||||
|
||||
def iter(self, dummy=False):
|
||||
for p in self.data.split(b','):
|
||||
p = re.search(b'(\d*)(\w)(\d*)', p)
|
||||
id, type, line = p.groups()
|
||||
p = re.search(b'(\d*)(\w)(\d*)(\w)', p)
|
||||
id, type, line, family = p.groups()
|
||||
id = int(id)
|
||||
type = defTypeR [type.decode()]
|
||||
line = int(line)
|
||||
yield(id, type, line)
|
||||
family = family.decode()
|
||||
yield(id, type, line, family)
|
||||
if dummy:
|
||||
yield(maxId, None, None)
|
||||
yield(maxId, None, None, None)
|
||||
|
||||
def append(self, id, type, line):
|
||||
def append(self, id, type, line, family):
|
||||
if type not in defTypeD:
|
||||
return
|
||||
p = str(id) + defTypeD[type] + str(line)
|
||||
p = str(id) + defTypeD[type] + str(line) + family
|
||||
if self.data != b'':
|
||||
p = ',' + p
|
||||
self.data += p.encode()
|
||||
|
||||
def pack(self):
|
||||
return self.data
|
||||
return self.data + b'#' + self.families
|
||||
|
||||
def add_family(self, family):
|
||||
family = family.encode()
|
||||
if not family in self.families.split(b','):
|
||||
if self.families != b'':
|
||||
family = b',' + family
|
||||
self.families += family
|
||||
|
||||
def get_families(self):
|
||||
return self.families.decode().split(',')
|
||||
|
||||
class PathList:
|
||||
'''Stores associations between a blob ID and a file path.
|
||||
|
|
@ -100,7 +113,8 @@ class PathList:
|
|||
return self.data
|
||||
|
||||
class RefList:
|
||||
'''Stores a mapping from blob ID to list of lines.'''
|
||||
'''Stores a mapping from blob ID to list of lines
|
||||
and the corresponding family.'''
|
||||
def __init__(self, data=b''):
|
||||
self.data = data
|
||||
|
||||
|
|
@ -110,16 +124,17 @@ class RefList:
|
|||
while s.tell() < size:
|
||||
line = s.readline()
|
||||
line = line [:-1]
|
||||
b,c = line.split(b':')
|
||||
b,c,d = line.split(b':')
|
||||
b = int(b.decode())
|
||||
c = c.decode()
|
||||
yield(b, c)
|
||||
d = d.decode()
|
||||
yield(b, c, d)
|
||||
s.close()
|
||||
if dummy:
|
||||
yield(maxId, None)
|
||||
yield(maxId, None, None)
|
||||
|
||||
def append(self, id, lines):
|
||||
p = str(id) + ':' + lines + '\n'
|
||||
def append(self, id, lines, family):
|
||||
p = str(id) + ':' + lines + ':' + family + '\n'
|
||||
self.data += p.encode()
|
||||
|
||||
def pack(self):
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# Elixir Python definitions for Barebox
|
||||
|
||||
exec(open('dtsi.py').read())
|
||||
exec(open('kconfig.py').read())
|
||||
exec(open('commonkconfig.py').read())
|
||||
exec(open('cpppathinc.py').read())
|
||||
exec(open('makefileo.py').read())
|
||||
exec(open('makefiledtb.py').read())
|
||||
|
|
|
|||
5
http/filters/commonkconfig.py
Normal file
5
http/filters/commonkconfig.py
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
# Common filters for Kconfig
|
||||
|
||||
exec(open('kconfig.py').read())
|
||||
exec(open('kconfigidents.py').read())
|
||||
exec(open('makefilekconfig.py').read())
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
# Elixir Python definitions for Coreboot
|
||||
|
||||
exec(open('dtsi.py').read())
|
||||
exec(open('kconfig.py').read())
|
||||
exec(open('commonkconfig.py').read())
|
||||
|
|
|
|||
|
|
@ -8,11 +8,11 @@ def keep_idents(m):
|
|||
|
||||
def replace_idents(m):
|
||||
i = idents[decode_number(m.group(1)) - 1]
|
||||
return '<a href="'+version+'/ident/'+i+'">'+i+'</a>'
|
||||
return '<a href="'+version+'/'+family+'/ident/'+i+'">'+i+'</a>'
|
||||
|
||||
ident_filters = {
|
||||
'case': 'any',
|
||||
'prerex': '\033\[31m(.*?)\033\[0m',
|
||||
'prerex': '\033\[31m(?!CONFIG_)(.*?)\033\[0m',
|
||||
'prefunc': keep_idents,
|
||||
'postrex': '__KEEPIDENTS__([A-J]+)',
|
||||
'postfunc': replace_idents
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ def replace_kconfig(m):
|
|||
kconfig_filters = {
|
||||
'case': 'filename',
|
||||
'match': {'Kconfig'},
|
||||
'prerex': '^(\s*)(source)(\s*)\"(.*?)\"',
|
||||
'prerex': '^(\s*)(source)(\s*)\"([\w/_\.-]+)\"',
|
||||
'prefunc': keep_kconfig,
|
||||
'postrex': '__KEEPKCONFIG__([A-J]+)',
|
||||
'postfunc': replace_kconfig
|
||||
|
|
|
|||
27
http/filters/kconfigidents.py
Normal file
27
http/filters/kconfigidents.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
# Filter for kconfig identifier links
|
||||
|
||||
kconfigidents = []
|
||||
|
||||
def keep_kconfigidents(m):
|
||||
kconfigidents.append(m.group(1))
|
||||
return '__KEEPKCONFIGIDENTS__' + encode_number(len(kconfigidents))
|
||||
|
||||
def replace_kconfigidents(m):
|
||||
i = kconfigidents[decode_number(m.group(1)) - 1]
|
||||
|
||||
n = i
|
||||
#Remove the CONFIG_ when we are in a Kconfig file
|
||||
if family == 'K':
|
||||
n = n[7:]
|
||||
|
||||
return '<a href="'+version+'/K/ident/'+i+'">'+n+'</a>'
|
||||
|
||||
kconfigident_filters = {
|
||||
'case': 'any',
|
||||
'prerex': '\033\[31m(?=CONFIG_)(.*?)\033\[0m',
|
||||
'prefunc': keep_kconfigidents,
|
||||
'postrex': '__KEEPKCONFIGIDENTS__([A-J]+)',
|
||||
'postfunc': replace_kconfigidents
|
||||
}
|
||||
|
||||
filters.append(kconfigident_filters)
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
# Elixir Python definitions for Linux
|
||||
|
||||
exec(open('dtsi.py').read())
|
||||
exec(open('kconfig.py').read())
|
||||
exec(open('commonkconfig.py').read())
|
||||
exec(open('makefileo.py').read())
|
||||
exec(open('makefiledtb.py').read())
|
||||
exec(open('makefiledir.py').read())
|
||||
|
|
|
|||
22
http/filters/makefilekconfig.py
Normal file
22
http/filters/makefilekconfig.py
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
# Filters for Kconfig used in Makefiles
|
||||
|
||||
makefilekconfig = []
|
||||
|
||||
def keep_makefilekconfig(m):
|
||||
makefilekconfig.append(m.group(1))
|
||||
return '$(__KEEPMAKEFILEKCONFIG__' + encode_number(len(makefilekconfig)) + ')'
|
||||
|
||||
def replace_makefilekconfig(m):
|
||||
i = makefilekconfig[decode_number(m.group(1)) - 1]
|
||||
return '<a href="'+version+'/K/ident/'+i+'">'+i+'</a>'
|
||||
|
||||
makefilekconfig_filters = {
|
||||
'case': 'filename',
|
||||
'match': {'Makefile'},
|
||||
'prerex': '\$\((CONFIG_\w+)\)',
|
||||
'prefunc': keep_makefilekconfig,
|
||||
'postrex': '__KEEPMAKEFILEKCONFIG__([A-J]+)',
|
||||
'postfunc': replace_makefilekconfig
|
||||
}
|
||||
|
||||
filters.append(makefilekconfig_filters)
|
||||
|
|
@ -1,3 +1,3 @@
|
|||
# Elixir Python definitions for qemu
|
||||
|
||||
exec(open('kconfig.py').read())
|
||||
exec(open('commonkconfig.py').read())
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# Elixir Python definitions for U-Boot
|
||||
|
||||
exec(open('dtsi.py').read())
|
||||
exec(open('kconfig.py').read())
|
||||
exec(open('commonkconfig.py').read())
|
||||
exec(open('cpppathinc.py').read())
|
||||
exec(open('makefileo.py').read())
|
||||
exec(open('makefiledtb.py').read())
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# Elixir Python definitions for Zephyr
|
||||
|
||||
exec(open('dtsi.py').read())
|
||||
exec(open('kconfig.py').read())
|
||||
exec(open('commonkconfig.py').read())
|
||||
exec(open('cpppathinc.py').read())
|
||||
|
|
|
|||
|
|
@ -253,6 +253,10 @@ h2 {
|
|||
padding: 0.5em;
|
||||
padding-top: 0;
|
||||
}
|
||||
.search form {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
}
|
||||
.search button:focus,
|
||||
.search button:hover {
|
||||
color: #000;
|
||||
|
|
@ -262,10 +266,19 @@ h2 {
|
|||
color: #000;
|
||||
background: #ddd;
|
||||
padding-right: 3em;
|
||||
min-width: 0;
|
||||
flex: 4;
|
||||
}
|
||||
.search input:focus {
|
||||
background: #eee;
|
||||
}
|
||||
.search select {
|
||||
font-size: 0.9em;
|
||||
padding: 0.45em;
|
||||
margin: 0;
|
||||
min-width: 0;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.filter {
|
||||
padding: 0.5em;
|
||||
|
|
|
|||
24
http/web.py
24
http/web.py
|
|
@ -48,14 +48,18 @@ status = 200
|
|||
|
||||
url = os.environ.get('REQUEST_URI') or os.environ.get('SCRIPT_URL')
|
||||
# Split the URL into its components (project, version, cmd, arg)
|
||||
m = search('^/([^/]*)/([^/]*)/([^/]*)(.*)$', url)
|
||||
m = search('^/([^/]*)/([^/]*)(?:/([^/]))?/([^/]*)(.*)$', url)
|
||||
|
||||
if m:
|
||||
project = m.group(1)
|
||||
version = m.group(2)
|
||||
version_decoded = parse.unquote(version)
|
||||
cmd = m.group(3)
|
||||
arg = m.group(4)
|
||||
family = m.group(3)
|
||||
cmd = m.group(4)
|
||||
arg = m.group(5)
|
||||
|
||||
if family == None:
|
||||
family = 'C'
|
||||
|
||||
basedir = os.environ['LXR_PROJ_DIR']
|
||||
datadir = basedir + '/' + project + '/data'
|
||||
|
|
@ -80,15 +84,16 @@ if m:
|
|||
ident = arg[1:]
|
||||
form = cgi.FieldStorage()
|
||||
ident2 = form.getvalue('i')
|
||||
family2 = form.getvalue('f')
|
||||
if ident == '' and ident2:
|
||||
status = 302
|
||||
ident2 = parse.quote(ident2.strip())
|
||||
location = '/'+project+'/'+version+'/ident/'+ident2
|
||||
location = '/'+project+'/'+version+'/'+family2+'/ident/'+ident2
|
||||
else:
|
||||
mode = 'ident'
|
||||
if not(ident and search('^[A-Za-z0-9_-]*$', ident)):
|
||||
ident = ''
|
||||
url = 'ident/'+ident
|
||||
url = family + '/ident/' + ident
|
||||
else:
|
||||
status = 400
|
||||
else:
|
||||
|
|
@ -132,6 +137,7 @@ data = {
|
|||
'project': project,
|
||||
'projects': projects,
|
||||
'ident': ident,
|
||||
'family': family,
|
||||
'breadcrumb': '<a class="project" href="'+version+'/source">/</a>'
|
||||
}
|
||||
|
||||
|
|
@ -249,9 +255,11 @@ if mode == 'source':
|
|||
import pygments.lexers
|
||||
import pygments.formatters
|
||||
|
||||
filename, extension = os.path.splitext(path)
|
||||
fname = os.path.basename(path)
|
||||
filename, extension = os.path.splitext(fname)
|
||||
extension = extension[1:].lower()
|
||||
filename = os.path.basename(filename)
|
||||
family = query('family', fname)
|
||||
data['family'] = family
|
||||
|
||||
# Source common filter definitions
|
||||
os.chdir('filters')
|
||||
|
|
@ -302,7 +310,7 @@ if mode == 'source':
|
|||
elif mode == 'ident':
|
||||
data['title'] = ident+' identifier - '+title_suffix
|
||||
|
||||
symbol_definitions, symbol_references, symbol_doccomments_UNUSED = query('ident', tag, ident)
|
||||
symbol_definitions, symbol_references, symbol_doccomments_UNUSED = query('ident', tag, ident, family)
|
||||
|
||||
print('<div class="lxrident">')
|
||||
if len(symbol_definitions):
|
||||
|
|
|
|||
28
lib.py
28
lib.py
|
|
@ -183,6 +183,28 @@ def getDataDir():
|
|||
def currentProject():
|
||||
return os.path.basename(os.path.dirname(getDataDir()))
|
||||
|
||||
def hasSupportedExt(filename):
|
||||
ext = os.path.splitext(filename)[1]
|
||||
return ext.lower() in ['.c', '.cc', '.cpp', '.c++', '.cxx', '.h', '.s']
|
||||
def getFileFamily(filename):
|
||||
name, ext = os.path.splitext(filename)
|
||||
|
||||
if ext.lower() in ['.c', '.cc', '.cpp', '.c++', '.cxx', '.h', '.s'] :
|
||||
return 'C' # C file family and ASM
|
||||
elif ext.lower() in ['.dts', '.dtsi'] :
|
||||
return 'D' # Devicetree files
|
||||
elif name.lower()[:7] in ['kconfig'] and not ext.lower() in ['.rst']:
|
||||
# Some files are named like Kconfig-nommu so we only check the first 7 letters
|
||||
# We also exclude documentation files that can be named kconfig
|
||||
return 'K' # Kconfig files
|
||||
else :
|
||||
return None
|
||||
|
||||
compatibility_list = {
|
||||
'C' : ['C', 'K'],
|
||||
'K' : ['K'],
|
||||
'D' : ['D']
|
||||
}
|
||||
|
||||
# Check if families are compatible
|
||||
# First argument can be a list of different families
|
||||
# Second argument is the key for chossing the right array in the compatibility list
|
||||
def compatibleFamily(file_family, requested_family):
|
||||
return any(item in file_family for item in compatibility_list[requested_family])
|
||||
|
|
|
|||
51
query.py
51
query.py
|
|
@ -146,14 +146,24 @@ def query(cmd, *args):
|
|||
version = args[0]
|
||||
path = args[1]
|
||||
|
||||
if lib.hasSupportedExt(path):
|
||||
filename = os.path.basename(path)
|
||||
family = lib.getFileFamily(filename)
|
||||
|
||||
if family != None:
|
||||
buffer = BytesIO()
|
||||
tokens = scriptLines('tokenize-file', version, path)
|
||||
tokens = scriptLines('tokenize-file', version, path, family)
|
||||
even = True
|
||||
|
||||
prefix = b''
|
||||
if family == 'K':
|
||||
prefix = b'CONFIG_'
|
||||
|
||||
for tok in tokens:
|
||||
even = not even
|
||||
if even and db.defs.exists(tok) and lib.isIdent(tok):
|
||||
tok = b'\033[31m' + tok + b'\033[0m'
|
||||
tok2 = prefix + tok
|
||||
if (even and db.defs.exists(tok2) and lib.isIdent(tok2)
|
||||
and lib.compatibleFamily(db.defs.get(tok2).get_families(), family)):
|
||||
tok = b'\033[31m' + tok2 + b'\033[0m'
|
||||
else:
|
||||
tok = lib.unescape(tok)
|
||||
buffer.write(tok)
|
||||
|
|
@ -161,12 +171,20 @@ def query(cmd, *args):
|
|||
else:
|
||||
return decode(script('get-file', version, path))
|
||||
|
||||
elif cmd == 'family':
|
||||
# Get the family of a given file
|
||||
|
||||
filename = args[0]
|
||||
|
||||
return lib.getFileFamily(filename)
|
||||
|
||||
elif cmd == 'ident':
|
||||
|
||||
# Returns identifier search results
|
||||
|
||||
version = args[0]
|
||||
ident = args[1]
|
||||
family = args[2]
|
||||
|
||||
symbol_definitions = []
|
||||
symbol_references = []
|
||||
|
|
@ -196,9 +214,9 @@ def query(cmd, *args):
|
|||
# values. Therefore, we can sequentially step through the defs, refs,
|
||||
# and docs for each file in a version.
|
||||
|
||||
def_idx, def_type, def_line = next(defs_this_ident)
|
||||
ref_idx, ref_lines = next(refs)
|
||||
doc_idx, doc_line = next(docs)
|
||||
def_idx, def_type, def_line, def_family = next(defs_this_ident)
|
||||
ref_idx, ref_lines, ref_family = next(refs)
|
||||
doc_idx, doc_line, doc_family = next(docs)
|
||||
|
||||
dBuf = []
|
||||
rBuf = []
|
||||
|
|
@ -207,19 +225,21 @@ def query(cmd, *args):
|
|||
for file_idx, file_path in files_this_version:
|
||||
# Advance defs, refs, and docs to the current file
|
||||
while def_idx < file_idx:
|
||||
def_idx, def_type, def_line = next(defs_this_ident)
|
||||
def_idx, def_type, def_line, def_family = next(defs_this_ident)
|
||||
while ref_idx < file_idx:
|
||||
ref_idx, ref_lines = next(refs)
|
||||
ref_idx, ref_lines, ref_family = next(refs)
|
||||
while doc_idx < file_idx:
|
||||
doc_idx, doc_line = next(docs)
|
||||
doc_idx, doc_line, doc_family = next(docs)
|
||||
|
||||
# Copy information about this identifier into dBuf, rBuf, and docBuf.
|
||||
while def_idx == file_idx:
|
||||
dBuf.append((file_path, def_type, def_line))
|
||||
def_idx, def_type, def_line = next(defs_this_ident)
|
||||
if def_family == family:
|
||||
dBuf.append((file_path, def_type, def_line))
|
||||
def_idx, def_type, def_line, def_family = next(defs_this_ident)
|
||||
|
||||
if ref_idx == file_idx:
|
||||
rBuf.append((file_path, ref_lines))
|
||||
if lib.compatibleFamily(family, ref_family):
|
||||
rBuf.append((file_path, ref_lines))
|
||||
|
||||
if doc_idx == file_idx: # TODO should this be a `while`?
|
||||
docBuf.append((file_path, doc_line))
|
||||
|
|
@ -239,8 +259,8 @@ def query(cmd, *args):
|
|||
else:
|
||||
return('Unknown subcommand: ' + cmd + '\n')
|
||||
|
||||
def cmd_ident(version, ident, **kwargs):
|
||||
symbol_definitions, symbol_references, symbol_doccomments = query("ident", version, ident)
|
||||
def cmd_ident(version, ident, family, **kwargs):
|
||||
symbol_definitions, symbol_references, symbol_doccomments = query("ident", version, ident, family)
|
||||
print("Symbol Definitions:")
|
||||
for symbol_definition in symbol_definitions:
|
||||
print(symbol_definition)
|
||||
|
|
@ -266,6 +286,7 @@ if __name__ == "__main__":
|
|||
|
||||
ident_subparser = subparsers.add_parser('ident', help="Get definitions and references of an identifier")
|
||||
ident_subparser.add_argument('ident', type=str, help="The name of the identifier")
|
||||
ident_subparser.add_argument('family', type=str, help="The file family requested")
|
||||
ident_subparser.set_defaults(func=cmd_ident)
|
||||
|
||||
file_subparser = subparsers.add_parser('file', help="Get a source file")
|
||||
|
|
|
|||
50
script.sh
50
script.sh
|
|
@ -101,9 +101,15 @@ tokenize_file()
|
|||
ref="$v:`denormalize $opt2`"
|
||||
fi
|
||||
|
||||
if [ $opt3 = "D" ]; then #Don't cut around '-' in devicetrees
|
||||
regex='s%((/\*.*?\*/|//.*?\001|[^'"'"']"(\\.|.)*?"|# *include *<.*?>|[^\w-])+)([\w-]+)?%\1\n\4\n%g'
|
||||
else
|
||||
regex='s%((/\*.*?\*/|//.*?\001|[^'"'"']"(\\.|.)*?"|# *include *<.*?>|\W)+)(\w+)?%\1\n\4\n%g'
|
||||
fi
|
||||
|
||||
git cat-file blob $ref 2>/dev/null |
|
||||
tr '\n' '\1' |
|
||||
perl -pe 's%((/\*.*?\*/|//.*?\001|[^'"'"']"(\\.|.)*?"|# *include *<.*?>|\W)+)(\w+)?%\1\n\4\n%g' |
|
||||
perl -pe "$regex" |
|
||||
head -n -1
|
||||
}
|
||||
|
||||
|
|
@ -136,12 +142,49 @@ untokenize()
|
|||
}
|
||||
|
||||
parse_defs()
|
||||
{
|
||||
case $opt3 in
|
||||
"C")
|
||||
parse_defs_C
|
||||
;;
|
||||
"K")
|
||||
parse_defs_K
|
||||
;;
|
||||
"D")
|
||||
parse_defs_D
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
parse_defs_C()
|
||||
{
|
||||
tmp=`mktemp -d`
|
||||
full_path=$tmp/$opt2
|
||||
git cat-file blob "$opt1" > "$full_path"
|
||||
ctags -x --c-kinds=+p-m "$full_path" |
|
||||
grep -av "^operator " |
|
||||
ctags -x --kinds-c=+p-m "$full_path" |
|
||||
grep -avE "^operator |CONFIG_" |
|
||||
awk '{print $1" "$2" "$3}'
|
||||
rm "$full_path"
|
||||
rmdir $tmp
|
||||
}
|
||||
|
||||
parse_defs_K()
|
||||
{
|
||||
tmp=`mktemp -d`
|
||||
full_path=$tmp/$opt2
|
||||
git cat-file blob "$opt1" > "$full_path"
|
||||
ctags -x --language-force=kconfig "$full_path" |
|
||||
awk '{print "CONFIG_"$1" "$2" "$3}'
|
||||
rm "$full_path"
|
||||
rmdir $tmp
|
||||
}
|
||||
|
||||
parse_defs_D()
|
||||
{
|
||||
tmp=`mktemp -d`
|
||||
full_path=$tmp/$opt2
|
||||
git cat-file blob "$opt1" > "$full_path"
|
||||
ctags -x --language-force=dts "$full_path" |
|
||||
awk '{print $1" "$2" "$3}'
|
||||
rm "$full_path"
|
||||
rmdir $tmp
|
||||
|
|
@ -171,6 +214,7 @@ test $# -gt 0 || set help
|
|||
cmd=$1
|
||||
opt1=$2
|
||||
opt2=$3
|
||||
opt3=$4
|
||||
shift
|
||||
|
||||
denormalize()
|
||||
|
|
|
|||
|
|
@ -80,12 +80,12 @@ ok( (-r File::Spec->catfile($db_dir, $_)), "$_ exists" )
|
|||
# Spot-check some identifiers
|
||||
|
||||
run_produces_ok('ident query (nonexistent)',
|
||||
[$query_py, qw(v5.4 ident SOME_NONEXISTENT_IDENTIFIER_XYZZY_PLUGH)],
|
||||
[$query_py, qw(v5.4 ident SOME_NONEXISTENT_IDENTIFIER_XYZZY_PLUGH C)],
|
||||
[qr{^Symbol Definitions:}, qr{^Symbol References:}, qr{^\s*$}],
|
||||
MUST_SUCCEED);
|
||||
|
||||
run_produces_ok('ident query (existent)',
|
||||
[$query_py, qw(v5.4 ident i2c_acpi_notify)],
|
||||
[$query_py, qw(v5.4 ident i2c_acpi_notify C)],
|
||||
[qr{^Symbol Definitions:}, qr{^Symbol References:},
|
||||
qr{drivers/i2c/i2c-core-acpi\.c.+\b402\b.+\bfunction\b}, # def
|
||||
qr{drivers/i2c/i2c-core-acpi\.c.+\b402,439} # refs
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ ok_or_die( -d $tenv->lxr_data_dir, 'database dir exists',
|
|||
# Spot-check some identifiers
|
||||
|
||||
run_produces_ok('doc-comment query (nonexistent)',
|
||||
[$tenv->query_py, qw(v5.4 ident SOME_NONEXISTENT_IDENTIFIER_XYZZY_PLUGH)],
|
||||
[$tenv->query_py, qw(v5.4 ident SOME_NONEXISTENT_IDENTIFIER_XYZZY_PLUGH C)],
|
||||
[
|
||||
qr{^Documented in:},
|
||||
{doc => { not => qr{/} }}, # No file paths in the doc section
|
||||
|
|
@ -53,7 +53,7 @@ run_produces_ok('doc-comment query (nonexistent)',
|
|||
MUST_SUCCEED);
|
||||
|
||||
run_produces_ok('doc-comment query (existent but not documented)',
|
||||
[$tenv->query_py, qw(v5.4 ident gsb_buffer)], # in drivers/i2c/i2c-core-acpi.c
|
||||
[$tenv->query_py, qw(v5.4 ident gsb_buffer C)], # in drivers/i2c/i2c-core-acpi.c
|
||||
[
|
||||
qr{^Documented in:},
|
||||
{doc => { not => qr{/} }}
|
||||
|
|
@ -61,7 +61,7 @@ run_produces_ok('doc-comment query (existent but not documented)',
|
|||
MUST_SUCCEED);
|
||||
|
||||
run_produces_ok('ident query (existent, function, documented in C file)',
|
||||
[$tenv->query_py, qw(v5.4 ident i2c_acpi_get_i2c_resource)],
|
||||
[$tenv->query_py, qw(v5.4 ident i2c_acpi_get_i2c_resource C)],
|
||||
[
|
||||
qr{^Documented in:},
|
||||
{doc => qr{drivers/i2c/i2c-core-acpi\.c.+\b45\b}},
|
||||
|
|
@ -69,7 +69,7 @@ run_produces_ok('ident query (existent, function, documented in C file)',
|
|||
MUST_SUCCEED);
|
||||
|
||||
run_produces_ok('ident query (existent, function, documented in C file, #102)',
|
||||
[$tenv->query_py, qw(v5.4 ident documented_function_XYZZY)],
|
||||
[$tenv->query_py, qw(v5.4 ident documented_function_XYZZY C)],
|
||||
[
|
||||
qr{^Documented in:},
|
||||
{doc => qr{issue102\.c.+\b6\b}},
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ class APITest(testing.TestCase):
|
|||
self.app = create_ident_getter()
|
||||
|
||||
def test_identifier_not_found(self):
|
||||
result = self.simulate_get('/ident/tree/SOME_NONEXISTENT_IDENTIFIER', query_string="version=latest")
|
||||
result = self.simulate_get('/ident/tree/SOME_NONEXISTENT_IDENTIFIER', query_string="version=latest&family=C")
|
||||
|
||||
self.assertEqual(result.status_code, 200)
|
||||
self.assertEqual(result.json, {'definitions': [], 'references':[]})
|
||||
|
|
@ -33,14 +33,13 @@ class APITest(testing.TestCase):
|
|||
self.assertEqual(result.json["description"], required_response.description)
|
||||
|
||||
def test_existing_identifier(self):
|
||||
result_for_specific_version = self.simulate_get('/ident/tree/of_i2c_get_board_info', query_string="version=v5.4")
|
||||
result_for_latest_version = self.simulate_get('/ident/tree/of_i2c_get_board_info', query_string="version=latest")
|
||||
result_for_specific_version = self.simulate_get('/ident/tree/of_i2c_get_board_info', query_string="version=v5.4&family=C")
|
||||
result_for_latest_version = self.simulate_get('/ident/tree/of_i2c_get_board_info', query_string="version=latest&family=C")
|
||||
|
||||
expected_json = {
|
||||
'definitions':
|
||||
[
|
||||
{'path': 'drivers/i2c/i2c-core-of.c', 'line': 22, 'type': 'function'},
|
||||
{'path': 'drivers/i2c/i2c-core-of.c', 'line': 62, 'type': 'variable'},
|
||||
{'path': 'include/linux/i2c.h', 'line': 968, 'type': 'function'},
|
||||
{'path': 'include/linux/i2c.h', 'line': 941, 'type': 'prototype'}
|
||||
],
|
||||
|
|
@ -55,4 +54,4 @@ class APITest(testing.TestCase):
|
|||
self.assertEqual(result_for_latest_version.status_code, 200)
|
||||
|
||||
self.assertEqual(result_for_specific_version.json, expected_json)
|
||||
self.assertEqual(result_for_latest_version.json, expected_json)
|
||||
self.assertEqual(result_for_latest_version.json, expected_json)
|
||||
|
|
|
|||
|
|
@ -23,7 +23,13 @@
|
|||
{{breadcrumb}}
|
||||
</div>
|
||||
<div class="search">
|
||||
<form method="post" action="{{version}}/ident">
|
||||
<form method="post" action="{{version}}/ident">
|
||||
<select name="f">
|
||||
<option value="C" {% if family=="C" %} selected="selected"{% endif %}>C/CPP/ASM</option>
|
||||
<option value="K" {% if family=="K" %} selected="selected"{% endif %}>Kconfig</option>
|
||||
<option value="D" {% if family=="D" %} selected="selected"{% endif %}>Devicetree</option>
|
||||
</select>
|
||||
|
||||
<input placeholder="Search Identifier" type="text" name="i" value="{{ident}}"/>
|
||||
<button class="icon-search"><span class="screenreader">Go get it</span></button>
|
||||
</form>
|
||||
|
|
|
|||
388
update.py
388
update.py
|
|
@ -27,158 +27,316 @@ import lib
|
|||
import data
|
||||
import os
|
||||
from data import PathList
|
||||
from threading import Thread, Lock, Event, Condition
|
||||
|
||||
verbose = False
|
||||
|
||||
db = data.DB(lib.getDataDir(), readonly=False)
|
||||
|
||||
# Store new blobs hashed and file names (without path) for new tag
|
||||
|
||||
def updateBlobIDs(tag):
|
||||
hash_file_lock = Lock() #Lock for db.hash and db.file
|
||||
defs_lock = Lock() #Lock for db.defs
|
||||
tag_ready = Condition() #Waiting for new tags
|
||||
|
||||
if db.vars.exists('numBlobs'):
|
||||
idx = db.vars.get('numBlobs')
|
||||
else:
|
||||
idx = 0
|
||||
new_idxes = [] # (new idxes, Event idxes ready, Event defs ready)
|
||||
|
||||
# Get blob hashes and associated file names (without path)
|
||||
blobs = scriptLines('list-blobs', '-f', tag)
|
||||
tags_done = False #True if all tags have been added to new_idxes
|
||||
|
||||
newIdxes = []
|
||||
for blob in blobs:
|
||||
hash, filename = blob.split(b' ',maxsplit=1)
|
||||
if not db.blob.exists(hash):
|
||||
db.blob.put(hash, idx)
|
||||
db.hash.put(idx, hash)
|
||||
db.file.put(idx, filename)
|
||||
newIdxes.append(idx)
|
||||
|
||||
class UpdateIdVersion(Thread):
|
||||
def __init__(self, tag_buf):
|
||||
Thread.__init__(self, name="UpdateIdVersionElixir")
|
||||
self.tag_buf = tag_buf
|
||||
|
||||
def run(self):
|
||||
global new_idxes, tags_done, tag_ready
|
||||
self.index = 0
|
||||
|
||||
for tag in self.tag_buf:
|
||||
|
||||
new_idxes.append((self.update_blob_ids(tag), Event(), Event()))
|
||||
|
||||
progress(tag.decode() + ': ' + str(len(new_idxes[self.index][0])) +
|
||||
' new blobs', self.index+1)
|
||||
|
||||
self.update_versions(tag)
|
||||
|
||||
new_idxes[self.index][1].set() #Tell that the tag is ready
|
||||
|
||||
self.index += 1
|
||||
|
||||
#Wake up waiting threads
|
||||
with tag_ready:
|
||||
tag_ready.notify_all()
|
||||
|
||||
tags_done = True
|
||||
|
||||
def update_blob_ids(self, tag):
|
||||
|
||||
global hash_file_lock
|
||||
|
||||
if db.vars.exists('numBlobs'):
|
||||
idx = db.vars.get('numBlobs')
|
||||
else:
|
||||
idx = 0
|
||||
|
||||
# Get blob hashes and associated file names (without path)
|
||||
blobs = scriptLines('list-blobs', '-f', tag)
|
||||
|
||||
new_idxes = []
|
||||
for blob in blobs:
|
||||
hash, filename = blob.split(b' ',maxsplit=1)
|
||||
if not db.blob.exists(hash):
|
||||
db.blob.put(hash, idx)
|
||||
|
||||
with hash_file_lock:
|
||||
db.hash.put(idx, hash)
|
||||
db.file.put(idx, filename)
|
||||
|
||||
new_idxes.append(idx)
|
||||
if verbose:
|
||||
print(f"New blob #{idx} {hash}:{filename}")
|
||||
idx += 1
|
||||
db.vars.put('numBlobs', idx)
|
||||
return new_idxes
|
||||
|
||||
def update_versions(self, tag):
|
||||
|
||||
# Get blob hashes and associated file paths
|
||||
blobs = scriptLines('list-blobs', '-p', tag)
|
||||
buf = []
|
||||
|
||||
for blob in blobs:
|
||||
hash, path = blob.split(b' ', maxsplit=1)
|
||||
idx = db.blob.get(hash)
|
||||
buf.append((idx, path))
|
||||
|
||||
buf = sorted(buf)
|
||||
obj = PathList()
|
||||
for idx, path in buf:
|
||||
obj.append(idx, path)
|
||||
if verbose:
|
||||
print(f"New blob #{idx} {hash}:{filename}")
|
||||
idx += 1
|
||||
db.vars.put('numBlobs', idx)
|
||||
return newIdxes
|
||||
print(f"Tag {tag}: adding #{idx} {path}")
|
||||
db.vers.put(tag, obj, sync=True)
|
||||
|
||||
def updateVersions(tag):
|
||||
|
||||
# Get blob hashes and associated file paths
|
||||
blobs = scriptLines('list-blobs', '-p', tag)
|
||||
buf = []
|
||||
|
||||
for blob in blobs:
|
||||
hash, path = blob.split(b' ', maxsplit=1)
|
||||
idx = db.blob.get(hash)
|
||||
buf.append((idx, path))
|
||||
class UpdateDefs(Thread):
|
||||
def __init__(self):
|
||||
Thread.__init__(self, name="UpdateDefsElixir")
|
||||
|
||||
buf = sorted(buf)
|
||||
obj = PathList()
|
||||
for idx, path in buf:
|
||||
obj.append(idx, path)
|
||||
if verbose:
|
||||
print(f"Tag {tag}: adding #{idx} {path}")
|
||||
db.vers.put(tag, obj, sync=True)
|
||||
def run(self):
|
||||
global new_idxes, tags_done, tag_ready
|
||||
|
||||
def updateDefinitions(idxes):
|
||||
for idx in idxes:
|
||||
if (idx % 1000 == 0): progress('defs: ' + str(idx))
|
||||
hash = db.hash.get(idx)
|
||||
filename = db.file.get(idx)
|
||||
self.index = 0
|
||||
|
||||
if not lib.hasSupportedExt(filename): continue
|
||||
while(not (tags_done and self.index == len(new_idxes))):
|
||||
if(self.index == len(new_idxes)):
|
||||
#Wait for new tags
|
||||
with tag_ready:
|
||||
tag_ready.wait()
|
||||
continue
|
||||
|
||||
lines = scriptLines('parse-defs', hash, filename)
|
||||
for l in lines:
|
||||
ident, type, line = l.split(b' ')
|
||||
type = type.decode()
|
||||
line = int(line.decode())
|
||||
new_idxes[self.index][1].wait() #Make sure the tag is ready
|
||||
|
||||
if db.defs.exists(ident):
|
||||
obj = db.defs.get(ident)
|
||||
else:
|
||||
obj = data.DefList()
|
||||
self.update_definitions(new_idxes[self.index][0])
|
||||
|
||||
obj.append(idx, type, line)
|
||||
if verbose:
|
||||
print(f"def {type} {ident} in #{idx} @ {line}")
|
||||
db.defs.put(ident, obj)
|
||||
new_idxes[self.index][2].set() #Tell that UpdateDefs processed the tag
|
||||
|
||||
def updateReferences(idxes):
|
||||
for idx in idxes:
|
||||
if (idx % 1000 == 0): progress('refs: ' + str(idx))
|
||||
hash = db.hash.get(idx)
|
||||
filename = db.file.get(idx)
|
||||
self.index += 1
|
||||
|
||||
if not lib.hasSupportedExt(filename): continue
|
||||
|
||||
tokens = scriptLines('tokenize-file', '-b', hash)
|
||||
even = True
|
||||
lineNum = 1
|
||||
idents = {}
|
||||
for tok in tokens:
|
||||
even = not even
|
||||
if even:
|
||||
if db.defs.exists(tok) and lib.isIdent(tok):
|
||||
if tok in idents:
|
||||
idents[tok] += ',' + str(lineNum)
|
||||
def update_definitions(self, idxes):
|
||||
global hash_file_lock, defs_lock
|
||||
|
||||
for idx in idxes:
|
||||
if (idx % 1000 == 0): progress('defs: ' + str(idx), self.index+1)
|
||||
|
||||
with hash_file_lock:
|
||||
hash = db.hash.get(idx)
|
||||
filename = db.file.get(idx)
|
||||
|
||||
family = lib.getFileFamily(filename);
|
||||
if family == None: continue
|
||||
|
||||
lines = scriptLines('parse-defs', hash, filename, family)
|
||||
for l in lines:
|
||||
ident, type, line = l.split(b' ')
|
||||
type = type.decode()
|
||||
line = int(line.decode())
|
||||
|
||||
with defs_lock:
|
||||
if db.defs.exists(ident):
|
||||
obj = db.defs.get(ident)
|
||||
else:
|
||||
idents[tok] = str(lineNum)
|
||||
else:
|
||||
lineNum += tok.count(b'\1')
|
||||
obj = data.DefList()
|
||||
|
||||
for ident, lines in idents.items():
|
||||
if db.refs.exists(ident):
|
||||
obj = db.refs.get(ident)
|
||||
else:
|
||||
obj = data.RefList()
|
||||
obj.add_family(family)
|
||||
obj.append(idx, type, line, family)
|
||||
if verbose:
|
||||
print(f"def {type} {ident} in #{idx} @ {line}")
|
||||
with defs_lock:
|
||||
db.defs.put(ident, obj)
|
||||
|
||||
obj.append(idx, lines)
|
||||
if verbose:
|
||||
print(f"ref: {ident} in #{idx} @ {lines}")
|
||||
db.refs.put(ident, obj)
|
||||
|
||||
def updateDocComments(idxes):
|
||||
for idx in idxes:
|
||||
if (idx % 1000 == 0): progress('docs: ' + str(idx))
|
||||
hash = db.hash.get(idx)
|
||||
filename = db.file.get(idx)
|
||||
class UpdateRefs(Thread):
|
||||
def __init__(self):
|
||||
Thread.__init__(self, name="UpdateRefsElixir")
|
||||
|
||||
if not lib.hasSupportedExt(filename): continue
|
||||
def run(self):
|
||||
global new_idxes, tags_done
|
||||
|
||||
lines = scriptLines('parse-docs', hash, filename)
|
||||
for l in lines:
|
||||
ident, line = l.split(b' ')
|
||||
line = int(line.decode())
|
||||
self.index = 0
|
||||
|
||||
if db.docs.exists(ident):
|
||||
obj = db.docs.get(ident)
|
||||
else:
|
||||
obj = data.RefList()
|
||||
while(not (tags_done and self.index == len(new_idxes))):
|
||||
if(self.index == len(new_idxes)):
|
||||
#Wait for new tags
|
||||
with tag_ready:
|
||||
tag_ready.wait()
|
||||
continue
|
||||
|
||||
obj.append(idx, str(line))
|
||||
if verbose:
|
||||
print(f"doc: {ident} in #{idx} @ {line}")
|
||||
db.docs.put(ident, obj)
|
||||
new_idxes[self.index][1].wait() #Make sure the tag is ready
|
||||
new_idxes[self.index][2].wait() #Make sure UpdateDefs processed the tag
|
||||
|
||||
self.update_references(new_idxes[self.index][0])
|
||||
|
||||
self.index += 1
|
||||
|
||||
def update_references(self, idxes):
|
||||
global hash_file_lock, defs_lock
|
||||
|
||||
for idx in idxes:
|
||||
if (idx % 1000 == 0): progress('refs: ' + str(idx), self.index+1)
|
||||
|
||||
with hash_file_lock:
|
||||
hash = db.hash.get(idx)
|
||||
filename = db.file.get(idx)
|
||||
|
||||
family = lib.getFileFamily(filename)
|
||||
if family == None: continue
|
||||
|
||||
prefix = b''
|
||||
# Kconfig values are saved as CONFIG_<value>
|
||||
if family == 'K':
|
||||
prefix = b'CONFIG_'
|
||||
|
||||
tokens = scriptLines('tokenize-file', '-b', hash, family)
|
||||
even = True
|
||||
line_num = 1
|
||||
idents = {}
|
||||
for tok in tokens:
|
||||
even = not even
|
||||
if even:
|
||||
tok = prefix + tok
|
||||
|
||||
with defs_lock:
|
||||
if db.defs.exists(tok) and lib.isIdent(tok):
|
||||
if tok in idents:
|
||||
idents[tok] += ',' + str(line_num)
|
||||
else:
|
||||
idents[tok] = str(line_num)
|
||||
|
||||
else:
|
||||
line_num += tok.count(b'\1')
|
||||
|
||||
for ident, lines in idents.items():
|
||||
if db.refs.exists(ident):
|
||||
obj = db.refs.get(ident)
|
||||
else:
|
||||
obj = data.RefList()
|
||||
|
||||
obj.append(idx, lines, family)
|
||||
if verbose:
|
||||
print(f"ref: {ident} in #{idx} @ {lines}")
|
||||
db.refs.put(ident, obj)
|
||||
|
||||
|
||||
class UpdateDocs(Thread):
|
||||
def __init__(self):
|
||||
Thread.__init__(self, name="UpdateDocsElixir")
|
||||
|
||||
def run(self):
|
||||
global new_idxes, tags_done
|
||||
|
||||
self.index = 0
|
||||
|
||||
while(not (tags_done and self.index == len(new_idxes))):
|
||||
if(self.index == len(new_idxes)):
|
||||
#Wait for new tags
|
||||
with tag_ready:
|
||||
tag_ready.wait()
|
||||
continue
|
||||
|
||||
new_idxes[self.index][1].wait() #Make sure the tag is ready
|
||||
|
||||
self.update_doc_comments(new_idxes[self.index][0])
|
||||
|
||||
self.index += 1
|
||||
|
||||
def update_doc_comments(self, idxes):
|
||||
global hash_file_lock
|
||||
|
||||
for idx in idxes:
|
||||
if (idx % 1000 == 0): progress('docs: ' + str(idx), self.index+1)
|
||||
|
||||
with hash_file_lock:
|
||||
hash = db.hash.get(idx)
|
||||
filename = db.file.get(idx)
|
||||
|
||||
family = lib.getFileFamily(filename)
|
||||
if family == None: continue
|
||||
|
||||
lines = scriptLines('parse-docs', hash, filename)
|
||||
for l in lines:
|
||||
ident, line = l.split(b' ')
|
||||
line = int(line.decode())
|
||||
|
||||
if db.docs.exists(ident):
|
||||
obj = db.docs.get(ident)
|
||||
else:
|
||||
obj = data.RefList()
|
||||
|
||||
obj.append(idx, str(line), family)
|
||||
if verbose:
|
||||
print(f"doc: {ident} in #{idx} @ {line}")
|
||||
db.docs.put(ident, obj)
|
||||
|
||||
|
||||
def progress(msg, current):
|
||||
print('{} - {} ({:.0%})'.format(project, msg, current/num_tags))
|
||||
|
||||
def progress(msg):
|
||||
print('{} - {} ({:.0%})'.format(project, msg, tagCount/numTags))
|
||||
|
||||
# Main
|
||||
|
||||
tagBuf = []
|
||||
tag_buf = []
|
||||
for tag in scriptLines('list-tags'):
|
||||
if not db.vers.exists(tag):
|
||||
tagBuf.append(tag)
|
||||
tag_buf.append(tag)
|
||||
|
||||
numTags = len(tagBuf)
|
||||
tagCount = 0
|
||||
num_tags = len(tag_buf)
|
||||
project = lib.currentProject()
|
||||
|
||||
print(project + ' - found ' + str(len(tagBuf)) + ' new tags')
|
||||
print(project + ' - found ' + str(len(tag_buf)) + ' new tags')
|
||||
|
||||
for tag in tagBuf:
|
||||
tagCount +=1
|
||||
newIdxes = updateBlobIDs(tag)
|
||||
progress(tag.decode() + ': ' + str(len(newIdxes)) + ' new blobs')
|
||||
updateVersions(tag)
|
||||
updateDefinitions(newIdxes)
|
||||
updateReferences(newIdxes)
|
||||
updateDocComments(newIdxes)
|
||||
id_version_thread = UpdateIdVersion(tag_buf)
|
||||
defs_thread = UpdateDefs()
|
||||
refs_thread = UpdateRefs()
|
||||
docs_thread = UpdateDocs()
|
||||
|
||||
#Start to process tags
|
||||
id_version_thread.start()
|
||||
|
||||
#Wait until the first tag is ready
|
||||
with tag_ready:
|
||||
tag_ready.wait()
|
||||
|
||||
#Start remaining threads
|
||||
defs_thread.start()
|
||||
refs_thread.start()
|
||||
docs_thread.start()
|
||||
|
||||
#Make sure all threads finished
|
||||
id_version_thread.join()
|
||||
defs_thread.join()
|
||||
refs_thread.join()
|
||||
docs_thread.join()
|
||||
|
|
|
|||
|
|
@ -16,16 +16,16 @@ verbose = False
|
|||
#List of test elements
|
||||
project = 'linux'
|
||||
versions = ['latest', 'v5.6.2']
|
||||
idents = [ 'loopback',
|
||||
'devm_register_reboot_notifier',
|
||||
'notrace',
|
||||
'arch_local_irq_restore',
|
||||
'blk_queue_dma_alignment',
|
||||
'spinlock_t',
|
||||
'max',
|
||||
'task_struct',
|
||||
'eth_header',
|
||||
'sk_buff' ]
|
||||
idents = [ ('loopback', 'C'),
|
||||
('devm_register_reboot_notifier', 'C'),
|
||||
('notrace', 'C'),
|
||||
('arch_local_irq_restore', 'C'),
|
||||
('blk_queue_dma_alignment', 'C'),
|
||||
('spinlock_t', 'C'),
|
||||
('max', 'C'),
|
||||
('task_struct', 'C'),
|
||||
('eth_header', 'C'),
|
||||
('sk_buff', 'C') ]
|
||||
|
||||
files = [ '/block/partitions/osf.c',
|
||||
'/mm/kasan/quarantine.c',
|
||||
|
|
@ -68,7 +68,7 @@ def get_ident(ident, version):
|
|||
if version == 'latest':
|
||||
version = query('latest')
|
||||
|
||||
return query('ident', version, ident)
|
||||
return query('ident', version, ident[0], ident[1])
|
||||
|
||||
def get_file(path, version):
|
||||
if version == 'latest':
|
||||
|
|
@ -146,4 +146,4 @@ for version in versions:
|
|||
print((BOLD + "Min:" + NORMAL + " {0:.6f} ms\n"
|
||||
+ BOLD + "Max:" + NORMAL + " {1:.6f} ms\n"
|
||||
+ BOLD + "Average:" + NORMAL + " {2:.6f} ms\n"
|
||||
).format(files_min, files_max, files_average))
|
||||
).format(files_min, files_max, files_average))
|
||||
|
|
|
|||
Loading…
Reference in a new issue