Move query and get_idents* functions into the Query class

This commit is contained in:
Franciszek Stachura 2024-01-04 17:36:28 +01:00 committed by Michael Opdenacker
parent 910bdd7938
commit 55bb9d47b4

652
query.py
View file

@ -50,9 +50,6 @@ class Query:
self.dts_comp_support = int(self.script('dts-comp'))
self.db = data.DB(data_dir, readonly=True, dtscomp=self.dts_comp_support)
def query(self, cmd, *args):
return query(cmd, *args, ctx=self)
def script(self, *args):
return script(*args, env=self.getEnv())
@ -60,330 +57,337 @@ class Query:
return scriptLines(*args, env=self.getEnv())
def getEnv(self):
return {"LXR_REPO_DIR": self.repo_dir}
return {
"LXR_REPO_DIR": self.repo_dir,
"LXR_DATA_DIR": self.data_dir,
}
def query(self, cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in self.scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if num == 1:
tag, = taginfo
elif num == 2:
submenu,tag = taginfo
elif num ==3:
topmenu,submenu,tag = taginfo
if self.db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
previous = None
tag = ''
index = 0
# If we get the same tag twice, we are at the oldest one
while not self.db.vers.exists(tag) and previous != tag:
previous = tag
tag = decode(self.script('get-latest', str(index))).rstrip('\n')
index += 1
return tag
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(self.script('get-type', version, path)).strip()
elif cmd == 'exist':
# Returns True if the requested file exists, otherwise returns False
version = args[0]
path = args[1]
dirname, filename = os.path.split(path)
entries = decode(self.script('get-dir', version, dirname)).split("\n")[:-1]
for entry in entries:
fname = entry.split(" ")[1]
if fname == filename:
return True
return False
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(self.script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
filename = os.path.basename(path)
family = lib.getFileFamily(filename)
if family != None:
buffer = BytesIO()
tokens = self.scriptLines('tokenize-file', version, path, family)
even = True
prefix = b''
if family == 'K':
prefix = b'CONFIG_'
for tok in tokens:
even = not even
tok2 = prefix + tok
if (even and self.db.defs.exists(tok2) and
(lib.compatibleFamily(self.db.defs.get(tok2).get_families(), family) or
lib.compatibleMacro(self.db.defs.get(tok2).get_macros(), family))):
tok = b'\033[31m' + tok2 + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(self.script('get-file', version, path))
elif cmd == 'family':
# Get the family of a given file
filename = args[0]
return lib.getFileFamily(filename)
elif cmd == 'dts-comp':
# Get state of dts_comp_support
return self.dts_comp_support
elif cmd == 'dts-comp-exists':
# Check if a dts compatible string exists
ident = args[0]
if self.dts_comp_support:
return self.db.comps.exists(ident)
else:
return False
elif cmd == 'keys':
# Return all keys of a given database
# /!\ This can take a while /!\
name = args[0]
if name == 'vars':
return self.db.vars.get_keys()
elif name == 'blob':
return self.db.blob.get_keys()
elif name == 'hash':
return self.db.hash.get_keys()
elif name == 'file':
return self.db.file.get_keys()
elif name == 'vers':
return self.db.vers.get_keys()
elif name == 'defs':
return self.db.defs.get_keys()
elif name == 'refs':
return self.db.refs.get_keys()
elif name == 'docs':
return self.db.docs.get_keys()
elif name == 'comps' and self.dts_comp_support:
return self.db.comps.get_keys()
elif name == 'comps_docs' and self.dts_comp_support:
return self.db.comps_docs.get_keys()
else:
return []
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
family = args[2]
# DT bindings compatible strings are handled differently
if family == 'B':
return self.get_idents_comps(version, ident)
else:
return self.get_idents_defs(version, ident, family)
else:
return 'Unknown subcommand: ' + cmd + '\n'
def get_idents_comps(self, version, ident):
# DT bindings compatible strings are handled differently
# They are defined in C files
# Used in DT files
# Documented in documentation files
symbol_c = []
symbol_dts = []
symbol_docs = []
# DT compatible strings are quoted in the database
ident = parse.quote(ident)
if not self.dts_comp_support or not self.db.comps.exists(ident):
return symbol_c, symbol_dts, symbol_docs
files_this_version = self.db.vers.get(version).iter()
comps = self.db.comps.get(ident).iter(dummy=True)
if self.db.comps_docs.exists(ident):
comps_docs = self.db.comps_docs.get(ident).iter(dummy=True)
else:
comps_docs = data.RefList().iter(dummy=True)
comps_idx, comps_lines, comps_family = next(comps)
comps_docs_idx, comps_docs_lines, comps_docs_family = next(comps_docs)
compsCBuf = [] # C/CPP/ASM files
compsDBuf = [] # DT files
compsBBuf = [] # DT bindings docs files
for file_idx, file_path in files_this_version:
while comps_idx < file_idx:
comps_idx, comps_lines, comps_family = next(comps)
while comps_docs_idx < file_idx:
comps_docs_idx, comps_docs_lines, comps_docs_family = next(comps_docs)
if comps_idx == file_idx:
if comps_family == 'C':
compsCBuf.append((file_path, comps_lines))
elif comps_family == 'D':
compsDBuf.append((file_path, comps_lines))
if comps_docs_idx == file_idx:
compsBBuf.append((file_path, comps_docs_lines))
for path, cline in sorted(compsCBuf):
symbol_c.append(SymbolInstance(path, cline, 'compatible'))
for path, dlines in sorted(compsDBuf):
symbol_dts.append(SymbolInstance(path, dlines))
for path, blines in sorted(compsBBuf):
symbol_docs.append(SymbolInstance(path, blines))
return symbol_c, symbol_dts, symbol_docs
def get_idents_defs(self, version, ident, family):
symbol_definitions = []
symbol_references = []
symbol_doccomments = []
if not self.db.defs.exists(ident):
return symbol_definitions, symbol_references, symbol_doccomments
if not self.db.vers.exists(version):
return symbol_definitions, symbol_references, symbol_doccomments
files_this_version = self.db.vers.get(version).iter()
this_ident = self.db.defs.get(ident)
defs_this_ident = this_ident.iter(dummy=True)
macros_this_ident = this_ident.get_macros()
# FIXME: see why we can have a discrepancy between defs_this_ident and refs
if self.db.refs.exists(ident):
refs = self.db.refs.get(ident).iter(dummy=True)
else:
refs = data.RefList().iter(dummy=True)
if self.db.docs.exists(ident):
docs = self.db.docs.get(ident).iter(dummy=True)
else:
docs = data.RefList().iter(dummy=True)
# vers, defs, refs, and docs are all populated by update.py in order of
# idx, and there is a one-to-one mapping between blob hashes and idx
# values. Therefore, we can sequentially step through the defs, refs,
# and docs for each file in a version.
def_idx, def_type, def_line, def_family = next(defs_this_ident)
ref_idx, ref_lines, ref_family = next(refs)
doc_idx, doc_line, doc_family = next(docs)
dBuf = []
rBuf = []
docBuf = []
for file_idx, file_path in files_this_version:
# Advance defs, refs, and docs to the current file
while def_idx < file_idx:
def_idx, def_type, def_line, def_family = next(defs_this_ident)
while ref_idx < file_idx:
ref_idx, ref_lines, ref_family = next(refs)
while doc_idx < file_idx:
doc_idx, doc_line, doc_family = next(docs)
# Copy information about this identifier into dBuf, rBuf, and docBuf.
while def_idx == file_idx:
if (def_family == family or family == 'A'
or lib.compatibleMacro(macros_this_ident, family)):
dBuf.append((file_path, def_type, def_line))
def_idx, def_type, def_line, def_family = next(defs_this_ident)
if ref_idx == file_idx:
if lib.compatibleFamily(family, ref_family) or family == 'A':
rBuf.append((file_path, ref_lines))
if doc_idx == file_idx: # TODO should this be a `while`?
docBuf.append((file_path, doc_line))
# Sort dBuf by path name before sorting by type in the loop
dBuf.sort()
for path, type, dline in sorted(dBuf, key=lambda d: d[1], reverse=True):
symbol_definitions.append(SymbolInstance(path, dline, type))
for path, rlines in sorted(rBuf):
symbol_references.append(SymbolInstance(path, rlines))
for path, docline in sorted(docBuf):
symbol_doccomments.append(SymbolInstance(path, docline))
return symbol_definitions, symbol_references, symbol_doccomments
default_query = Query(lib.getDataDir(), lib.getRepoDir())
def query(cmd, *args, ctx=default_query):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in ctx.scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if num == 1:
tag, = taginfo
elif num == 2:
submenu,tag = taginfo
elif num ==3:
topmenu,submenu,tag = taginfo
if ctx.db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
previous = None
tag = ''
index = 0
# If we get the same tag twice, we are at the oldest one
while not ctx.db.vers.exists(tag) and previous != tag:
previous = tag
tag = decode(ctx.script('get-latest', str(index))).rstrip('\n')
index += 1
return tag
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(ctx.script('get-type', version, path)).strip()
elif cmd == 'exist':
# Returns True if the requested file exists, otherwise returns False
version = args[0]
path = args[1]
dirname, filename = os.path.split(path)
entries = decode(ctx.script('get-dir', version, dirname)).split("\n")[:-1]
for entry in entries:
fname = entry.split(" ")[1]
if fname == filename:
return True
return False
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(ctx.script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
filename = os.path.basename(path)
family = lib.getFileFamily(filename)
if family != None:
buffer = BytesIO()
tokens = ctx.scriptLines('tokenize-file', version, path, family)
even = True
prefix = b''
if family == 'K':
prefix = b'CONFIG_'
for tok in tokens:
even = not even
tok2 = prefix + tok
if (even and ctx.db.defs.exists(tok2) and
(lib.compatibleFamily(ctx.db.defs.get(tok2).get_families(), family) or
lib.compatibleMacro(ctx.db.defs.get(tok2).get_macros(), family))):
tok = b'\033[31m' + tok2 + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(ctx.script('get-file', version, path))
elif cmd == 'family':
# Get the family of a given file
filename = args[0]
return lib.getFileFamily(filename)
elif cmd == 'dts-comp':
# Get state of dts_comp_support
return ctx.dts_comp_support
elif cmd == 'dts-comp-exists':
# Check if a dts compatible string exists
ident = args[0]
if ctx.dts_comp_support:
return ctx.db.comps.exists(ident)
else:
return False
elif cmd == 'keys':
# Return all keys of a given database
# /!\ This can take a while /!\
name = args[0]
if name == 'vars':
return ctx.db.vars.get_keys()
elif name == 'blob':
return ctx.db.blob.get_keys()
elif name == 'hash':
return ctx.db.hash.get_keys()
elif name == 'file':
return ctx.db.file.get_keys()
elif name == 'vers':
return ctx.db.vers.get_keys()
elif name == 'defs':
return ctx.db.defs.get_keys()
elif name == 'refs':
return ctx.db.refs.get_keys()
elif name == 'docs':
return ctx.db.docs.get_keys()
elif name == 'comps' and ctx.dts_comp_support:
return ctx.db.comps.get_keys()
elif name == 'comps_docs' and ctx.dts_comp_support:
return ctx.db.comps_docs.get_keys()
else:
return []
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
family = args[2]
# DT bindings compatible strings are handled differently
if family == 'B':
return get_idents_comps(version, ident, ctx=ctx)
else:
return get_idents_defs(version, ident, family, ctx=ctx)
else:
return 'Unknown subcommand: ' + cmd + '\n'
def get_idents_comps(version, ident, ctx=default_query):
# DT bindings compatible strings are handled differently
# They are defined in C files
# Used in DT files
# Documented in documentation files
symbol_c = []
symbol_dts = []
symbol_docs = []
# DT compatible strings are quoted in the database
ident = parse.quote(ident)
if not ctx.dts_comp_support or not ctx.db.comps.exists(ident):
return symbol_c, symbol_dts, symbol_docs
files_this_version = ctx.db.vers.get(version).iter()
comps = ctx.db.comps.get(ident).iter(dummy=True)
if ctx.db.comps_docs.exists(ident):
comps_docs = ctx.db.comps_docs.get(ident).iter(dummy=True)
else:
comps_docs = data.RefList().iter(dummy=True)
comps_idx, comps_lines, comps_family = next(comps)
comps_docs_idx, comps_docs_lines, comps_docs_family = next(comps_docs)
compsCBuf = [] # C/CPP/ASM files
compsDBuf = [] # DT files
compsBBuf = [] # DT bindings docs files
for file_idx, file_path in files_this_version:
while comps_idx < file_idx:
comps_idx, comps_lines, comps_family = next(comps)
while comps_docs_idx < file_idx:
comps_docs_idx, comps_docs_lines, comps_docs_family = next(comps_docs)
if comps_idx == file_idx:
if comps_family == 'C':
compsCBuf.append((file_path, comps_lines))
elif comps_family == 'D':
compsDBuf.append((file_path, comps_lines))
if comps_docs_idx == file_idx:
compsBBuf.append((file_path, comps_docs_lines))
for path, cline in sorted(compsCBuf):
symbol_c.append(SymbolInstance(path, cline, 'compatible'))
for path, dlines in sorted(compsDBuf):
symbol_dts.append(SymbolInstance(path, dlines))
for path, blines in sorted(compsBBuf):
symbol_docs.append(SymbolInstance(path, blines))
return symbol_c, symbol_dts, symbol_docs
def get_idents_defs(version, ident, family, ctx=default_query):
symbol_definitions = []
symbol_references = []
symbol_doccomments = []
if not ctx.db.defs.exists(ident):
return symbol_definitions, symbol_references, symbol_doccomments
if not ctx.db.vers.exists(version):
return symbol_definitions, symbol_references, symbol_doccomments
files_this_version = ctx.db.vers.get(version).iter()
this_ident = ctx.db.defs.get(ident)
defs_this_ident = this_ident.iter(dummy=True)
macros_this_ident = this_ident.get_macros()
# FIXME: see why we can have a discrepancy between defs_this_ident and refs
if ctx.db.refs.exists(ident):
refs = ctx.db.refs.get(ident).iter(dummy=True)
else:
refs = data.RefList().iter(dummy=True)
if ctx.db.docs.exists(ident):
docs = ctx.db.docs.get(ident).iter(dummy=True)
else:
docs = data.RefList().iter(dummy=True)
# vers, defs, refs, and docs are all populated by update.py in order of
# idx, and there is a one-to-one mapping between blob hashes and idx
# values. Therefore, we can sequentially step through the defs, refs,
# and docs for each file in a version.
def_idx, def_type, def_line, def_family = next(defs_this_ident)
ref_idx, ref_lines, ref_family = next(refs)
doc_idx, doc_line, doc_family = next(docs)
dBuf = []
rBuf = []
docBuf = []
for file_idx, file_path in files_this_version:
# Advance defs, refs, and docs to the current file
while def_idx < file_idx:
def_idx, def_type, def_line, def_family = next(defs_this_ident)
while ref_idx < file_idx:
ref_idx, ref_lines, ref_family = next(refs)
while doc_idx < file_idx:
doc_idx, doc_line, doc_family = next(docs)
# Copy information about this identifier into dBuf, rBuf, and docBuf.
while def_idx == file_idx:
if (def_family == family or family == 'A'
or lib.compatibleMacro(macros_this_ident, family)):
dBuf.append((file_path, def_type, def_line))
def_idx, def_type, def_line, def_family = next(defs_this_ident)
if ref_idx == file_idx:
if lib.compatibleFamily(family, ref_family) or family == 'A':
rBuf.append((file_path, ref_lines))
if doc_idx == file_idx: # TODO should this be a `while`?
docBuf.append((file_path, doc_line))
# Sort dBuf by path name before sorting by type in the loop
dBuf.sort()
for path, type, dline in sorted(dBuf, key=lambda d: d[1], reverse=True):
symbol_definitions.append(SymbolInstance(path, dline, type))
for path, rlines in sorted(rBuf):
symbol_references.append(SymbolInstance(path, rlines))
for path, docline in sorted(docBuf):
symbol_doccomments.append(SymbolInstance(path, docline))
return symbol_definitions, symbol_references, symbol_doccomments
def query(cmd, *args):
return default_query.query(cmd, *args)
def cmd_ident(version, ident, family, **kwargs):
symbol_definitions, symbol_references, symbol_doccomments = query("ident", version, ident, family)