From abbf08646af1c385fb7cfee83a235f49f43bd832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Bouillot?= Date: Sun, 5 Feb 2017 21:25:50 +0100 Subject: [PATCH] Split data module and add dummy element to iterators --- data.py | 167 +++++++++++++++++++++++++++++++++++++++++++++++++++ lib.py | 177 ------------------------------------------------------- query.py | 26 +++----- 3 files changed, 174 insertions(+), 196 deletions(-) create mode 100755 data.py diff --git a/data.py b/data.py new file mode 100755 index 0000000..990e6e8 --- /dev/null +++ b/data.py @@ -0,0 +1,167 @@ +#!/usr/bin/python3 + +import bsddb3 +from struct import pack, unpack +from binascii import hexlify, unhexlify +from io import BytesIO +import re +from lib import autoBytes + +def pack_hash (a): + a = a.encode ('ascii') + a = unhexlify (a) + a = pack ('20s', a) + return a + +def unpack_hash (a): + a = unpack ('20s', a) + a = a[0] + a = hexlify (a) + a = a.decode ('ascii') + return a + +def pack_int (a): + a = pack ('>L', a) + return a + +def unpack_int (a): + a = unpack ('>L', a) + a = a[0] + return a + +def append_reflist (a, idx, string): + idx = pack_int (idx) + a = a + idx + string + b'E' + return a + +################################################################################## + +defTypeD = { + 'define': 'd', + 'enum': 'e', + 'enumerator': 'E', + 'function': 'f', + 'label': 'l', + 'macro': 'M', + 'member': 'm', + 'struct': 's', + 'typedef': 't', + 'union': 'u', + 'variable': 'v' } + +defTypeR = { + 'd': 'define', + 'e': 'enum', + 'E': 'enumerator', + 'f': 'function', + 'l': 'label', + 'M': 'macro', + 'm': 'member', + 's': 'struct', + 't': 'typedef', + 'u': 'union', + 'v': 'variable' } + +################################################################################## + +maxId = 999999999 + +class DefList: + def __init__ (self, data): + self.data = data + + def iter (self, dummy=False): + for p in self.data.split (b','): + p = re.search (b'(\d*)(\w)(\d*)', p) + id, type, line = p.groups() + id = int (id) + type = defTypeR [type.decode()] + line = int (line) + yield (id, type, line) + if dummy: + yield (maxId, None, None) + +class PathList: + def __init__ (self, data): + self.data = data + + def iter (self, dummy=False): + for p in self.data.split (b'\n'): + p = re.search (b'(\d*)\t(.*)$', p) + id, path = p.groups() + id = int (id) + path = path.decode() + yield (id, path) + if dummy: + yield (maxId, None) + +from io import BytesIO + +class RefList: + def __init__ (self, data): + if type (data) is bytes: + self.data = data + else: + self.data = b'' + + def iter (self, dummy=False): + size = len (self.data) + s = BytesIO (self.data) + while s.tell() < size: + b = s.read (4) + b = unpack_int (b) + + # Reading byte by byte isn't optimal + t = BytesIO() + d = s.read (1) + while d != b'E': + t.write (d) + d = s.read (1) + c = t.getvalue() + c = c.decode() + t.close() + yield (b, c) + s.close() + if dummy: + yield (maxId, None) + +import os.path + +class DirDB: + def __init__ (self, dirname, contentType): + # FIXME: hardcoded path + self.path = 'databases/' + dirname + '/' + self.ctype = contentType + + def exists (self, key): + return os.path.isfile (self.path + 'v' + key) + + def get (self, key): + f = open (self.path + 'v' + key) + data = f.read() + data = data.encode() + f.close() + return self.ctype (data) + +class BsdDB: + def __init__ (self, filename, contentType): + self.db = bsddb3.db.DB() + # FIXME: hardcoded path + self.db.open ('databases/' + filename, flags=bsddb3.db.DB_RDONLY) + self.ctype = contentType + + def exists (self, key): + key = autoBytes (key) + return self.db.exists (key) + + def get (self, key): + key = autoBytes (key) + p = self.db.get (key) + p = self.ctype (p) + return p + +class DB: + def __init__ (self): + self.vers = DirDB ('versions', PathList) + self.defs = BsdDB ('definitions.db', DefList) + self.refs = BsdDB ('identrefs.db', RefList) diff --git a/lib.py b/lib.py index 6ea1f6b..b83ea46 100755 --- a/lib.py +++ b/lib.py @@ -1,7 +1,6 @@ #!/usr/bin/python3 import subprocess -import bsddb3 import re def echo (bstr): @@ -44,179 +43,3 @@ def autoBytes (arg): if type (arg) is str: arg = arg.encode() return arg - -########################################################################## - -from struct import pack, unpack -from binascii import hexlify, unhexlify -from io import BytesIO - -def pack_hash (a): - a = a.encode ('ascii') - a = unhexlify (a) - a = pack ('20s', a) - return a - -def unpack_hash (a): - a = unpack ('20s', a) - a = a[0] - a = hexlify (a) - a = a.decode ('ascii') - return a - -def pack_int (a): - a = pack ('>L', a) - return a - -def unpack_int (a): - a = unpack ('>L', a) - a = a[0] - return a - -def append_reflist (a, idx, string): - idx = pack_int (idx) - a = a + idx + string + b'E' - return a - -#def unpack_reflist (a): -# size = len (a) -# s = BytesIO (a) -# res = [] -# while s.tell() < size: -# b = s.read (4) -# b = unpack_int (b) -# -# # Reading byte by byte isn't optimal -# t = BytesIO() -# d = s.read (1) -# while d != b'E': -# t.write (d) -# d = s.read (1) -# c = t.getvalue() -# t.close() -# -# res.append ((b,c)) -# s.close() -# return res - -################################################################################## - -defTypeD = { - 'define': 'd', - 'enum': 'e', - 'enumerator': 'E', - 'function': 'f', - 'label': 'l', - 'macro': 'M', - 'member': 'm', - 'struct': 's', - 'typedef': 't', - 'union': 'u', - 'variable': 'v' } - -defTypeR = { - 'd': 'define', - 'e': 'enum', - 'E': 'enumerator', - 'f': 'function', - 'l': 'label', - 'M': 'macro', - 'm': 'member', - 's': 'struct', - 't': 'typedef', - 'u': 'union', - 'v': 'variable' } - -################################################################################## - -class DefList: - def __init__ (self, data): - self.data = data - - def iter (self): - for p in self.data.split (b','): - p = re.search (b'(\d*)(\w)(\d*)', p) - id, type, line = p.groups() - id = int (id) - type = defTypeR [type.decode()] - line = int (line) - yield (id, type, line) - -class PathList: - def __init__ (self, data): - self.data = data - - def iter (self): - for p in self.data.split (b'\n'): - p = re.search (b'(\d*)\t(.*)$', p) - id, path = p.groups() - id = int (id) - path = path.decode() - yield (id, path) - -from io import BytesIO - -class RefList: - def __init__ (self, data): - self.data = data - - def iter (self): - size = len (self.data) - s = BytesIO (self.data) - while s.tell() < size: - b = s.read (4) - b = unpack_int (b) - - # Reading byte by byte isn't optimal - t = BytesIO() - d = s.read (1) - while d != b'E': - t.write (d) - d = s.read (1) - c = t.getvalue() - c = c.decode() - t.close() - yield (b, c) - s.close() - -import os.path - -class DirDB: - def __init__ (self, dirname, contentType): - # FIXME: hardcoded path - self.path = 'databases/' + dirname + '/' - self.ctype = contentType - - def exists (self, key): - return os.path.isfile (self.path + 'v' + key) - - def get (self, key): - f = open (self.path + 'v' + key) - data = f.read() - data = data.encode() - f.close() - return self.ctype (data) - -class BsdDB: - def __init__ (self, filename, contentType): - self.db = bsddb3.db.DB() - # FIXME: hardcoded path - self.db.open ('databases/' + filename, flags=bsddb3.db.DB_RDONLY) - self.ctype = contentType - - def exists (self, key): - key = autoBytes (key) - return self.db.exists (key) - - def get (self, key): - key = autoBytes (key) - p = self.db.get (key) - p = self.ctype (p) - return p - -class DB: - def __init__ (self): - self.vers = DirDB ('versions', PathList) - self.defs = BsdDB ('definitions.db', DefList) - self.refs = BsdDB ('identrefs.db', RefList) - diff --git a/query.py b/query.py index fe8f1c5..6c27e3f 100755 --- a/query.py +++ b/query.py @@ -3,8 +3,9 @@ from sys import argv from lib import echo, script, scriptLines import lib +import data -db = lib.DB() +db = data.DB() cmd = argv[1] @@ -46,8 +47,8 @@ elif cmd == 'ident': exit() vers = db.vers.get (version).iter() - defs = db.defs.get (ident).iter() - refs = db.refs.get (ident).iter() + defs = db.defs.get (ident).iter (dummy=True) + refs = db.refs.get (ident).iter (dummy=True) id2, type, dline = next (defs) id3, rlines = next (refs) @@ -55,24 +56,13 @@ elif cmd == 'ident': dBuf = [] rBuf = [] - maxId = 999999999 - for id1, path in vers: while id1 > id2: - try: - id2, type, dline = next (defs) - except StopIteration: - id2 = maxId - + id2, type, dline = next (defs) while id1 > id3: - try: - id3, rlines = next (refs) - except StopIteration: - id3 = maxId - + id3, rlines = next (refs) if id1 == id2: dBuf.append ((path, type, dline)) - if id1 == id3: rBuf.append ((path, rlines)) @@ -80,9 +70,7 @@ elif cmd == 'ident': for path, type, dline in sorted (dBuf): print (path + ': ' + str (dline) + ' (' + type + ')') - print() - - print ('Referenced in', len (rBuf), 'files:') + print ('\nReferenced in', len (rBuf), 'files:') for path, rlines in sorted (rBuf): print (path + ': ' + rlines)