Split data module and add dummy element to iterators
This commit is contained in:
Mikaël Bouillot
parent
a479750bee
commit
abbf08646a
3 changed files with 174 additions and 196 deletions
167
data.py
Executable file
167
data.py
Executable file
|
|
@ -0,0 +1,167 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import bsddb3
|
||||
from struct import pack, unpack
|
||||
from binascii import hexlify, unhexlify
|
||||
from io import BytesIO
|
||||
import re
|
||||
from lib import autoBytes
|
||||
|
||||
def pack_hash (a):
|
||||
a = a.encode ('ascii')
|
||||
a = unhexlify (a)
|
||||
a = pack ('20s', a)
|
||||
return a
|
||||
|
||||
def unpack_hash (a):
|
||||
a = unpack ('20s', a)
|
||||
a = a[0]
|
||||
a = hexlify (a)
|
||||
a = a.decode ('ascii')
|
||||
return a
|
||||
|
||||
def pack_int (a):
|
||||
a = pack ('>L', a)
|
||||
return a
|
||||
|
||||
def unpack_int (a):
|
||||
a = unpack ('>L', a)
|
||||
a = a[0]
|
||||
return a
|
||||
|
||||
def append_reflist (a, idx, string):
|
||||
idx = pack_int (idx)
|
||||
a = a + idx + string + b'E'
|
||||
return a
|
||||
|
||||
##################################################################################
|
||||
|
||||
defTypeD = {
|
||||
'define': 'd',
|
||||
'enum': 'e',
|
||||
'enumerator': 'E',
|
||||
'function': 'f',
|
||||
'label': 'l',
|
||||
'macro': 'M',
|
||||
'member': 'm',
|
||||
'struct': 's',
|
||||
'typedef': 't',
|
||||
'union': 'u',
|
||||
'variable': 'v' }
|
||||
|
||||
defTypeR = {
|
||||
'd': 'define',
|
||||
'e': 'enum',
|
||||
'E': 'enumerator',
|
||||
'f': 'function',
|
||||
'l': 'label',
|
||||
'M': 'macro',
|
||||
'm': 'member',
|
||||
's': 'struct',
|
||||
't': 'typedef',
|
||||
'u': 'union',
|
||||
'v': 'variable' }
|
||||
|
||||
##################################################################################
|
||||
|
||||
maxId = 999999999
|
||||
|
||||
class DefList:
|
||||
def __init__ (self, data):
|
||||
self.data = data
|
||||
|
||||
def iter (self, dummy=False):
|
||||
for p in self.data.split (b','):
|
||||
p = re.search (b'(\d*)(\w)(\d*)', p)
|
||||
id, type, line = p.groups()
|
||||
id = int (id)
|
||||
type = defTypeR [type.decode()]
|
||||
line = int (line)
|
||||
yield (id, type, line)
|
||||
if dummy:
|
||||
yield (maxId, None, None)
|
||||
|
||||
class PathList:
|
||||
def __init__ (self, data):
|
||||
self.data = data
|
||||
|
||||
def iter (self, dummy=False):
|
||||
for p in self.data.split (b'\n'):
|
||||
p = re.search (b'(\d*)\t(.*)$', p)
|
||||
id, path = p.groups()
|
||||
id = int (id)
|
||||
path = path.decode()
|
||||
yield (id, path)
|
||||
if dummy:
|
||||
yield (maxId, None)
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
class RefList:
|
||||
def __init__ (self, data):
|
||||
if type (data) is bytes:
|
||||
self.data = data
|
||||
else:
|
||||
self.data = b''
|
||||
|
||||
def iter (self, dummy=False):
|
||||
size = len (self.data)
|
||||
s = BytesIO (self.data)
|
||||
while s.tell() < size:
|
||||
b = s.read (4)
|
||||
b = unpack_int (b)
|
||||
|
||||
# Reading byte by byte isn't optimal
|
||||
t = BytesIO()
|
||||
d = s.read (1)
|
||||
while d != b'E':
|
||||
t.write (d)
|
||||
d = s.read (1)
|
||||
c = t.getvalue()
|
||||
c = c.decode()
|
||||
t.close()
|
||||
yield (b, c)
|
||||
s.close()
|
||||
if dummy:
|
||||
yield (maxId, None)
|
||||
|
||||
import os.path
|
||||
|
||||
class DirDB:
|
||||
def __init__ (self, dirname, contentType):
|
||||
# FIXME: hardcoded path
|
||||
self.path = 'databases/' + dirname + '/'
|
||||
self.ctype = contentType
|
||||
|
||||
def exists (self, key):
|
||||
return os.path.isfile (self.path + 'v' + key)
|
||||
|
||||
def get (self, key):
|
||||
f = open (self.path + 'v' + key)
|
||||
data = f.read()
|
||||
data = data.encode()
|
||||
f.close()
|
||||
return self.ctype (data)
|
||||
|
||||
class BsdDB:
|
||||
def __init__ (self, filename, contentType):
|
||||
self.db = bsddb3.db.DB()
|
||||
# FIXME: hardcoded path
|
||||
self.db.open ('databases/' + filename, flags=bsddb3.db.DB_RDONLY)
|
||||
self.ctype = contentType
|
||||
|
||||
def exists (self, key):
|
||||
key = autoBytes (key)
|
||||
return self.db.exists (key)
|
||||
|
||||
def get (self, key):
|
||||
key = autoBytes (key)
|
||||
p = self.db.get (key)
|
||||
p = self.ctype (p)
|
||||
return p
|
||||
|
||||
class DB:
|
||||
def __init__ (self):
|
||||
self.vers = DirDB ('versions', PathList)
|
||||
self.defs = BsdDB ('definitions.db', DefList)
|
||||
self.refs = BsdDB ('identrefs.db', RefList)
|
||||
177
lib.py
177
lib.py
|
|
@ -1,7 +1,6 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import subprocess
|
||||
import bsddb3
|
||||
import re
|
||||
|
||||
def echo (bstr):
|
||||
|
|
@ -44,179 +43,3 @@ def autoBytes (arg):
|
|||
if type (arg) is str:
|
||||
arg = arg.encode()
|
||||
return arg
|
||||
|
||||
##########################################################################
|
||||
|
||||
from struct import pack, unpack
|
||||
from binascii import hexlify, unhexlify
|
||||
from io import BytesIO
|
||||
|
||||
def pack_hash (a):
|
||||
a = a.encode ('ascii')
|
||||
a = unhexlify (a)
|
||||
a = pack ('20s', a)
|
||||
return a
|
||||
|
||||
def unpack_hash (a):
|
||||
a = unpack ('20s', a)
|
||||
a = a[0]
|
||||
a = hexlify (a)
|
||||
a = a.decode ('ascii')
|
||||
return a
|
||||
|
||||
def pack_int (a):
|
||||
a = pack ('>L', a)
|
||||
return a
|
||||
|
||||
def unpack_int (a):
|
||||
a = unpack ('>L', a)
|
||||
a = a[0]
|
||||
return a
|
||||
|
||||
def append_reflist (a, idx, string):
|
||||
idx = pack_int (idx)
|
||||
a = a + idx + string + b'E'
|
||||
return a
|
||||
|
||||
#def unpack_reflist (a):
|
||||
# size = len (a)
|
||||
# s = BytesIO (a)
|
||||
# res = []
|
||||
# while s.tell() < size:
|
||||
# b = s.read (4)
|
||||
# b = unpack_int (b)
|
||||
#
|
||||
# # Reading byte by byte isn't optimal
|
||||
# t = BytesIO()
|
||||
# d = s.read (1)
|
||||
# while d != b'E':
|
||||
# t.write (d)
|
||||
# d = s.read (1)
|
||||
# c = t.getvalue()
|
||||
# t.close()
|
||||
#
|
||||
# res.append ((b,c))
|
||||
# s.close()
|
||||
# return res
|
||||
|
||||
##################################################################################
|
||||
|
||||
defTypeD = {
|
||||
'define': 'd',
|
||||
'enum': 'e',
|
||||
'enumerator': 'E',
|
||||
'function': 'f',
|
||||
'label': 'l',
|
||||
'macro': 'M',
|
||||
'member': 'm',
|
||||
'struct': 's',
|
||||
'typedef': 't',
|
||||
'union': 'u',
|
||||
'variable': 'v' }
|
||||
|
||||
defTypeR = {
|
||||
'd': 'define',
|
||||
'e': 'enum',
|
||||
'E': 'enumerator',
|
||||
'f': 'function',
|
||||
'l': 'label',
|
||||
'M': 'macro',
|
||||
'm': 'member',
|
||||
's': 'struct',
|
||||
't': 'typedef',
|
||||
'u': 'union',
|
||||
'v': 'variable' }
|
||||
|
||||
##################################################################################
|
||||
|
||||
class DefList:
|
||||
def __init__ (self, data):
|
||||
self.data = data
|
||||
|
||||
def iter (self):
|
||||
for p in self.data.split (b','):
|
||||
p = re.search (b'(\d*)(\w)(\d*)', p)
|
||||
id, type, line = p.groups()
|
||||
id = int (id)
|
||||
type = defTypeR [type.decode()]
|
||||
line = int (line)
|
||||
yield (id, type, line)
|
||||
|
||||
class PathList:
|
||||
def __init__ (self, data):
|
||||
self.data = data
|
||||
|
||||
def iter (self):
|
||||
for p in self.data.split (b'\n'):
|
||||
p = re.search (b'(\d*)\t(.*)$', p)
|
||||
id, path = p.groups()
|
||||
id = int (id)
|
||||
path = path.decode()
|
||||
yield (id, path)
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
class RefList:
|
||||
def __init__ (self, data):
|
||||
self.data = data
|
||||
|
||||
def iter (self):
|
||||
size = len (self.data)
|
||||
s = BytesIO (self.data)
|
||||
while s.tell() < size:
|
||||
b = s.read (4)
|
||||
b = unpack_int (b)
|
||||
|
||||
# Reading byte by byte isn't optimal
|
||||
t = BytesIO()
|
||||
d = s.read (1)
|
||||
while d != b'E':
|
||||
t.write (d)
|
||||
d = s.read (1)
|
||||
c = t.getvalue()
|
||||
c = c.decode()
|
||||
t.close()
|
||||
yield (b, c)
|
||||
s.close()
|
||||
|
||||
import os.path
|
||||
|
||||
class DirDB:
|
||||
def __init__ (self, dirname, contentType):
|
||||
# FIXME: hardcoded path
|
||||
self.path = 'databases/' + dirname + '/'
|
||||
self.ctype = contentType
|
||||
|
||||
def exists (self, key):
|
||||
return os.path.isfile (self.path + 'v' + key)
|
||||
|
||||
def get (self, key):
|
||||
f = open (self.path + 'v' + key)
|
||||
data = f.read()
|
||||
data = data.encode()
|
||||
f.close()
|
||||
return self.ctype (data)
|
||||
|
||||
class BsdDB:
|
||||
def __init__ (self, filename, contentType):
|
||||
self.db = bsddb3.db.DB()
|
||||
# FIXME: hardcoded path
|
||||
self.db.open ('databases/' + filename, flags=bsddb3.db.DB_RDONLY)
|
||||
self.ctype = contentType
|
||||
|
||||
def exists (self, key):
|
||||
key = autoBytes (key)
|
||||
return self.db.exists (key)
|
||||
|
||||
def get (self, key):
|
||||
key = autoBytes (key)
|
||||
p = self.db.get (key)
|
||||
p = self.ctype (p)
|
||||
return p
|
||||
|
||||
class DB:
|
||||
def __init__ (self):
|
||||
self.vers = DirDB ('versions', PathList)
|
||||
self.defs = BsdDB ('definitions.db', DefList)
|
||||
self.refs = BsdDB ('identrefs.db', RefList)
|
||||
|
||||
|
|
|
|||
26
query.py
26
query.py
|
|
@ -3,8 +3,9 @@
|
|||
from sys import argv
|
||||
from lib import echo, script, scriptLines
|
||||
import lib
|
||||
import data
|
||||
|
||||
db = lib.DB()
|
||||
db = data.DB()
|
||||
|
||||
cmd = argv[1]
|
||||
|
||||
|
|
@ -46,8 +47,8 @@ elif cmd == 'ident':
|
|||
exit()
|
||||
|
||||
vers = db.vers.get (version).iter()
|
||||
defs = db.defs.get (ident).iter()
|
||||
refs = db.refs.get (ident).iter()
|
||||
defs = db.defs.get (ident).iter (dummy=True)
|
||||
refs = db.refs.get (ident).iter (dummy=True)
|
||||
|
||||
id2, type, dline = next (defs)
|
||||
id3, rlines = next (refs)
|
||||
|
|
@ -55,24 +56,13 @@ elif cmd == 'ident':
|
|||
dBuf = []
|
||||
rBuf = []
|
||||
|
||||
maxId = 999999999
|
||||
|
||||
for id1, path in vers:
|
||||
while id1 > id2:
|
||||
try:
|
||||
id2, type, dline = next (defs)
|
||||
except StopIteration:
|
||||
id2 = maxId
|
||||
|
||||
id2, type, dline = next (defs)
|
||||
while id1 > id3:
|
||||
try:
|
||||
id3, rlines = next (refs)
|
||||
except StopIteration:
|
||||
id3 = maxId
|
||||
|
||||
id3, rlines = next (refs)
|
||||
if id1 == id2:
|
||||
dBuf.append ((path, type, dline))
|
||||
|
||||
if id1 == id3:
|
||||
rBuf.append ((path, rlines))
|
||||
|
||||
|
|
@ -80,9 +70,7 @@ elif cmd == 'ident':
|
|||
for path, type, dline in sorted (dBuf):
|
||||
print (path + ': ' + str (dline) + ' (' + type + ')')
|
||||
|
||||
print()
|
||||
|
||||
print ('Referenced in', len (rBuf), 'files:')
|
||||
print ('\nReferenced in', len (rBuf), 'files:')
|
||||
for path, rlines in sorted (rBuf):
|
||||
print (path + ': ' + rlines)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue