portugalfuturista/elixir
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Database update script, new data format

Browse source
This commit is contained in:
Mikaël Bouillot 2017-02-06 15:15:51 +01:00
parent 3f0b68a0d7
commit 0bf34d7751
5 changed files with 190 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

116
data.py
View file

@ -1,40 +1,11 @@
#!/usr/bin/python3
import bsddb3
from struct import pack, unpack
from binascii import hexlify, unhexlify
from io import BytesIO
import re
from lib import autoBytes
import os.path
def pack_hash (a):
a = a.encode ('ascii')
a = unhexlify (a)
a = pack ('20s', a)
return a
def unpack_hash (a):
a = unpack ('20s', a)
a = a[0]
a = hexlify (a)
a = a.decode ('ascii')
return a
def pack_int (a):
a = pack ('>L', a)
return a
def unpack_int (a):
a = unpack ('>L', a)
a = a[0]
return a
def append_reflist (a, idx, string):
idx = pack_int (idx)
a = a + idx + string + b'E'
return a
##################################################################################
defTypeR = {
@ -57,7 +28,7 @@ defTypeD = {v: k for k, v in defTypeR.items()}
maxId = 999999999
class DefList:
def __init__ (self, data):
def __init__ (self, data=b''):
self.data = data
def iter (self, dummy=False):
@ -71,72 +42,72 @@ class DefList:
if dummy:
yield (maxId, None, None)
def append (self, id, type, line):
if type not in defTypeD:
return
p = str(id) + defTypeD[type] + str(line)
if self.data != b'':
p = ',' + p
self.data += p.encode()
def pack (self):
return self.data
class PathList:
def __init__ (self, data):
def __init__ (self, data=b''):
self.data = data
def iter (self, dummy=False):
for p in self.data.split (b'\n'):
p = re.search (b'(\d*)\t(.*)$', p)
id, path = p.groups()
if (p == b''): continue
id, path = p.split (b' ')
id = int (id)
path = path.decode()
yield (id, path)
if dummy:
yield (maxId, None)
def append (self, id, path):
p = str(id).encode() + b' ' + path
self.data = self.data + p + b'\n'
def pack (self):
return self.data
from io import BytesIO
class RefList:
def __init__ (self, data):
if type (data) is bytes:
self.data = data
else:
self.data = b''
def __init__ (self, data=b''):
self.data = data
def iter (self, dummy=False):
size = len (self.data)
s = BytesIO (self.data)
while s.tell() < size:
b = s.read (4)
b = unpack_int (b)
# Reading byte by byte isn't optimal
t = BytesIO()
d = s.read (1)
while d != b'E':
t.write (d)
d = s.read (1)
c = t.getvalue()
line = s.readline()
line = line [:-1]
b,c = line.split (b':')
b = int (b.decode())
c = c.decode()
t.close()
yield (b, c)
s.close()
if dummy:
yield (maxId, None)
import os.path
def append (self, id, lines):
p = str(id) + ':' + lines + '\n'
self.data += p.encode()
class DirDB:
def __init__ (self, dirname, contentType):
self.path = dirname + '/'
self.ctype = contentType
def exists (self, key):
return os.path.isfile (self.path + 'v' + key)
def get (self, key):
f = open (self.path + 'v' + key)
data = f.read()
data = data.encode()
f.close()
return self.ctype (data)
def pack (self):
return self.data
class BsdDB:
def __init__ (self, filename, contentType):
self.filename = filename
self.db = bsddb3.db.DB()
self.db.open (filename, flags=bsddb3.db.DB_RDONLY)
self.db.open (filename,
flags=bsddb3.db.DB_CREATE, # FIXME: handle locks
dbtype=bsddb3.db.DB_BTREE)
self.ctype = contentType
def exists (self, key):
@ -149,6 +120,13 @@ class BsdDB:
p = self.ctype (p)
return p
def put (self, key, val):
key = autoBytes (key)
val = autoBytes (val)
if type (val) is not bytes:
val = val.pack()
self.db.put (key, val)
class DB:
def __init__ (self, dir):
if os.path.isdir (dir):
@ -156,6 +134,10 @@ class DB:
else:
raise FileNotFoundError
self.vers = DirDB (dir + '/versions', PathList)
self.vars = BsdDB (dir + '/variables.db', lambda x: int (x.decode()) )
self.blob = BsdDB (dir + '/blobs.db', lambda x: int (x.decode()) )
self.hash = BsdDB (dir + '/hashes.db', lambda x: x )
self.file = BsdDB (dir + '/filenames.db', lambda x: x.decode() )
self.vers = BsdDB (dir + '/versions.db', PathList)
self.defs = BsdDB (dir + '/definitions.db', DefList)
self.refs = BsdDB (dir + '/identrefs.db', RefList)
self.refs = BsdDB (dir + '/references.db', RefList)

6
lib.py
View file

@ -32,7 +32,9 @@ def unescape (bstr):
return bstr
def isIdent (bstr):
if re.search (b'_', bstr):
if len (bstr) < 3:
return False
elif re.search (b'_', bstr):
return True
elif re.search (b'^[A-Z0-9]*$', bstr):
return True
@ -42,4 +44,6 @@ def isIdent (bstr):
def autoBytes (arg):
if type (arg) is str:
arg = arg.encode()
elif type (arg) is int:
arg = str(arg).encode()
return arg

6
query.py
View file

@ -33,10 +33,10 @@ elif cmd == 'file':
if ext == '.c' or ext == '.h':
tokens = scriptLines ('tokenize-file', version, path)
toBe = True
even = True
for tok in tokens:
toBe = not toBe
if toBe and db.defs.exists (tok) and lib.isIdent (tok):
even = not even
if even and db.defs.exists (tok) and lib.isIdent (tok):
tok = b'\033[31m' + tok + b'\033[0m'
else:
tok = lib.unescape (tok)

10
script.sh
View file

@ -15,6 +15,8 @@ shift
case $cmd in
list-tags)
git tag |
head -n 10 |
sed 's/^v//' |
sed 's/$/.0/' |
sort -V |
sed 's/\.0$//'
@ -51,7 +53,13 @@ case $cmd in
;;
tokenize-file)
git cat-file blob v$1:$2 |
if [ "$1" = -b ]; then
ref=$2
else
ref=v$1:$2
fi
git cat-file blob $ref |
tr '\n<>' '\1\2\3' |
sed 's/\/\*/</g' |
sed 's/\*\//>/g' |

124
update.py Executable file
View file

@ -0,0 +1,124 @@
#!/usr/bin/python3
from sys import argv
from lib import echo, script, scriptLines
import lib
import data
import os
from data import PathList
try:
dbDir = os.environ['LXR_DATA_DIR']
except KeyError:
print (argv[0] + ': LXR_DATA_DIR needs to be set')
exit (1)
db = data.DB (dbDir)
def updateBlobIDs (tag):
if db.vars.exists ('numBlobs'):
idx = db.vars.get ('numBlobs')
else:
idx = 0
blobs = scriptLines ('list-blobs', '-f', tag)
newBlobs = []
for blob in blobs:
hash, filename = blob.split (b' ')
if not db.blob.exists (hash):
db.blob.put (hash, idx)
db.hash.put (idx, hash)
db.file.put (idx, filename)
newBlobs.append (idx)
idx += 1
db.vars.put ('numBlobs', idx)
return newBlobs
def updateVersions (tag):
blobs = scriptLines ('list-blobs', '-p', tag)
buf = []
for blob in blobs:
hash, path = blob.split (b' ')
idx = db.blob.get (hash)
buf.append ((idx, path))
buf = sorted (buf)
obj = PathList()
for idx, path in buf:
obj.append (idx, path)
db.vers.put (tag, obj)
def updateDefinitions (blobs):
for blob in blobs:
if (blob % 100 == 0): print ('D:', blob)
hash = db.hash.get (blob)
filename = db.file.get (blob)
ext = filename[-2:]
if not (ext == '.c' or ext == '.h'): continue
lines = scriptLines ('parse-defs', hash, filename)
for l in lines:
ident, type, line = l.split (b' ')
type = type.decode()
line = int (line.decode())
if db.defs.exists (ident):
obj = db.defs.get (ident)
else:
obj = data.DefList()
obj.append (blob, type, line)
db.defs.put (ident, obj)
def updateReferences (blobs):
for blob in blobs:
if (blob % 100 == 0): print ('R:', blob)
hash = db.hash.get (blob)
filename = db.file.get (blob)
ext = filename[-2:]
if not (ext == '.c' or ext == '.h'): continue
tokens = scriptLines ('tokenize-file', '-b', hash)
even = True
lineNum = 1
idents = {}
for tok in tokens:
even = not even
if even:
if db.defs.exists (tok) and lib.isIdent (tok):
if tok in idents:
idents[tok] += ',' + str(lineNum)
else:
idents[tok] = str(lineNum)
else:
lineNum += tok.count (b'\1')
for ident, lines in idents.items():
if db.refs.exists (ident):
obj = db.refs.get (ident)
else:
obj = data.RefList()
obj.append (blob, lines)
db.refs.put (ident, obj)
# Main
tagBuf = []
for tag in scriptLines ('list-tags'):
if not db.vers.exists (tag):
tagBuf.append (tag)
print ('Found ' + str(len(tagBuf)) + ' new tags')
for tag in tagBuf:
print (tag.decode(), end=': ')
newBlobs = updateBlobIDs (tag)
print (str(len(newBlobs)) + ' new blobs')
updateVersions (tag)
updateDefinitions (newBlobs)
updateReferences (newBlobs)