update.py: Use 7 threads and reduce locking
Use 7 threads to do more things at once and spread the load Lock file before the loop so we don't lock and unlock all the time This will hopefully reduce database creation time Signed-off-by: Maxime Chretien <maxime.chretien@bootlin.com>
This commit is contained in:
Maxime Chretien
parent
b487a73d25
commit
9ff3cff564
1 changed files with 110 additions and 73 deletions
183
update.py
183
update.py
|
|
@ -35,12 +35,21 @@ db = data.DB(lib.getDataDir(), readonly=False)
|
||||||
|
|
||||||
hash_file_lock = Lock() #Lock for db.hash and db.file
|
hash_file_lock = Lock() #Lock for db.hash and db.file
|
||||||
defs_lock = Lock() #Lock for db.defs
|
defs_lock = Lock() #Lock for db.defs
|
||||||
|
refs_lock = Lock() #Lock for db.refs
|
||||||
|
docs_lock = Lock() #Lock for db.docs
|
||||||
tag_ready = Condition() #Waiting for new tags
|
tag_ready = Condition() #Waiting for new tags
|
||||||
|
|
||||||
new_idxes = [] # (new idxes, Event idxes ready, Event defs ready)
|
new_idxes = [] # (new idxes, Event idxes ready, Event defs ready)
|
||||||
|
|
||||||
tags_done = False #True if all tags have been added to new_idxes
|
tags_done = False #True if all tags have been added to new_idxes
|
||||||
|
|
||||||
|
#Progress variables
|
||||||
|
tags_defs = 0
|
||||||
|
tags_defs_lock = Lock()
|
||||||
|
tags_refs = 0
|
||||||
|
tags_refs_lock = Lock()
|
||||||
|
tags_docs = 0
|
||||||
|
tags_docs_lock = Lock()
|
||||||
|
|
||||||
class UpdateIdVersion(Thread):
|
class UpdateIdVersion(Thread):
|
||||||
def __init__(self, tag_buf):
|
def __init__(self, tag_buf):
|
||||||
|
|
@ -120,16 +129,18 @@ class UpdateIdVersion(Thread):
|
||||||
|
|
||||||
|
|
||||||
class UpdateDefs(Thread):
|
class UpdateDefs(Thread):
|
||||||
def __init__(self):
|
def __init__(self, odd):
|
||||||
Thread.__init__(self, name="UpdateDefsElixir")
|
Thread.__init__(self, name="UpdateDefsElixir")
|
||||||
|
if odd:
|
||||||
|
self.index = 1
|
||||||
|
else:
|
||||||
|
self.index = 0
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
global new_idxes, tags_done, tag_ready
|
global new_idxes, tags_done, tag_ready, tags_defs, tags_defs_lock
|
||||||
|
|
||||||
self.index = 0
|
while(not (tags_done and self.index >= len(new_idxes))):
|
||||||
|
if(self.index >= len(new_idxes)):
|
||||||
while(not (tags_done and self.index == len(new_idxes))):
|
|
||||||
if(self.index == len(new_idxes)):
|
|
||||||
#Wait for new tags
|
#Wait for new tags
|
||||||
with tag_ready:
|
with tag_ready:
|
||||||
tag_ready.wait()
|
tag_ready.wait()
|
||||||
|
|
@ -137,18 +148,21 @@ class UpdateDefs(Thread):
|
||||||
|
|
||||||
new_idxes[self.index][1].wait() #Make sure the tag is ready
|
new_idxes[self.index][1].wait() #Make sure the tag is ready
|
||||||
|
|
||||||
|
with tags_defs_lock:
|
||||||
|
tags_defs += 1
|
||||||
|
|
||||||
self.update_definitions(new_idxes[self.index][0])
|
self.update_definitions(new_idxes[self.index][0])
|
||||||
|
|
||||||
new_idxes[self.index][2].set() #Tell that UpdateDefs processed the tag
|
new_idxes[self.index][2].set() #Tell that UpdateDefs processed the tag
|
||||||
|
|
||||||
self.index += 1
|
self.index += 2
|
||||||
|
|
||||||
|
|
||||||
def update_definitions(self, idxes):
|
def update_definitions(self, idxes):
|
||||||
global hash_file_lock, defs_lock
|
global hash_file_lock, defs_lock, tags_defs
|
||||||
|
|
||||||
for idx in idxes:
|
for idx in idxes:
|
||||||
if (idx % 1000 == 0): progress('defs: ' + str(idx), self.index+1)
|
if (idx % 1000 == 0): progress('defs: ' + str(idx), tags_defs)
|
||||||
|
|
||||||
with hash_file_lock:
|
with hash_file_lock:
|
||||||
hash = db.hash.get(idx)
|
hash = db.hash.get(idx)
|
||||||
|
|
@ -158,36 +172,38 @@ class UpdateDefs(Thread):
|
||||||
if family == None: continue
|
if family == None: continue
|
||||||
|
|
||||||
lines = scriptLines('parse-defs', hash, filename, family)
|
lines = scriptLines('parse-defs', hash, filename, family)
|
||||||
for l in lines:
|
|
||||||
ident, type, line = l.split(b' ')
|
|
||||||
type = type.decode()
|
|
||||||
line = int(line.decode())
|
|
||||||
|
|
||||||
with defs_lock:
|
with defs_lock:
|
||||||
|
for l in lines:
|
||||||
|
ident, type, line = l.split(b' ')
|
||||||
|
type = type.decode()
|
||||||
|
line = int(line.decode())
|
||||||
|
|
||||||
if db.defs.exists(ident):
|
if db.defs.exists(ident):
|
||||||
obj = db.defs.get(ident)
|
obj = db.defs.get(ident)
|
||||||
else:
|
else:
|
||||||
obj = data.DefList()
|
obj = data.DefList()
|
||||||
|
|
||||||
obj.add_family(family)
|
obj.add_family(family)
|
||||||
obj.append(idx, type, line, family)
|
obj.append(idx, type, line, family)
|
||||||
if verbose:
|
if verbose:
|
||||||
print(f"def {type} {ident} in #{idx} @ {line}")
|
print(f"def {type} {ident} in #{idx} @ {line}")
|
||||||
with defs_lock:
|
|
||||||
db.defs.put(ident, obj)
|
db.defs.put(ident, obj)
|
||||||
|
|
||||||
|
|
||||||
class UpdateRefs(Thread):
|
class UpdateRefs(Thread):
|
||||||
def __init__(self):
|
def __init__(self, odd):
|
||||||
Thread.__init__(self, name="UpdateRefsElixir")
|
Thread.__init__(self, name="UpdateRefsElixir")
|
||||||
|
if odd:
|
||||||
|
self.index = 1
|
||||||
|
else:
|
||||||
|
self.index = 0
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
global new_idxes, tags_done
|
global new_idxes, tags_done, tags_refs, tags_refs_lock
|
||||||
|
|
||||||
self.index = 0
|
while(not (tags_done and self.index >= len(new_idxes))):
|
||||||
|
if(self.index >= len(new_idxes)):
|
||||||
while(not (tags_done and self.index == len(new_idxes))):
|
|
||||||
if(self.index == len(new_idxes)):
|
|
||||||
#Wait for new tags
|
#Wait for new tags
|
||||||
with tag_ready:
|
with tag_ready:
|
||||||
tag_ready.wait()
|
tag_ready.wait()
|
||||||
|
|
@ -196,15 +212,18 @@ class UpdateRefs(Thread):
|
||||||
new_idxes[self.index][1].wait() #Make sure the tag is ready
|
new_idxes[self.index][1].wait() #Make sure the tag is ready
|
||||||
new_idxes[self.index][2].wait() #Make sure UpdateDefs processed the tag
|
new_idxes[self.index][2].wait() #Make sure UpdateDefs processed the tag
|
||||||
|
|
||||||
|
with tags_refs_lock:
|
||||||
|
tags_refs += 1
|
||||||
|
|
||||||
self.update_references(new_idxes[self.index][0])
|
self.update_references(new_idxes[self.index][0])
|
||||||
|
|
||||||
self.index += 1
|
self.index += 2
|
||||||
|
|
||||||
def update_references(self, idxes):
|
def update_references(self, idxes):
|
||||||
global hash_file_lock, defs_lock
|
global hash_file_lock, defs_lock, refs_lock, tags_refs
|
||||||
|
|
||||||
for idx in idxes:
|
for idx in idxes:
|
||||||
if (idx % 1000 == 0): progress('refs: ' + str(idx), self.index+1)
|
if (idx % 1000 == 0): progress('refs: ' + str(idx), tags_refs)
|
||||||
|
|
||||||
with hash_file_lock:
|
with hash_file_lock:
|
||||||
hash = db.hash.get(idx)
|
hash = db.hash.get(idx)
|
||||||
|
|
@ -222,44 +241,47 @@ class UpdateRefs(Thread):
|
||||||
even = True
|
even = True
|
||||||
line_num = 1
|
line_num = 1
|
||||||
idents = {}
|
idents = {}
|
||||||
for tok in tokens:
|
with defs_lock:
|
||||||
even = not even
|
for tok in tokens:
|
||||||
if even:
|
even = not even
|
||||||
tok = prefix + tok
|
if even:
|
||||||
|
tok = prefix + tok
|
||||||
|
|
||||||
with defs_lock:
|
|
||||||
if db.defs.exists(tok) and lib.isIdent(tok):
|
if db.defs.exists(tok) and lib.isIdent(tok):
|
||||||
if tok in idents:
|
if tok in idents:
|
||||||
idents[tok] += ',' + str(line_num)
|
idents[tok] += ',' + str(line_num)
|
||||||
else:
|
else:
|
||||||
idents[tok] = str(line_num)
|
idents[tok] = str(line_num)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
line_num += tok.count(b'\1')
|
line_num += tok.count(b'\1')
|
||||||
|
|
||||||
for ident, lines in idents.items():
|
with refs_lock:
|
||||||
if db.refs.exists(ident):
|
for ident, lines in idents.items():
|
||||||
obj = db.refs.get(ident)
|
if db.refs.exists(ident):
|
||||||
else:
|
obj = db.refs.get(ident)
|
||||||
obj = data.RefList()
|
else:
|
||||||
|
obj = data.RefList()
|
||||||
|
|
||||||
obj.append(idx, lines, family)
|
obj.append(idx, lines, family)
|
||||||
if verbose:
|
if verbose:
|
||||||
print(f"ref: {ident} in #{idx} @ {lines}")
|
print(f"ref: {ident} in #{idx} @ {lines}")
|
||||||
db.refs.put(ident, obj)
|
db.refs.put(ident, obj)
|
||||||
|
|
||||||
|
|
||||||
class UpdateDocs(Thread):
|
class UpdateDocs(Thread):
|
||||||
def __init__(self):
|
def __init__(self, odd):
|
||||||
Thread.__init__(self, name="UpdateDocsElixir")
|
Thread.__init__(self, name="UpdateDocsElixir")
|
||||||
|
if odd:
|
||||||
|
self.index = 1
|
||||||
|
else:
|
||||||
|
self.index = 0
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
global new_idxes, tags_done
|
global new_idxes, tags_done, tags_docs, tags_docs_lock
|
||||||
|
|
||||||
self.index = 0
|
while(not (tags_done and self.index >= len(new_idxes))):
|
||||||
|
if(self.index >= len(new_idxes)):
|
||||||
while(not (tags_done and self.index == len(new_idxes))):
|
|
||||||
if(self.index == len(new_idxes)):
|
|
||||||
#Wait for new tags
|
#Wait for new tags
|
||||||
with tag_ready:
|
with tag_ready:
|
||||||
tag_ready.wait()
|
tag_ready.wait()
|
||||||
|
|
@ -267,15 +289,18 @@ class UpdateDocs(Thread):
|
||||||
|
|
||||||
new_idxes[self.index][1].wait() #Make sure the tag is ready
|
new_idxes[self.index][1].wait() #Make sure the tag is ready
|
||||||
|
|
||||||
|
with tags_docs_lock:
|
||||||
|
tags_docs += 1
|
||||||
|
|
||||||
self.update_doc_comments(new_idxes[self.index][0])
|
self.update_doc_comments(new_idxes[self.index][0])
|
||||||
|
|
||||||
self.index += 1
|
self.index += 2
|
||||||
|
|
||||||
def update_doc_comments(self, idxes):
|
def update_doc_comments(self, idxes):
|
||||||
global hash_file_lock
|
global hash_file_lock, docs_lock, tags_docs
|
||||||
|
|
||||||
for idx in idxes:
|
for idx in idxes:
|
||||||
if (idx % 1000 == 0): progress('docs: ' + str(idx), self.index+1)
|
if (idx % 1000 == 0): progress('docs: ' + str(idx), tags_docs)
|
||||||
|
|
||||||
with hash_file_lock:
|
with hash_file_lock:
|
||||||
hash = db.hash.get(idx)
|
hash = db.hash.get(idx)
|
||||||
|
|
@ -285,19 +310,20 @@ class UpdateDocs(Thread):
|
||||||
if family == None: continue
|
if family == None: continue
|
||||||
|
|
||||||
lines = scriptLines('parse-docs', hash, filename)
|
lines = scriptLines('parse-docs', hash, filename)
|
||||||
for l in lines:
|
with docs_lock:
|
||||||
ident, line = l.split(b' ')
|
for l in lines:
|
||||||
line = int(line.decode())
|
ident, line = l.split(b' ')
|
||||||
|
line = int(line.decode())
|
||||||
|
|
||||||
if db.docs.exists(ident):
|
if db.docs.exists(ident):
|
||||||
obj = db.docs.get(ident)
|
obj = db.docs.get(ident)
|
||||||
else:
|
else:
|
||||||
obj = data.RefList()
|
obj = data.RefList()
|
||||||
|
|
||||||
obj.append(idx, str(line), family)
|
obj.append(idx, str(line), family)
|
||||||
if verbose:
|
if verbose:
|
||||||
print(f"doc: {ident} in #{idx} @ {line}")
|
print(f"doc: {ident} in #{idx} @ {line}")
|
||||||
db.docs.put(ident, obj)
|
db.docs.put(ident, obj)
|
||||||
|
|
||||||
|
|
||||||
def progress(msg, current):
|
def progress(msg, current):
|
||||||
|
|
@ -317,9 +343,14 @@ project = lib.currentProject()
|
||||||
print(project + ' - found ' + str(len(tag_buf)) + ' new tags')
|
print(project + ' - found ' + str(len(tag_buf)) + ' new tags')
|
||||||
|
|
||||||
id_version_thread = UpdateIdVersion(tag_buf)
|
id_version_thread = UpdateIdVersion(tag_buf)
|
||||||
defs_thread = UpdateDefs()
|
#One half of the threads process the odd indexes of new_idxes
|
||||||
refs_thread = UpdateRefs()
|
#While the other half process the even indexes of new_idxes
|
||||||
docs_thread = UpdateDocs()
|
defs_thread_even = UpdateDefs(False)
|
||||||
|
defs_thread_odd = UpdateDefs(True)
|
||||||
|
refs_thread_even = UpdateRefs(False)
|
||||||
|
refs_thread_odd = UpdateRefs(True)
|
||||||
|
docs_thread_even = UpdateDocs(False)
|
||||||
|
docs_thread_odd = UpdateDocs(True)
|
||||||
|
|
||||||
#Start to process tags
|
#Start to process tags
|
||||||
id_version_thread.start()
|
id_version_thread.start()
|
||||||
|
|
@ -329,12 +360,18 @@ with tag_ready:
|
||||||
tag_ready.wait()
|
tag_ready.wait()
|
||||||
|
|
||||||
#Start remaining threads
|
#Start remaining threads
|
||||||
defs_thread.start()
|
defs_thread_even.start()
|
||||||
refs_thread.start()
|
refs_thread_even.start()
|
||||||
docs_thread.start()
|
docs_thread_even.start()
|
||||||
|
defs_thread_odd.start()
|
||||||
|
refs_thread_odd.start()
|
||||||
|
docs_thread_odd.start()
|
||||||
|
|
||||||
#Make sure all threads finished
|
#Make sure all threads finished
|
||||||
id_version_thread.join()
|
id_version_thread.join()
|
||||||
defs_thread.join()
|
defs_thread_even.join()
|
||||||
refs_thread.join()
|
defs_thread_odd.join()
|
||||||
docs_thread.join()
|
refs_thread_even.join()
|
||||||
|
refs_thread_odd.join()
|
||||||
|
docs_thread_even.join()
|
||||||
|
docs_thread_odd.join()
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue