diff --git a/update.py b/update.py index 28da939..796cbc2 100755 --- a/update.py +++ b/update.py @@ -35,12 +35,21 @@ db = data.DB(lib.getDataDir(), readonly=False) hash_file_lock = Lock() #Lock for db.hash and db.file defs_lock = Lock() #Lock for db.defs +refs_lock = Lock() #Lock for db.refs +docs_lock = Lock() #Lock for db.docs tag_ready = Condition() #Waiting for new tags new_idxes = [] # (new idxes, Event idxes ready, Event defs ready) tags_done = False #True if all tags have been added to new_idxes +#Progress variables +tags_defs = 0 +tags_defs_lock = Lock() +tags_refs = 0 +tags_refs_lock = Lock() +tags_docs = 0 +tags_docs_lock = Lock() class UpdateIdVersion(Thread): def __init__(self, tag_buf): @@ -120,16 +129,18 @@ class UpdateIdVersion(Thread): class UpdateDefs(Thread): - def __init__(self): + def __init__(self, odd): Thread.__init__(self, name="UpdateDefsElixir") + if odd: + self.index = 1 + else: + self.index = 0 def run(self): - global new_idxes, tags_done, tag_ready + global new_idxes, tags_done, tag_ready, tags_defs, tags_defs_lock - self.index = 0 - - while(not (tags_done and self.index == len(new_idxes))): - if(self.index == len(new_idxes)): + while(not (tags_done and self.index >= len(new_idxes))): + if(self.index >= len(new_idxes)): #Wait for new tags with tag_ready: tag_ready.wait() @@ -137,18 +148,21 @@ class UpdateDefs(Thread): new_idxes[self.index][1].wait() #Make sure the tag is ready + with tags_defs_lock: + tags_defs += 1 + self.update_definitions(new_idxes[self.index][0]) new_idxes[self.index][2].set() #Tell that UpdateDefs processed the tag - self.index += 1 + self.index += 2 def update_definitions(self, idxes): - global hash_file_lock, defs_lock + global hash_file_lock, defs_lock, tags_defs for idx in idxes: - if (idx % 1000 == 0): progress('defs: ' + str(idx), self.index+1) + if (idx % 1000 == 0): progress('defs: ' + str(idx), tags_defs) with hash_file_lock: hash = db.hash.get(idx) @@ -158,36 +172,38 @@ class UpdateDefs(Thread): if family == None: continue lines = scriptLines('parse-defs', hash, filename, family) - for l in lines: - ident, type, line = l.split(b' ') - type = type.decode() - line = int(line.decode()) - with defs_lock: + with defs_lock: + for l in lines: + ident, type, line = l.split(b' ') + type = type.decode() + line = int(line.decode()) + if db.defs.exists(ident): obj = db.defs.get(ident) else: obj = data.DefList() - obj.add_family(family) - obj.append(idx, type, line, family) - if verbose: - print(f"def {type} {ident} in #{idx} @ {line}") - with defs_lock: + obj.add_family(family) + obj.append(idx, type, line, family) + if verbose: + print(f"def {type} {ident} in #{idx} @ {line}") db.defs.put(ident, obj) class UpdateRefs(Thread): - def __init__(self): + def __init__(self, odd): Thread.__init__(self, name="UpdateRefsElixir") + if odd: + self.index = 1 + else: + self.index = 0 def run(self): - global new_idxes, tags_done + global new_idxes, tags_done, tags_refs, tags_refs_lock - self.index = 0 - - while(not (tags_done and self.index == len(new_idxes))): - if(self.index == len(new_idxes)): + while(not (tags_done and self.index >= len(new_idxes))): + if(self.index >= len(new_idxes)): #Wait for new tags with tag_ready: tag_ready.wait() @@ -196,15 +212,18 @@ class UpdateRefs(Thread): new_idxes[self.index][1].wait() #Make sure the tag is ready new_idxes[self.index][2].wait() #Make sure UpdateDefs processed the tag + with tags_refs_lock: + tags_refs += 1 + self.update_references(new_idxes[self.index][0]) - self.index += 1 + self.index += 2 def update_references(self, idxes): - global hash_file_lock, defs_lock + global hash_file_lock, defs_lock, refs_lock, tags_refs for idx in idxes: - if (idx % 1000 == 0): progress('refs: ' + str(idx), self.index+1) + if (idx % 1000 == 0): progress('refs: ' + str(idx), tags_refs) with hash_file_lock: hash = db.hash.get(idx) @@ -222,44 +241,47 @@ class UpdateRefs(Thread): even = True line_num = 1 idents = {} - for tok in tokens: - even = not even - if even: - tok = prefix + tok + with defs_lock: + for tok in tokens: + even = not even + if even: + tok = prefix + tok - with defs_lock: if db.defs.exists(tok) and lib.isIdent(tok): if tok in idents: idents[tok] += ',' + str(line_num) else: idents[tok] = str(line_num) - else: - line_num += tok.count(b'\1') + else: + line_num += tok.count(b'\1') - for ident, lines in idents.items(): - if db.refs.exists(ident): - obj = db.refs.get(ident) - else: - obj = data.RefList() + with refs_lock: + for ident, lines in idents.items(): + if db.refs.exists(ident): + obj = db.refs.get(ident) + else: + obj = data.RefList() - obj.append(idx, lines, family) - if verbose: - print(f"ref: {ident} in #{idx} @ {lines}") - db.refs.put(ident, obj) + obj.append(idx, lines, family) + if verbose: + print(f"ref: {ident} in #{idx} @ {lines}") + db.refs.put(ident, obj) class UpdateDocs(Thread): - def __init__(self): + def __init__(self, odd): Thread.__init__(self, name="UpdateDocsElixir") + if odd: + self.index = 1 + else: + self.index = 0 def run(self): - global new_idxes, tags_done + global new_idxes, tags_done, tags_docs, tags_docs_lock - self.index = 0 - - while(not (tags_done and self.index == len(new_idxes))): - if(self.index == len(new_idxes)): + while(not (tags_done and self.index >= len(new_idxes))): + if(self.index >= len(new_idxes)): #Wait for new tags with tag_ready: tag_ready.wait() @@ -267,15 +289,18 @@ class UpdateDocs(Thread): new_idxes[self.index][1].wait() #Make sure the tag is ready + with tags_docs_lock: + tags_docs += 1 + self.update_doc_comments(new_idxes[self.index][0]) - self.index += 1 + self.index += 2 def update_doc_comments(self, idxes): - global hash_file_lock + global hash_file_lock, docs_lock, tags_docs for idx in idxes: - if (idx % 1000 == 0): progress('docs: ' + str(idx), self.index+1) + if (idx % 1000 == 0): progress('docs: ' + str(idx), tags_docs) with hash_file_lock: hash = db.hash.get(idx) @@ -285,19 +310,20 @@ class UpdateDocs(Thread): if family == None: continue lines = scriptLines('parse-docs', hash, filename) - for l in lines: - ident, line = l.split(b' ') - line = int(line.decode()) + with docs_lock: + for l in lines: + ident, line = l.split(b' ') + line = int(line.decode()) - if db.docs.exists(ident): - obj = db.docs.get(ident) - else: - obj = data.RefList() + if db.docs.exists(ident): + obj = db.docs.get(ident) + else: + obj = data.RefList() - obj.append(idx, str(line), family) - if verbose: - print(f"doc: {ident} in #{idx} @ {line}") - db.docs.put(ident, obj) + obj.append(idx, str(line), family) + if verbose: + print(f"doc: {ident} in #{idx} @ {line}") + db.docs.put(ident, obj) def progress(msg, current): @@ -317,9 +343,14 @@ project = lib.currentProject() print(project + ' - found ' + str(len(tag_buf)) + ' new tags') id_version_thread = UpdateIdVersion(tag_buf) -defs_thread = UpdateDefs() -refs_thread = UpdateRefs() -docs_thread = UpdateDocs() +#One half of the threads process the odd indexes of new_idxes +#While the other half process the even indexes of new_idxes +defs_thread_even = UpdateDefs(False) +defs_thread_odd = UpdateDefs(True) +refs_thread_even = UpdateRefs(False) +refs_thread_odd = UpdateRefs(True) +docs_thread_even = UpdateDocs(False) +docs_thread_odd = UpdateDocs(True) #Start to process tags id_version_thread.start() @@ -329,12 +360,18 @@ with tag_ready: tag_ready.wait() #Start remaining threads -defs_thread.start() -refs_thread.start() -docs_thread.start() +defs_thread_even.start() +refs_thread_even.start() +docs_thread_even.start() +defs_thread_odd.start() +refs_thread_odd.start() +docs_thread_odd.start() #Make sure all threads finished id_version_thread.join() -defs_thread.join() -refs_thread.join() -docs_thread.join() +defs_thread_even.join() +defs_thread_odd.join() +refs_thread_even.join() +refs_thread_odd.join() +docs_thread_even.join() +docs_thread_odd.join()