From 1bd4cd464a1a98c8810f482d9be8e5bbad2fc4a0 Mon Sep 17 00:00:00 2001 From: Maxime Chretien Date: Mon, 8 Jun 2020 11:55:17 +0200 Subject: [PATCH] update.py: Split updateComps in two threads One thread that parse C and DT files and one that parse Documentations files Signed-off-by: Maxime Chretien --- update.py | 65 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/update.py b/update.py index ba5cf42..2f99b77 100755 --- a/update.py +++ b/update.py @@ -37,7 +37,7 @@ dts_comp_support = int(script('dts-comp')) db = data.DB(lib.getDataDir(), readonly=False, dtscomp=dts_comp_support) # Number of cpu threads (+2 for version indexing) -cpu = 8 +cpu = 10 threads_list = [] hash_file_lock = Lock() # Lock for db.hash and db.file @@ -49,7 +49,7 @@ comps_lock = Lock() # Lock for db.comps comps_docs_lock = Lock() # Lock for db.comps_docs tag_ready = Condition() # Waiting for new tags -new_idxes = [] # (new idxes, Event idxes ready, Event defs ready) +new_idxes = [] # (new idxes, Event idxes ready, Event defs ready, Event comps ready) bindings_idxes = [] # DT bindings documentation files tags_done = False # True if all tags have been added to new_idxes @@ -63,6 +63,8 @@ tags_docs = 0 tags_docs_lock = Lock() tags_comps = 0 tags_comps_lock = Lock() +tags_comps_docs = 0 +tags_comps_docs_lock = Lock() class UpdateIds(Thread): def __init__(self, tag_buf): @@ -75,7 +77,7 @@ class UpdateIds(Thread): for tag in self.tag_buf: - new_idxes.append((self.update_blob_ids(tag), Event(), Event())) + new_idxes.append((self.update_blob_ids(tag), Event(), Event(), Event())) progress(tag.decode() + ': ' + str(len(new_idxes[self.index][0])) + ' new blobs', self.index+1) @@ -386,7 +388,7 @@ class UpdateComps(Thread): self.update_compatibles(new_idxes[self.index][0]) - self.update_compatibles_bindings(new_idxes[self.index][0]) + new_idxes[self.index][3].set() # Tell that UpdateComps processed the tag self.index += self.inc @@ -426,11 +428,38 @@ class UpdateComps(Thread): print(f"comps: {ident} in #{idx} @ {line}") db.comps.put(ident, obj) + +class UpdateCompsDocs(Thread): + def __init__(self, start, inc): + Thread.__init__(self, name="UpdateCompsDocsElixir") + self.index = start + self.inc = inc # Equivalient to the number of comps_docs threads + + def run(self): + global new_idxes, tags_done, tags_comps_docs, tags_comps_docs_lock + + while(not (tags_done and self.index >= len(new_idxes))): + if(self.index >= len(new_idxes)): + # Wait for new tags + with tag_ready: + tag_ready.wait() + continue + + new_idxes[self.index][1].wait() # Make sure the tag is ready + new_idxes[self.index][3].wait() # Make sure UpdateComps processed the tag + + with tags_comps_docs_lock: + tags_comps_docs += 1 + + self.update_compatibles_bindings(new_idxes[self.index][0]) + + self.index += self.inc + def update_compatibles_bindings(self, idxes): - global hash_file_lock, comps_lock, tags_comps, bindings_idxes + global hash_file_lock, comps_lock, comps_docs_lock, tags_comps_docs, bindings_idxes for idx in idxes: - if (idx % 1000 == 0): progress('comps_docs: ' + str(idx), tags_comps) + if (idx % 1000 == 0): progress('comps_docs: ' + str(idx), tags_comps_docs) if not idx in bindings_idxes: # Parse only bindings doc files continue @@ -476,28 +505,30 @@ def progress(msg, current): if len(argv) >= 2 and argv[1].isdigit() : cpu = int(argv[1]) - if cpu < 4 : - cpu = 4 + if cpu < 5 : + cpu = 5 # Distribute threads among functions using the following rules : # There are more (or equal) refs threads than others # There are more (or equal) defs threads than docs or comps threads -# Example : if cpu=6 : defs=2, refs=2, docs=1, comps=1 -# if cpu=7 : defs=2, refs=3, docs=1, comps=1 -# if cpu=8 : defs=2, refs=2, docs=2, comps=2 -# if cpu=11: defs=3, refs=4, docs=2, comps=2 -quo, rem = divmod(cpu, 4) +# Example : if cpu=6 : defs=1, refs=2, docs=1, comps=1, comps_docs=1 +# if cpu=7 : defs=2, refs=2, docs=1, comps=1, comps_docs=1 +# if cpu=8 : defs=2, refs=3, docs=1, comps=1, comps_docs=1 +# if cpu=11: defs=2, refs=3, docs=2, comps=2, comps_docs=2 +quo, rem = divmod(cpu, 5) num_th_refs = quo num_th_defs = quo num_th_docs = quo -# If DT bindings support is enabled, use $quo threads for that +# If DT bindings support is enabled, use $quo threads for each of the 2 threads # Otherwise add them to the remaining threads if dts_comp_support: num_th_comps = quo + num_th_comps_docs = quo else : num_th_comps = 0 - rem += quo + num_th_comps_docs = 0 + rem += 2*quo quo, rem = divmod(rem, 2) num_th_defs += quo @@ -528,6 +559,10 @@ for i in range(num_th_docs): # Define comps threads for i in range(num_th_comps): threads_list.append(UpdateComps(i, num_th_comps)) +# Define comps_docs threads +for i in range(num_th_comps_docs): + threads_list.append(UpdateCompsDocs(i, num_th_comps_docs)) + # Start to process tags ids_thread.start()