From e1dd2b4881aba5479d240c9073199cdd7eaa031e Mon Sep 17 00:00:00 2001 From: Michael Opdenacker Date: Fri, 5 May 2023 15:50:43 +0200 Subject: [PATCH] update.py: simplify and optimize thread management Instead of having a complicated thread count scheme and ending up wasting time waiting for the docs task to finish with a too low thread count at the end, let's run all tasks with same number of threads, which by default is the number of CPUs in the system. This way, the indexing work always uses as many CPUs as possible, especially when same tasks are finished before the others. The OS shouldn't bother if we try to run more threads than the actual number of CPUs. Set the ELIXIR_THREADS environment variable if you want to use a lower number of threads, typically on a desktop machine on which you want some responsiveness. Signed-off-by: Michael Opdenacker --- update.py | 58 ++++++++++--------------------------------------------- 1 file changed, 10 insertions(+), 48 deletions(-) diff --git a/update.py b/update.py index ae662f9..59f74c1 100755 --- a/update.py +++ b/update.py @@ -39,8 +39,6 @@ compatibles_parser = FindCompatibleDTS() db = data.DB(lib.getDataDir(), readonly=False, dtscomp=dts_comp_support) -# Number of cpu threads (+2 for version indexing) -cpu = 10 threads_list = [] hash_file_lock = Lock() # Lock for db.hash and db.file @@ -544,36 +542,9 @@ def progress(msg, current): # Check number of threads arg if len(argv) >= 2 and argv[1].isdigit() : - cpu = int(argv[1]) - - if cpu < 5 : - cpu = 5 - -# Distribute threads among functions using the following rules : -# There are more (or equal) refs threads than others -# There are more (or equal) defs threads than docs or comps threads -# Example : if cpu=6 : defs=1, refs=2, docs=1, comps=1, comps_docs=1 -# if cpu=7 : defs=2, refs=2, docs=1, comps=1, comps_docs=1 -# if cpu=8 : defs=2, refs=3, docs=1, comps=1, comps_docs=1 -# if cpu=11: defs=2, refs=3, docs=2, comps=2, comps_docs=2 -quo, rem = divmod(cpu, 5) -num_th_refs = quo -num_th_defs = quo -num_th_docs = quo - -# If DT bindings support is enabled, use $quo threads for each of the 2 threads -# Otherwise add them to the remaining threads -if dts_comp_support: - num_th_comps = quo - num_th_comps_docs = quo -else : - num_th_comps = 0 - num_th_comps_docs = 0 - rem += 2*quo - -quo, rem = divmod(rem, 2) -num_th_defs += quo -num_th_refs += quo + rem + cpus = int(argv[1]) +else: + cpus = os.cpu_count() tag_buf = [] for tag in scriptLines('list-tags'): @@ -591,22 +562,13 @@ if not num_tags: threads_list.append(UpdateIds(tag_buf)) threads_list.append(UpdateVersions(tag_buf)) -# Define defs threads -for i in range(num_th_defs): - threads_list.append(UpdateDefs(i, num_th_defs)) -# Define refs threads -for i in range(num_th_refs): - threads_list.append(UpdateRefs(i, num_th_refs)) -# Define docs threads -for i in range(num_th_docs): - threads_list.append(UpdateDocs(i, num_th_docs)) -# Define comps threads -for i in range(num_th_comps): - threads_list.append(UpdateComps(i, num_th_comps)) -# Define comps_docs threads -for i in range(num_th_comps_docs): - threads_list.append(UpdateCompsDocs(i, num_th_comps_docs)) - +# Define threads +for i in range(cpus): + threads_list.append(UpdateDefs(i, cpus)) + threads_list.append(UpdateRefs(i, cpus)) + threads_list.append(UpdateDocs(i, cpus)) + threads_list.append(UpdateComps(i, cpus)) + threads_list.append(UpdateCompsDocs(i, cpus)) # Start to process tags threads_list[0].start()