portugalfuturista/elixir
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

update.py: simplify and optimize thread management

Browse source 
Instead of having a complicated thread count scheme and
ending up wasting time waiting for the docs task to finish
with a too low thread count at the end, let's run all
tasks with same number of threads, which by default
is the number of CPUs in the system.

This way, the indexing work always uses as many CPUs
as possible, especially when same tasks are finished
before the others.

The OS shouldn't bother if we try to run more threads
than the actual number of CPUs.

Set the ELIXIR_THREADS environment variable
if you want to use a lower number of threads,
typically on a desktop machine on which you want some
responsiveness.

Signed-off-by: Michael Opdenacker <michael.opdenacker@bootlin.com>
This commit is contained in:
Michael Opdenacker 2023-05-05 15:50:43 +02:00
parent c156433520
commit e1dd2b4881
1 changed files with 10 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
update.py
View file

@ -39,8 +39,6 @@ compatibles_parser = FindCompatibleDTS()
db = data.DB(lib.getDataDir(), readonly=False, dtscomp=dts_comp_support) db = data.DB(lib.getDataDir(), readonly=False, dtscomp=dts_comp_support)
# Number of cpu threads (+2 for version indexing)
cpu = 10
threads_list = [] threads_list = []
hash_file_lock = Lock() # Lock for db.hash and db.file hash_file_lock = Lock() # Lock for db.hash and db.file
@ -544,36 +542,9 @@ def progress(msg, current):
# Check number of threads arg # Check number of threads arg
if len(argv) >= 2 and argv[1].isdigit() : if len(argv) >= 2 and argv[1].isdigit() :
cpu = int(argv[1]) cpus = int(argv[1])
else:
if cpu < 5 : cpus = os.cpu_count()
cpu = 5
# Distribute threads among functions using the following rules :
# There are more (or equal) refs threads than others
# There are more (or equal) defs threads than docs or comps threads
# Example : if cpu=6 : defs=1, refs=2, docs=1, comps=1, comps_docs=1
# if cpu=7 : defs=2, refs=2, docs=1, comps=1, comps_docs=1
# if cpu=8 : defs=2, refs=3, docs=1, comps=1, comps_docs=1
# if cpu=11: defs=2, refs=3, docs=2, comps=2, comps_docs=2
quo, rem = divmod(cpu, 5)
num_th_refs = quo
num_th_defs = quo
num_th_docs = quo
# If DT bindings support is enabled, use $quo threads for each of the 2 threads
# Otherwise add them to the remaining threads
if dts_comp_support:
num_th_comps = quo
num_th_comps_docs = quo
else :
num_th_comps = 0
num_th_comps_docs = 0
rem += 2*quo
quo, rem = divmod(rem, 2)
num_th_defs += quo
num_th_refs += quo + rem
tag_buf = [] tag_buf = []
for tag in scriptLines('list-tags'): for tag in scriptLines('list-tags'):
@ -591,22 +562,13 @@ if not num_tags:
threads_list.append(UpdateIds(tag_buf)) threads_list.append(UpdateIds(tag_buf))
threads_list.append(UpdateVersions(tag_buf)) threads_list.append(UpdateVersions(tag_buf))
# Define defs threads # Define threads
for i in range(num_th_defs): for i in range(cpus):
threads_list.append(UpdateDefs(i, num_th_defs)) threads_list.append(UpdateDefs(i, cpus))
# Define refs threads threads_list.append(UpdateRefs(i, cpus))
for i in range(num_th_refs): threads_list.append(UpdateDocs(i, cpus))
threads_list.append(UpdateRefs(i, num_th_refs)) threads_list.append(UpdateComps(i, cpus))
# Define docs threads threads_list.append(UpdateCompsDocs(i, cpus))
for i in range(num_th_docs):
threads_list.append(UpdateDocs(i, num_th_docs))
# Define comps threads
for i in range(num_th_comps):
threads_list.append(UpdateComps(i, num_th_comps))
# Define comps_docs threads
for i in range(num_th_comps_docs):
threads_list.append(UpdateCompsDocs(i, num_th_comps_docs))
# Start to process tags # Start to process tags
threads_list[0].start() threads_list[0].start()