From 0b8d735641f44b3ecefde30a3704f8d3f8092ab8 Mon Sep 17 00:00:00 2001 From: Franciszek Stachura Date: Fri, 27 Sep 2024 23:50:28 +0200 Subject: [PATCH] update: Make database usage thread safe Current update script serializes database access using mutexes. According to a user of Oracle support forums, this is not enough. https://forums.oracle.com/ords/apexds/post/berkeley-db-file-corrupted-while-operating-for-hours-panic-4953 > if you are accessing the same database from multiple threads or > multiple processes, they must share a cache (memory pool). In other > words, it is not sufficient to just make sure no DB->put or DB->get > operations are run simultaneously as you do with mutexes. Berkeley > DB also maintains information about database files across calls in > the cache, such as the list of free pages. If two threads accessing > a database file have independent freelists, they will eventually > both try to allocate the same page for different purposes, and the > structure of the file will be compromised. DB.open provides a flag that should be specified if database is to be shared between threads https://docs.oracle.com/cd/E17276_01/html/api_reference/C/dbopen.html > DB_THREAD > Cause the DB handle returned by DB->open() to be free-threaded; that > is, concurrently usable by multiple threads in the address space. > You should use this flag only in the absence of an encompassing > environment. While this probably won't solve all database concurrency issues (web accessing the database during updates likely still will behave weird) it could help with recent database corruption issues. https://docs.oracle.com/cd/E17276_01/html/programmer_reference/program_mt.html > The DB_THREAD flag must be specified to the DB_ENV->open() and > DB->open() methods if the Berkeley DB handles returned by those > interfaces will be used in the context of more than one thread. > Setting the DB_THREAD flag inconsistently may result in database > corruption. > When using the non-cursor Berkeley DB calls to retrieve key/data > items (for example, DB->get()), the memory to which the pointer > stored into the Dbt refers is valid only until the next call using > the DB handle returned by DB->open(). This includes any use of the > returned DB handle, including by another thread within the process. > > For this reason, if the DB_THREAD handle was specified to the > DB->open() method, either DB_DBT_MALLOC, DB_DBT_REALLOC or > DB_DBT_USERMEM must be specified in the DBT when performing any > non-cursor key or data retrieval. It seems that bsddb3 sets appropriate flags in DBTs for us if DB_THREAD is specified. https://hg.jcea.es/pybsddb/file/tip/src/Module/berkeleydb.c#l2025 (ctrl+f for DB_THREAD) I believe DBTs used in DB_put shouldn't require any extra flags because the DBTs are only read by Berkeley DB (doesn't matter if they get invalidated on the next call). --- elixir/data.py | 34 ++++++++++++++++++++-------------- update.py | 2 +- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/elixir/data.py b/elixir/data.py index 7e8c3ed..ccfc695 100755 --- a/elixir/data.py +++ b/elixir/data.py @@ -145,14 +145,20 @@ class RefList: return self.data class BsdDB: - def __init__(self, filename, readonly, contentType): + def __init__(self, filename, readonly, contentType, shared=False): self.filename = filename self.db = bsddb3.db.DB() + flags = 0 + if shared: + flags |= bsddb3.db.DB_THREAD if readonly: - self.db.open(filename, flags=bsddb3.db.DB_RDONLY) + flags |= bsddb3.db.DB_RDONLY + + if readonly: + self.db.open(filename, flags=flags) else: self.db.open(filename, - flags=bsddb3.db.DB_CREATE, + flags=flags | bsddb3.db.DB_CREATE, mode=0o644, dbtype=bsddb3.db.DB_BTREE) self.ctype = contentType @@ -183,7 +189,7 @@ class BsdDB: self.db.close() class DB: - def __init__(self, dir, readonly=True, dtscomp=False): + def __init__(self, dir, readonly=True, dtscomp=False, shared=False): if os.path.isdir(dir): self.dir = dir else: @@ -191,22 +197,22 @@ class DB: ro = readonly - self.vars = BsdDB(dir + '/variables.db', ro, lambda x: int(x.decode()) ) + self.vars = BsdDB(dir + '/variables.db', ro, lambda x: int(x.decode()), shared=shared) # Key-value store of basic information - self.blob = BsdDB(dir + '/blobs.db', ro, lambda x: int(x.decode()) ) + self.blob = BsdDB(dir + '/blobs.db', ro, lambda x: int(x.decode()), shared=shared) # Map hash to sequential integer serial number - self.hash = BsdDB(dir + '/hashes.db', ro, lambda x: x ) + self.hash = BsdDB(dir + '/hashes.db', ro, lambda x: x, shared=shared) # Map serial number back to hash - self.file = BsdDB(dir + '/filenames.db', ro, lambda x: x.decode() ) + self.file = BsdDB(dir + '/filenames.db', ro, lambda x: x.decode(), shared=shared) # Map serial number to filename - self.vers = BsdDB(dir + '/versions.db', ro, PathList) - self.defs = BsdDB(dir + '/definitions.db', ro, DefList) - self.refs = BsdDB(dir + '/references.db', ro, RefList) - self.docs = BsdDB(dir + '/doccomments.db', ro, RefList) + self.vers = BsdDB(dir + '/versions.db', ro, PathList, shared=shared) + self.defs = BsdDB(dir + '/definitions.db', ro, DefList, shared=shared) + self.refs = BsdDB(dir + '/references.db', ro, RefList, shared=shared) + self.docs = BsdDB(dir + '/doccomments.db', ro, RefList, shared=shared) self.dtscomp = dtscomp if dtscomp: - self.comps = BsdDB(dir + '/compatibledts.db', ro, RefList) - self.comps_docs = BsdDB(dir + '/compatibledts_docs.db', ro, RefList) + self.comps = BsdDB(dir + '/compatibledts.db', ro, RefList, shared=shared) + self.comps_docs = BsdDB(dir + '/compatibledts_docs.db', ro, RefList, shared=shared) # Use a RefList in case there are multiple doc comments for an identifier def close(self): diff --git a/update.py b/update.py index 3b8fee7..79cb4dc 100755 --- a/update.py +++ b/update.py @@ -37,7 +37,7 @@ dts_comp_support = int(script('dts-comp')) compatibles_parser = FindCompatibleDTS() -db = data.DB(lib.getDataDir(), readonly=False, dtscomp=dts_comp_support) +db = data.DB(lib.getDataDir(), readonly=False, shared=True, dtscomp=dts_comp_support) # Number of cpu threads (+2 for version indexing) cpu = 10