update: Make database usage thread safe

Current update script serializes database access using mutexes.
According to a user of Oracle support forums, this is not enough.

https://forums.oracle.com/ords/apexds/post/berkeley-db-file-corrupted-while-operating-for-hours-panic-4953
> if you are accessing the same database from multiple threads or
> multiple processes, they must share a cache (memory pool). In other
> words, it is not sufficient to just make sure no DB->put or DB->get
> operations are run simultaneously as you do with mutexes. Berkeley
> DB also maintains information about database files across calls in
> the cache, such as the list of free pages. If two threads accessing
> a database file have independent freelists, they will eventually
> both try to allocate the same page for different purposes, and the
> structure of the file will be compromised.

DB.open provides a flag that should be specified if database is to be
shared between threads

https://docs.oracle.com/cd/E17276_01/html/api_reference/C/dbopen.html

> DB_THREAD
> Cause the DB handle returned by DB->open() to be free-threaded; that
> is, concurrently usable by multiple threads in the address space.
> You should use this flag only in the absence of an encompassing
> environment.

While this probably won't solve all database concurrency issues (web
accessing the database during updates likely still will behave weird)
it could help with recent database corruption issues.

https://docs.oracle.com/cd/E17276_01/html/programmer_reference/program_mt.html

> The DB_THREAD flag must be specified to the DB_ENV->open() and
> DB->open() methods if the Berkeley DB handles returned by those
> interfaces will be used in the context of more than one thread.
> Setting the DB_THREAD flag inconsistently may result in database
> corruption.

> When using the non-cursor Berkeley DB calls to retrieve key/data
> items (for example, DB->get()), the memory to which the pointer
> stored into the Dbt refers is valid only until the next call using
> the DB handle returned by DB->open(). This includes any use of the
> returned DB handle, including by another thread within the process.
>
> For this reason, if the DB_THREAD handle was specified to the
> DB->open() method, either DB_DBT_MALLOC, DB_DBT_REALLOC or
> DB_DBT_USERMEM must be specified in the DBT when performing any
> non-cursor key or data retrieval.

It seems that bsddb3 sets appropriate flags in DBTs for us if DB_THREAD
is specified.

https://hg.jcea.es/pybsddb/file/tip/src/Module/berkeleydb.c#l2025

(ctrl+f for DB_THREAD)

I believe DBTs used in DB_put shouldn't require any extra flags because
the DBTs are only read by Berkeley DB (doesn't matter if they get
invalidated on the next call).

elixir/data.py

@@ -145,14 +145,20 @@ class RefList:
         return self.data
 
 class BsdDB:
-    def __init__(self, filename, readonly, contentType):
+    def __init__(self, filename, readonly, contentType, shared=False):
         self.filename = filename
         self.db = bsddb3.db.DB()
+        flags = 0
+        if shared:
+            flags |= bsddb3.db.DB_THREAD
         if readonly:
-            self.db.open(filename, flags=bsddb3.db.DB_RDONLY)
+            flags |= bsddb3.db.DB_RDONLY
+
+        if readonly:
+            self.db.open(filename, flags=flags)
         else:
             self.db.open(filename,
-                flags=bsddb3.db.DB_CREATE,
+                flags=flags | bsddb3.db.DB_CREATE,
                 mode=0o644,
                 dbtype=bsddb3.db.DB_BTREE)
         self.ctype = contentType
@@ -183,7 +189,7 @@ class BsdDB:
         self.db.close()
 
 class DB:
-    def __init__(self, dir, readonly=True, dtscomp=False):
+    def __init__(self, dir, readonly=True, dtscomp=False, shared=False):
         if os.path.isdir(dir):
             self.dir = dir
         else:
@@ -191,22 +197,22 @@ class DB:
 
         ro = readonly
 
-        self.vars = BsdDB(dir + '/variables.db', ro, lambda x: int(x.decode()) )
+        self.vars = BsdDB(dir + '/variables.db', ro, lambda x: int(x.decode()), shared=shared)
             # Key-value store of basic information
-        self.blob = BsdDB(dir + '/blobs.db', ro, lambda x: int(x.decode()) )
+        self.blob = BsdDB(dir + '/blobs.db', ro, lambda x: int(x.decode()), shared=shared)
             # Map hash to sequential integer serial number
-        self.hash = BsdDB(dir + '/hashes.db', ro, lambda x: x )
+        self.hash = BsdDB(dir + '/hashes.db', ro, lambda x: x, shared=shared)
             # Map serial number back to hash
-        self.file = BsdDB(dir + '/filenames.db', ro, lambda x: x.decode() )
+        self.file = BsdDB(dir + '/filenames.db', ro, lambda x: x.decode(), shared=shared)
             # Map serial number to filename
-        self.vers = BsdDB(dir + '/versions.db', ro, PathList)
+        self.vers = BsdDB(dir + '/versions.db', ro, PathList, shared=shared)
-        self.defs = BsdDB(dir + '/definitions.db', ro, DefList)
+        self.defs = BsdDB(dir + '/definitions.db', ro, DefList, shared=shared)
-        self.refs = BsdDB(dir + '/references.db', ro, RefList)
+        self.refs = BsdDB(dir + '/references.db', ro, RefList, shared=shared)
-        self.docs = BsdDB(dir + '/doccomments.db', ro, RefList)
+        self.docs = BsdDB(dir + '/doccomments.db', ro, RefList, shared=shared)
         self.dtscomp = dtscomp
         if dtscomp:
-            self.comps = BsdDB(dir + '/compatibledts.db', ro, RefList)
+            self.comps = BsdDB(dir + '/compatibledts.db', ro, RefList, shared=shared)
-            self.comps_docs = BsdDB(dir + '/compatibledts_docs.db', ro, RefList)
+            self.comps_docs = BsdDB(dir + '/compatibledts_docs.db', ro, RefList, shared=shared)
             # Use a RefList in case there are multiple doc comments for an identifier
 
     def close(self):

update.py

@@ -37,7 +37,7 @@ dts_comp_support = int(script('dts-comp'))
 
 compatibles_parser = FindCompatibleDTS()
 
-db = data.DB(lib.getDataDir(), readonly=False, dtscomp=dts_comp_support)
+db = data.DB(lib.getDataDir(), readonly=False, shared=True, dtscomp=dts_comp_support)
 
 # Number of cpu threads (+2 for version indexing)
 cpu = 10