From 2a7119fcff2b123c18a39e2298f82a8e6995dd7a Mon Sep 17 00:00:00 2001 From: Franciszek Stachura Date: Mon, 5 Aug 2024 21:07:04 +0200 Subject: [PATCH] autocomplete: Make faster by utilizing DB_SET_RANGE for prefix search https://stackoverflow.com/questions/12348346/berkeley-db-partial-match This *should* return the same results as the original autocomplete. Steps the original autocomplete takes to find identifiers: 1. It dumps all keys using db.keys() into a file query.py:Query.query('keys') data.py:BsdDB.get_keys() https://pybsddb.sourceforge.net/bsddb3.html - keys(txn=None) github.com/virtuozzo/cdn-bsddb3-python/blob/fbb1a877/Lib3/bsddb/dbobj.py#L171 github.com/virtuozzo/cdn-bsddb3-python/blob/fbb1a877/Modules/_bsddb.c#L8565 github.com/virtuozzo/cdn-bsddb3-python/blob/fbb1a877/Modules/_bsddb.c#L3730 seems that this just iterates over the db calling get with DB_NEXT on the cursor https://docs.oracle.com/cd/E17276_01/html/api_reference/C/dbcget.html 2. Iterates over the keys, looking for keys that start with provided string. The script stops when it finds 10 items. So it's assumed that the order of keys returned by get(DB_NEXT) will make sense. This version finds the first key that starts with the prefix using get(DB_SET_RANGE) and then, just like the previous autocomplete, iterates over the keys until it finds 10 matching keys. Now, could get(DB_SET_RANGE) skip keys (ex. point to some other key than the first key that starts with the prefix)? I think not - according to the docs, it should point to "the smallest key greater than or equal to the specified key". The comparison function determines what "greater or equal" means. The default comparison function compares keys lexically, with shorter keys before longer keys. https://docs.oracle.com/cd/E17276_01/html/api_reference/C/dbset_bt_compare.html I believe, although I couldn't find precise information about this, that order of keys when using DB_SET_RANGE shouldn't change. So tl;dr I'm mostly sure this should work the same way as it worked before, just faster. There could still be issues with how keys are ordered in results. The default function seems to just order characters by byte values. github.com/berkeleydb/libdb/blob/master/src/btree/bt_compare.c#L154 Also, this allows identifiers sent to autocomplete to contain commas. --- http/autocomplete.py | 70 ++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 41 deletions(-) diff --git a/http/autocomplete.py b/http/autocomplete.py index 4f7a692..9d0bd5c 100755 --- a/http/autocomplete.py +++ b/http/autocomplete.py @@ -18,10 +18,12 @@ # You should have received a copy of the GNU Affero General Public License # along with Elixir. If not, see . -import falcon -from urllib import parse import sys import os +import json +from urllib import parse +from bsddb3.db import DB_SET_RANGE +import falcon ELIXIR_DIR = os.path.dirname(os.path.realpath(__file__)) + '/..' @@ -29,6 +31,7 @@ if ELIXIR_DIR not in sys.path: sys.path = [ ELIXIR_DIR ] + sys.path import query +from lib import autoBytes class AutocompleteResource: def on_get(self, req, resp): @@ -44,57 +47,42 @@ class AutocompleteResource: q = query.Query(datadir, repodir) - # Create tmp directory for autocomplete - tmpdir = '/tmp/autocomplete/' + query_project - if not(os.path.isdir(tmpdir)): - os.makedirs(tmpdir, exist_ok=True) - latest = q.query('latest') - # Define some specific values for some families if query_family == 'B': - name = 'comps' + # DTS identifiers are stored quoted process = lambda x: parse.unquote(x) + db = q.db.comps else: - name = 'defs' process = lambda x: x + db = q.db.defs - # Init values for tmp files - filename = tmpdir + '/' + name - mode = 'r+' if os.path.exists(filename) else 'w+' + response = [] - # Open tmp file - # Fill it with the keys of the database only - # if the file is older than the database - f = open(filename, mode) - if not f.readline()[:-1] == latest: - f.seek(0) - f.truncate() - f.write(latest + "\n") - f.write('\n'.join([process(x.decode()) for x in q.query('keys', name)])) - f.seek(0) - f.readline() # Skip first line that store the version number - - # Prepare http response - response = '[' - - # Search for the 10 first matching elements in the tmp file - index = 0 - for i in f: - if i.startswith(query_string): - response += '"' + i[:-1] + '",' - index += 1 - - if index == 10: + i = 0 + cur = db.db.cursor() + query_bytes = autoBytes(parse.quote(query_string)) + # Find "the smallest key greater than or equal to the specified key" + # https://docs.oracle.com/cd/E17276_01/html/api_reference/C/dbcget.html + # In practice this should mean "the key that starts with provided prefix" + # See docs about the default comparison function for B-Tree databases: + # https://docs.oracle.com/cd/E17276_01/html/api_reference/C/dbset_bt_compare.html + key, _ = cur.get(query_bytes, DB_SET_RANGE) + while i <= 10: + if key.startswith(query_bytes): + # If found key starts with the prefix, add to response + # and move to the next key + i += 1 + response.append(process(key.decode("utf-8"))) + key, _ = cur.next() + else: + # If found key does not start with the prefix, stop break - # Complete and send response - response = response[:-1] + ']' - resp.text = response resp.status = falcon.HTTP_200 + resp.content_type = falcon.MEDIA_JSON + resp.media = response - # Close tmp file - f.close() def get_application(): app = falcon.App()