Improve progress reporting + factorize LXR_DATA_DIR access

- For long runs, allows to see which project we are in, and the
  percentage of processed tags

Signed-off-by: Michael Opdenacker <michael.opdenacker@bootlin.com>

lib.py

@@ -18,7 +18,7 @@
 #  You should have received a copy of the GNU Affero General Public License
 #  along with Elixir.  If not, see <http://www.gnu.org/licenses/>.
 
-import subprocess
+import subprocess, os
 
 def script (*args):
     args = ('./script.sh',) + args
@@ -169,3 +169,14 @@ def autoBytes (arg):
     elif type (arg) is int:
         arg = str(arg).encode()
     return arg
+
+def getDataDir ():
+    try:
+        dir=os.environ['LXR_DATA_DIR']
+    except KeyError:
+        print (argv[0] + ': LXR_DATA_DIR needs to be set')
+        exit (1)
+    return dir
+
+def currentProject ():
+    return os.path.basename (os.path.dirname (getDataDir ()))

query.py

@@ -23,13 +23,7 @@ import lib
 import data
 import os
 
-try:
+db = data.DB (lib.getDataDir(), readonly=True)
-    dbDir = os.environ['LXR_DATA_DIR']
-except KeyError:
-    print ('LXR_DATA_DIR needs to be set')
-    exit (1)
-
-db = data.DB (dbDir, readonly=True)
 
 from io import BytesIO
 

update.py

@@ -25,13 +25,7 @@ import data
 import os
 from data import PathList
 
-try:
+db = data.DB (lib.getDataDir (), readonly=False)
-    dbDir = os.environ['LXR_DATA_DIR']
-except KeyError:
-    print (argv[0] + ': LXR_DATA_DIR needs to be set')
-    exit (1)
-
-db = data.DB (dbDir, readonly=False)
 
 # Store new blobs hashed and file names (without path) for new tag
 
@@ -74,7 +68,7 @@ def updateVersions (tag):
 
 def updateDefinitions (blobs):
     for blob in blobs:
-        if (blob % 100 == 0): print ('D:', blob)
+        if (blob % 1000 == 0): progress ('defs: ' + str(blob))
         hash = db.hash.get (blob)
         filename = db.file.get (blob)
 
@@ -97,7 +91,7 @@ def updateDefinitions (blobs):
 
 def updateReferences (blobs):
     for blob in blobs:
-        if (blob % 100 == 0): print ('R:', blob)
+        if (blob % 1000 == 0): progress ('refs: ' + str(blob))
         hash = db.hash.get (blob)
         filename = db.file.get (blob)
 
@@ -128,6 +122,9 @@ def updateReferences (blobs):
             obj.append (blob, lines)
             db.refs.put (ident, obj)
 
+def progress (msg):
+    print ('{} - {} ({:.0%})'.format(project, msg, tagCount/numTags))
+
 # Main
 
 tagBuf = []
@@ -135,12 +132,16 @@ for tag in scriptLines ('list-tags'):
     if not db.vers.exists (tag):
         tagBuf.append (tag)
 
-print ('Found ' + str(len(tagBuf)) + ' new tags')
+numTags = len(tagBuf)
+tagCount = 0
+project = lib.currentProject ()
+
+progress ('found ' + str(len(tagBuf)) + ' new tags')
 
 for tag in tagBuf:
-    print (tag.decode(), end=': ')
+    tagCount +=1
     newBlobs = updateBlobIDs (tag)
-    print (str(len(newBlobs)) + ' new blobs')
+    progress (tag.decode() + ': ' + str(len(newBlobs)) + ' new blobs')
     updateVersions (tag)
     updateDefinitions (newBlobs)
     updateReferences (newBlobs)