database: Add support for dts compatible strings

They can be found in C, DTS and documentation files

Signed-off-by: Maxime Chretien <maxime.chretien@bootlin.com>

data.py

@@ -194,4 +194,5 @@ class DB:
         self.defs = BsdDB(dir + '/definitions.db', ro, DefList)
         self.refs = BsdDB(dir + '/references.db', ro, RefList)
         self.docs = BsdDB(dir + '/doccomments.db', ro, RefList)
+        self.comps = BsdDB(dir + '/compatibledts.db', ro, RefList)
             # Use a RefList in case there are multiple doc comments for an identifier

find_compatible_dts.py

@@ -21,6 +21,7 @@
 from sys import argv
 import re
 import os
+from urllib import parse
 
 usage_message = ("USAGE: find_compatible_dts.py <file> <family>\n"
                  "file : The file you want to search in\n"
@@ -68,7 +69,8 @@ except IOError:
     exit(1)
 
 # Iterate though lines and search for idents
-for num, line in enumerate(f, 1):
+num = 1
+for line in f:
     if family == 'C':
         ret = parse_c(line)
     elif family == 'D':
@@ -78,8 +80,10 @@ for num, line in enumerate(f, 1):
 
     if ret != []:
         for i in range(len(ret)):
-            ident_list += str(ret[i]) + ' ' + str(num) + '\n'
+            ident_list += str(parse.quote(ret[i])) + ' ' + str(num) + '\n'
 
-# Print the list (except the last \n) and exit
-print(ident_list[:-1])
+    num += 1
+
+# Print the list and exit
+print(ident_list, end='')
 exit(0)

projects/linux.sh

@@ -1,5 +1,8 @@
 # Elixir definitions for Linux
 
+# Enable DT bindings compatible strings support
+dts_comp_support=1
+
 get_tags()
 {
     git tag |

script.sh

@@ -30,6 +30,7 @@ script_path=`realpath "$0"`
 cd `dirname "$script_path"`
 script_dir=`pwd`
 cd "$cur_dir"
+dts_comp_support=0 # DT bindings compatible strings support (disable by default)
 
 version_dir()
 {
@@ -207,6 +208,21 @@ parse_docs()
     rm -rf "$tmpfile"
 }
 
+parse_comps()
+{
+    tmpfile=`mktemp`
+
+    git cat-file blob "$opt1" > "$tmpfile"
+    "$script_dir/find_compatible_dts.py" "$tmpfile" "$opt3" || exit "$?"
+
+    rm -rf "$tmpfile"
+}
+
+dts_comp()
+{
+    echo $dts_comp_support
+}
+
 project=$(basename `dirname $LXR_REPO_DIR`)
 
 plugin=$script_dir/projects/$project.sh
@@ -280,6 +296,14 @@ case $cmd in
         parse_docs
         ;;
 
+    parse-comps)
+        parse_comps
+        ;;
+
+    dts-comp)
+        dts_comp
+        ;;
+
     help)
         echo "Usage: $0 subcommand [args]..."
         exit 1

update.py

@@ -23,7 +23,7 @@
 # This is different from that blob's Git hash.
 
 from sys import argv
-from lib import scriptLines
+from lib import script, scriptLines
 import lib
 import data
 import os
@@ -34,27 +34,31 @@ verbose = False
 
 db = data.DB(lib.getDataDir(), readonly=False)
 
-#Number of cpu threads (+1 for version indexing)
+# Number of cpu threads (+1 for version indexing)
 cpu = 8
 threads_list = []
 
-hash_file_lock = Lock() #Lock for db.hash and db.file
-defs_lock = Lock() #Lock for db.defs
-refs_lock = Lock() #Lock for db.refs
-docs_lock = Lock() #Lock for db.docs
-tag_ready = Condition() #Waiting for new tags
+hash_file_lock = Lock() # Lock for db.hash and db.file
+defs_lock = Lock() # Lock for db.defs
+refs_lock = Lock() # Lock for db.refs
+docs_lock = Lock() # Lock for db.docs
+comps_lock = Lock() # Lock for db.comps
+tag_ready = Condition() # Waiting for new tags
 
 new_idxes = [] # (new idxes, Event idxes ready, Event defs ready)
+bindings_idxes = [] # DT bindings documentation files
 
-tags_done = False #True if all tags have been added to new_idxes
+tags_done = False # True if all tags have been added to new_idxes
 
-#Progress variables
+# Progress variables
 tags_defs = 0
 tags_defs_lock = Lock()
 tags_refs = 0
 tags_refs_lock = Lock()
 tags_docs = 0
 tags_docs_lock = Lock()
+tags_comps = 0
+tags_comps_lock = Lock()
 
 class UpdateIdVersion(Thread):
     def __init__(self, tag_buf):
@@ -74,11 +78,11 @@ class UpdateIdVersion(Thread):
 
             self.update_versions(tag)
 
-            new_idxes[self.index][1].set() #Tell that the tag is ready
+            new_idxes[self.index][1].set() # Tell that the tag is ready
 
             self.index += 1
 
-            #Wake up waiting threads
+            # Wake up waiting threads
             with tag_ready:
                 tag_ready.notify_all()
 
@@ -128,6 +132,11 @@ class UpdateIdVersion(Thread):
         obj = PathList()
         for idx, path in buf:
             obj.append(idx, path)
+
+            # Store DT bindings documentation files to parse them later
+            if path[:33] == 'Documentation/devicetree/bindings':
+                bindings_idxes.append(idx)
+
             if verbose:
                 print(f"Tag {tag}: adding #{idx} {path}")
         db.vers.put(tag, obj, sync=True)
@@ -325,8 +334,92 @@ class UpdateDocs(Thread):
                     db.docs.put(ident, obj)
 
 
+class UpdateComps(Thread):
+    def __init__(self, start, inc):
+        Thread.__init__(self, name="UpdateCompsElixir")
+        self.index = start
+        self.inc = inc # Equivalient to the number of comps threads
+
+    def run(self):
+        global new_idxes, tags_done, tags_comps, tags_comps_lock
+
+        while(not (tags_done and self.index >= len(new_idxes))):
+            if(self.index >= len(new_idxes)):
+                # Wait for new tags
+                with tag_ready:
+                    tag_ready.wait()
+                continue
+
+            new_idxes[self.index][1].wait() # Make sure the tag is ready
+
+            with tags_comps_lock:
+                tags_comps += 1
+
+            self.update_compatibles(new_idxes[self.index][0])
+
+            self.update_compatibles_bindings(new_idxes[self.index][0])
+
+            self.index += self.inc
+
+    def update_compatibles(self, idxes):
+        global hash_file_lock, comps_lock, tags_comps
+
+        for idx in idxes:
+            if (idx % 1000 == 0): progress('comps: ' + str(idx) + ' 1/2', tags_comps)
+
+            with hash_file_lock:
+                hash = db.hash.get(idx)
+                filename = db.file.get(idx)
+
+            family = lib.getFileFamily(filename)
+            if family in [None, 'K']: continue
+
+            lines = scriptLines('parse-comps', hash, filename, family)
+            with comps_lock:
+                for l in lines:
+                    ident, line = l.split(b' ')
+                    line = int(line.decode())
+
+                    if db.comps.exists(ident):
+                        obj = db.comps.get(ident)
+                    else:
+                        obj = data.RefList()
+
+                    obj.append(idx, str(line), family)
+                    if verbose:
+                        print(f"comps: {ident} in #{idx} @ {line}")
+                    db.comps.put(ident, obj)
+
+    def update_compatibles_bindings(self, idxes):
+        global hash_file_lock, comps_lock, tags_comps, bindings_idxes
+
+        for idx in idxes:
+            if (idx % 1000 == 0): progress('comps: ' + str(idx) + ' 2/2', tags_comp)
+
+            if not idx in bindings_idxes: # Parse only bindings doc files
+                continue
+
+            with hash_file_lock:
+                hash = db.hash.get(idx)
+                filename = db.file.get(idx)
+
+            family = 'B'
+            lines = scriptLines('parse-comps', hash, filename, family)
+            with comps_lock:
+                for l in lines:
+                    ident, line = l.split(b' ')
+                    line = int(line.decode())
+
+                    if db.comps.exists(ident):
+                        obj = db.comps.get(ident)
+                        obj.append(idx, str(line), family)
+                        if verbose:
+                            print(f"comps: {ident} in #{idx} @ {line}")
+                        db.comps.put(ident, obj)
+
+
 def progress(msg, current):
-    print('{} - {} ({:.0%})'.format(project, msg, current/num_tags))
+    print('{} - {} ({:.1%})'.format(project, msg, current/num_tags))
 
 
 # Main
@@ -335,23 +428,32 @@ def progress(msg, current):
 if len(argv) >= 2 and argv[1].isdigit() :
     cpu = int(argv[1])
 
-    if cpu < 3 :
-        cpu = 3
+    if cpu < 4 :
+        cpu = 4
 
 # Distribute threads among functions using the following rules :
-# There are more refs threads than others
-# There are more (or equal) defs threads than docs threads
-# Example : if cpu=6 : refs=3, defs=2, docs=1
-#           if cpu=7 : refs=3, defs=2, docs=2
-#           if cpu=8 : refs=4, defs=2, docs=2
-#           if cpu=11: refs=5, defs=3, docs=3
-quo, rem = divmod(cpu, 2)
+# There are more (or equal) refs threads than others
+# There are more (or equal) defs threads than docs or comps threads
+# Example : if cpu=6 : defs=2, refs=2, docs=1, comps=1
+#           if cpu=7 : defs=2, refs=3, docs=1, comps=1
+#           if cpu=8 : defs=2, refs=2, docs=2, comps=2
+#           if cpu=11: defs=3, refs=4, docs=2, comps=2
+quo, rem = divmod(cpu, 4)
 num_th_refs = quo
-
-quo, rem = divmod(quo+rem, 2)
-num_th_defs = quo + rem
+num_th_defs = quo
 num_th_docs = quo
 
+# If DT bindings support is enabled, use $quo threads for that
+# Otherwise add them to the remaining threads
+if int(script('dts-comp')):
+    num_th_comps = quo
+else :
+    num_th_comps = 0
+    rem += quo
+
+quo, rem = divmod(rem, 2)
+num_th_defs += quo
+num_th_refs += quo + rem
 
 tag_buf = []
 for tag in scriptLines('list-tags'):
@@ -374,6 +476,9 @@ for i in range(num_th_refs):
 # Define docs threads
 for i in range(num_th_docs):
     threads_list.append(UpdateDocs(i, num_th_docs))
+# Define comps threads
+for i in range(num_th_comps):
+    threads_list.append(UpdateComps(i, num_th_comps))
 
 # Start to process tags
 id_version_thread.start()