Add doc-comment extraction
- find-file-doc-comments.pl: new file - data.py: Add database to store doc-comment locations - script.sh: Add parse-docs subcommand - update.py: - Add code to process doc comments - Update some variable names in hopes of reducing confusion - query.py: - Add code to report doc comments - Update some variable names in hopes of reducing confusion Also: - t/TestEnvironment.pm: Add find_doc attribute - t/interact.pl: Don't die if update.py fails - t/TestHelpers.pm: Permit checking specific sections of query.py output - t/300: update regexes per the preceding - gitignore tags (ctags output) and .cache (api_test.py output)
This commit is contained in:
Christopher White
parent
60f206c9fa
commit
8a6031c8a1
10 changed files with 394 additions and 68 deletions
5
.gitignore
vendored
5
.gitignore
vendored
|
|
@ -1,4 +1,9 @@
|
|||
# Generated files
|
||||
__pycache__
|
||||
tags
|
||||
/.cache/
|
||||
|
||||
# Web-specific
|
||||
http/images
|
||||
http/*.html
|
||||
http/favicon.ico
|
||||
|
|
|
|||
11
data.py
11
data.py
|
|
@ -49,6 +49,8 @@ defTypeD = {v: k for k, v in defTypeR.items()}
|
|||
maxId = 999999999
|
||||
|
||||
class DefList:
|
||||
'''Stores associations between a blob ID, a type (e.g., "function"),
|
||||
and a line number.'''
|
||||
def __init__(self, data=b''):
|
||||
self.data = data
|
||||
|
||||
|
|
@ -75,6 +77,8 @@ class DefList:
|
|||
return self.data
|
||||
|
||||
class PathList:
|
||||
'''Stores associations between a blob ID and a file path.
|
||||
Inserted by update.py sorted by blob ID.'''
|
||||
def __init__(self, data=b''):
|
||||
self.data = data
|
||||
|
||||
|
|
@ -96,6 +100,7 @@ class PathList:
|
|||
return self.data
|
||||
|
||||
class RefList:
|
||||
'''Stores a mapping from blob ID to list of lines.'''
|
||||
def __init__(self, data=b''):
|
||||
self.data = data
|
||||
|
||||
|
|
@ -162,9 +167,15 @@ class DB:
|
|||
ro = readonly
|
||||
|
||||
self.vars = BsdDB(dir + '/variables.db', ro, lambda x: int(x.decode()) )
|
||||
# Key-value store of basic information
|
||||
self.blob = BsdDB(dir + '/blobs.db', ro, lambda x: int(x.decode()) )
|
||||
# Map hash to sequential integer serial number
|
||||
self.hash = BsdDB(dir + '/hashes.db', ro, lambda x: x )
|
||||
# Map serial number back to hash
|
||||
self.file = BsdDB(dir + '/filenames.db', ro, lambda x: x.decode() )
|
||||
# Map serial number to filename
|
||||
self.vers = BsdDB(dir + '/versions.db', ro, PathList)
|
||||
self.defs = BsdDB(dir + '/definitions.db', ro, DefList)
|
||||
self.refs = BsdDB(dir + '/references.db', ro, RefList)
|
||||
self.docs = BsdDB(dir + '/doccomments.db', ro, RefList)
|
||||
# Use a RefList in case there are multiple doc comments for an identifier
|
||||
|
|
|
|||
100
find-file-doc-comments.pl
Executable file
100
find-file-doc-comments.pl
Executable file
|
|
@ -0,0 +1,100 @@
|
|||
#!/usr/bin/env perl
|
||||
# find-file-doc-comments.pl: Find the doc comments for a file.
|
||||
# Usage: find-file-doc-comments.pl <C source file name>
|
||||
# By Christopher White <cwhite@d3engineering.com>
|
||||
# Copyright (c) 2019 D3 Engineering, LLC.
|
||||
# Licensed AGPLv3
|
||||
|
||||
use 5.010001;
|
||||
use strict;
|
||||
use warnings;
|
||||
use autodie;
|
||||
|
||||
my $VERBOSE = $ENV{V};
|
||||
|
||||
exit main(@ARGV);
|
||||
|
||||
sub main {
|
||||
die "Need a filename" unless @_;
|
||||
|
||||
# Do `script.sh parse-defs` on the file
|
||||
my @ctags = qx{ ctags -x --c-kinds=+p-m --language-force=C "$_[0]" |
|
||||
grep -av "^operator " |
|
||||
awk '{print \$1" "\$2" "\$3}' };
|
||||
die "Could not get ctags: $!" if $!;
|
||||
print "No ctags results" if $VERBOSE && !@ctags;
|
||||
return 0 unless @ctags;
|
||||
|
||||
# Make a list of [name, type, line] arrays
|
||||
my @ctags_parsed = map { [split] } @ctags;
|
||||
|
||||
# Flip it around to index functions by line
|
||||
my %function_lines;
|
||||
for my $tag (@ctags_parsed) {
|
||||
next unless $tag->[1] eq 'function';
|
||||
$function_lines{$tag->[2]} = $tag->[0];
|
||||
}
|
||||
|
||||
if($VERBOSE) {
|
||||
for my $tag (@ctags_parsed) {
|
||||
say $tag->[2], ': ', $tag->[0], ' is a(n) ', $tag->[1];
|
||||
}
|
||||
}
|
||||
|
||||
# Read the source file
|
||||
open my $fh, '<', $_[0];
|
||||
my @source_lines = (undef, <$fh>);
|
||||
# undef => indices in @source_lines match ctags's 1-based linenos
|
||||
close $fh;
|
||||
|
||||
# Work backwards through the file and look for doc comments
|
||||
my %doc_comments;
|
||||
|
||||
my $doc_comment_opener = qr{^\s*\/\*\*(?:\s|$)}; # Start of doc comment
|
||||
|
||||
for(my $lineno = $#source_lines ; $lineno >= 1 ; --$lineno) {
|
||||
next unless exists $function_lines{$lineno};
|
||||
my $func_name = $function_lines{$lineno};
|
||||
print "Checking for $func_name @ $lineno\n" if $VERBOSE;
|
||||
|
||||
my $this_doc_comment_header =
|
||||
qr{^\s+\*\s+\Q$func_name\E(?:\s|\(|$)};
|
||||
print " Regex is -$this_doc_comment_header-\n" if $VERBOSE;
|
||||
--$lineno;
|
||||
print " Line $lineno is: $source_lines[$lineno]\n" if $VERBOSE;
|
||||
|
||||
# Find the last line that could be a doc-comment header
|
||||
# for this function.
|
||||
while($source_lines[$lineno] =~
|
||||
qr{
|
||||
^\s*$ # Empty line
|
||||
| ^\s+\*\/ # End of comment
|
||||
| ^\s+\*(?:\s|$) # Continuation of comment
|
||||
| $this_doc_comment_header
|
||||
}x) {
|
||||
print "$source_lines[$lineno] passed\n" if $VERBOSE;
|
||||
--$lineno;
|
||||
}
|
||||
++$lineno; # Check the last line that matched,
|
||||
# because we may have just skipped past $this_doc_comment_header
|
||||
|
||||
# Is it actually a header for this function?
|
||||
print "Checking $source_lines[$lineno] for header\n" if $VERBOSE;
|
||||
next unless $source_lines[$lineno] =~ $this_doc_comment_header;
|
||||
|
||||
# We have found a header. Confirm it's a doc comment.
|
||||
--$lineno;
|
||||
next unless $source_lines[$lineno] =~ $doc_comment_opener;
|
||||
print " * Match\n" if $VERBOSE;
|
||||
|
||||
# We have found a doc comment for this function! Record it.
|
||||
push @{$doc_comments{$func_name}}, $lineno;
|
||||
}
|
||||
|
||||
# Report the doc comments for each function
|
||||
while(my ($funcname, $comment_lines) = each %doc_comments) {
|
||||
print "$funcname $_\n" foreach @$comment_lines;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
68
query.py
68
query.py
|
|
@ -146,37 +146,60 @@ def query(cmd, *args):
|
|||
|
||||
symbol_definitions = []
|
||||
symbol_references = []
|
||||
symbol_doccomments = []
|
||||
|
||||
if not db.defs.exists(ident):
|
||||
return symbol_definitions, symbol_references
|
||||
return symbol_definitions, symbol_references, symbol_doccomments
|
||||
|
||||
if not db.vers.exists(version):
|
||||
return symbol_definitions, symbol_references
|
||||
return symbol_definitions, symbol_references, symbol_doccomments
|
||||
|
||||
vers = db.vers.get(version).iter()
|
||||
defs = db.defs.get(ident).iter(dummy=True)
|
||||
# FIXME: see why we can have a discrepancy between defs and refs
|
||||
files_this_version = db.vers.get(version).iter()
|
||||
defs_this_ident = db.defs.get(ident).iter(dummy=True)
|
||||
# FIXME: see why we can have a discrepancy between defs_this_ident and refs
|
||||
if db.refs.exists(ident):
|
||||
refs = db.refs.get(ident).iter(dummy=True)
|
||||
else:
|
||||
refs = data.RefList().iter(dummy=True)
|
||||
|
||||
id2, type, dline = next(defs)
|
||||
id3, rlines = next(refs)
|
||||
if db.docs.exists(ident):
|
||||
docs = db.docs.get(ident).iter(dummy=True)
|
||||
else:
|
||||
docs = data.RefList().iter(dummy=True)
|
||||
|
||||
# vers, defs, refs, and docs are all populated by update.py in order of
|
||||
# idx, and there is a one-to-one mapping between blob hashes and idx
|
||||
# values. Therefore, we can sequentially step through the defs, refs,
|
||||
# and docs for each file in a version.
|
||||
|
||||
def_idx, def_type, def_line = next(defs_this_ident)
|
||||
ref_idx, ref_lines = next(refs)
|
||||
doc_idx, doc_line = next(docs)
|
||||
|
||||
dBuf = []
|
||||
rBuf = []
|
||||
docBuf = []
|
||||
|
||||
for file_idx, file_path in files_this_version:
|
||||
# Advance defs, refs, and docs to the current file
|
||||
while def_idx < file_idx:
|
||||
def_idx, def_type, def_line = next(defs_this_ident)
|
||||
while ref_idx < file_idx:
|
||||
ref_idx, ref_lines = next(refs)
|
||||
while doc_idx < file_idx:
|
||||
doc_idx, doc_line = next(docs)
|
||||
|
||||
# Copy information about this identifier into dBuf, rBuf, and docBuf.
|
||||
while def_idx == file_idx:
|
||||
dBuf.append((file_path, def_type, def_line))
|
||||
def_idx, def_type, def_line = next(defs_this_ident)
|
||||
|
||||
if ref_idx == file_idx:
|
||||
rBuf.append((file_path, ref_lines))
|
||||
|
||||
if doc_idx == file_idx: # TODO should this be a `while`?
|
||||
docBuf.append((file_path, doc_line))
|
||||
|
||||
for id1, path in vers:
|
||||
while id1 > id2:
|
||||
id2, type, dline = next(defs)
|
||||
while id1 > id3:
|
||||
id3, rlines = next(refs)
|
||||
while id1 == id2:
|
||||
dBuf.append((path, type, dline))
|
||||
id2, type, dline = next(defs)
|
||||
if id1 == id3:
|
||||
rBuf.append((path, rlines))
|
||||
|
||||
for path, type, dline in sorted(dBuf):
|
||||
symbol_definitions.append(SymbolInstance(path, dline, type))
|
||||
|
|
@ -184,13 +207,16 @@ def query(cmd, *args):
|
|||
for path, rlines in sorted(rBuf):
|
||||
symbol_references.append(SymbolInstance(path, rlines))
|
||||
|
||||
return symbol_definitions, symbol_references
|
||||
for path, docline in sorted(docBuf):
|
||||
symbol_doccomments.append(SymbolInstance(path, docline))
|
||||
|
||||
return symbol_definitions, symbol_references, symbol_doccomments
|
||||
|
||||
else:
|
||||
return('Unknown subcommand: ' + cmd + '\n')
|
||||
|
||||
def cmd_ident(version, ident, **kwargs):
|
||||
symbol_definitions, symbol_references = query("ident", version, ident)
|
||||
symbol_definitions, symbol_references, symbol_doccomments = query("ident", version, ident)
|
||||
print("Symbol Definitions:")
|
||||
for symbol_definition in symbol_definitions:
|
||||
print(symbol_definition)
|
||||
|
|
@ -199,6 +225,10 @@ def cmd_ident(version, ident, **kwargs):
|
|||
for symbol_reference in symbol_references:
|
||||
print(symbol_reference)
|
||||
|
||||
print("\nDocumented in:")
|
||||
for symbol_doccomment in symbol_doccomments:
|
||||
print(symbol_doccomment)
|
||||
|
||||
def cmd_file(version, path, **kwargs):
|
||||
code = query("file", version, path)
|
||||
print(code)
|
||||
|
|
|
|||
25
script.sh
25
script.sh
|
|
@ -2,8 +2,8 @@
|
|||
|
||||
# This file is part of Elixir, a source code cross-referencer.
|
||||
#
|
||||
# Copyright (C) 2017 Mikaël Bouillot
|
||||
# <mikael.bouillot@bootlin.com>
|
||||
# Copyright (C) 2017--2020 Mikaël Bouillot
|
||||
# <mikael.bouillot@bootlin.com> and contributors
|
||||
#
|
||||
# Elixir is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -23,6 +23,13 @@ if [ ! -d "$LXR_REPO_DIR" ]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# Get our path so we can find peer find-file-doc-comments.pl later
|
||||
cur_dir=`pwd`
|
||||
script_path=`realpath "$0"`
|
||||
cd `dirname "$script_path"`
|
||||
script_dir=`pwd`
|
||||
cd "$cur_dir"
|
||||
|
||||
version_dir()
|
||||
{
|
||||
cat;
|
||||
|
|
@ -140,6 +147,16 @@ parse_defs()
|
|||
rmdir $tmp
|
||||
}
|
||||
|
||||
parse_docs()
|
||||
{
|
||||
tmpfile=`mktemp`
|
||||
|
||||
git cat-file blob "$opt1" > "$tmpfile"
|
||||
"$script_dir/find-file-doc-comments.pl" "$tmpfile"
|
||||
|
||||
rm -rf "$tmpfile"
|
||||
}
|
||||
|
||||
project=$(basename `dirname $LXR_REPO_DIR`)
|
||||
|
||||
plugin=projects/$project.sh
|
||||
|
|
@ -208,6 +225,10 @@ case $cmd in
|
|||
parse_defs
|
||||
;;
|
||||
|
||||
parse-docs)
|
||||
parse_docs
|
||||
;;
|
||||
|
||||
help)
|
||||
echo "Usage: $0 subcommand [args]..."
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -48,7 +48,15 @@ run_produces_ok('doc-comment query (nonexistent)',
|
|||
[$tenv->query_py, qw(v5.4 ident SOME_NONEXISTENT_IDENTIFIER_XYZZY_PLUGH)],
|
||||
[
|
||||
qr{^Documented in:},
|
||||
{ not => qr{/} } # No file paths in the output
|
||||
{doc => { not => qr{/} }}, # No file paths in the doc section
|
||||
],
|
||||
MUST_SUCCEED);
|
||||
|
||||
run_produces_ok('doc-comment query (existent but not documented)',
|
||||
[$tenv->query_py, qw(v5.4 ident gsb_buffer)], # in drivers/i2c/i2c-core-acpi.c
|
||||
[
|
||||
qr{^Documented in:},
|
||||
{doc => { not => qr{/} }}
|
||||
],
|
||||
MUST_SUCCEED);
|
||||
|
||||
|
|
@ -56,7 +64,7 @@ run_produces_ok('ident query (existent, function, documented in C file)',
|
|||
[$tenv->query_py, qw(v5.4 ident i2c_acpi_get_i2c_resource)],
|
||||
[
|
||||
qr{^Documented in:},
|
||||
qr{drivers/i2c/i2c-core-acpi\.c.+\b45\b},
|
||||
{doc => qr{drivers/i2c/i2c-core-acpi\.c.+\b45\b}},
|
||||
],
|
||||
MUST_SUCCEED);
|
||||
|
||||
|
|
|
|||
|
|
@ -69,6 +69,10 @@ As L</script_sh>, but for C<query.py>.
|
|||
|
||||
As L</script_sh>, but for C<update.py>.
|
||||
|
||||
=head2 find_doc
|
||||
|
||||
As L</script_sh>, but for C<find-file-doc-comments.pl>.
|
||||
|
||||
=cut
|
||||
|
||||
has lxr_data_dir => ();
|
||||
|
|
@ -82,6 +86,9 @@ has query_py => (
|
|||
has update_py => (
|
||||
default => sub { find_program('update.py') }
|
||||
);
|
||||
has find_doc => (
|
||||
default => sub { find_program('find-file-doc-comments.pl') }
|
||||
);
|
||||
|
||||
# Internal attributes
|
||||
|
||||
|
|
@ -204,11 +211,12 @@ Returns a human-readable report of the current environment's state.
|
|||
sub report {
|
||||
my $self = shift;
|
||||
return <<EOT;
|
||||
Repository: @{[$self->lxr_repo_dir]}
|
||||
Database: @{[$self->lxr_data_dir]}
|
||||
script.sh: @{[$self->script_sh]}
|
||||
update.py: @{[$self->update_py]}
|
||||
query.py: @{[$self->query_py]}
|
||||
Repository: @{[$self->lxr_repo_dir || '<unknown>']}
|
||||
Database: @{[$self->lxr_data_dir || '<unknown>']}
|
||||
script.sh: @{[$self->script_sh || '<unknown>']}
|
||||
update.py: @{[$self->update_py || '<unknown>']}
|
||||
query.py: @{[$self->query_py || '<unknown>']}
|
||||
find-file-doc-comments.pl: @{[$self->find_doc || '<unknown>']}
|
||||
EOT
|
||||
} #report()
|
||||
|
||||
|
|
|
|||
135
t/TestHelpers.pm
135
t/TestHelpers.pm
|
|
@ -48,7 +48,7 @@ BEGIN {
|
|||
%EXPORT_TAGS = (
|
||||
all => [@EXPORT, @EXPORT_OK],
|
||||
);
|
||||
}
|
||||
} #BEGIN
|
||||
|
||||
# Forwards for internal functions
|
||||
sub line_mark_string;
|
||||
|
|
@ -80,7 +80,7 @@ this file. Usage:
|
|||
|
||||
sub sibling_abs_path {
|
||||
return File::Spec->rel2abs(File::Spec->catfile($FindBin::Bin, @_));
|
||||
}
|
||||
} #sibling_abs_path()
|
||||
|
||||
=head2 find_program
|
||||
|
||||
|
|
@ -157,12 +157,47 @@ Usage:
|
|||
run_produces_ok($desc, \@program_and_args, \@expected_regexes,
|
||||
<optional> $mustSucceed, <optional> $printOutput)
|
||||
|
||||
The test passes if each regex in C<@expected_regexes> matches at least one
|
||||
line in the output of C<@program_and_args>, and if each C<< { not => regex } >>
|
||||
in C<@expected_regexes> is NOT found in that output.
|
||||
The test passes if each condition in C<@conditions> is true.
|
||||
If C<$mustSucceed> is true, also tests for exit status 0 and empty stderr.
|
||||
If C<$printOutput> is true, prints the output of C<@program_and_args>.
|
||||
|
||||
=head3 Conditions that can be used any time
|
||||
|
||||
=over
|
||||
|
||||
=item *
|
||||
|
||||
A regex: true if the regex matches at least one line in the output of
|
||||
C<@program_and_args>
|
||||
|
||||
=item *
|
||||
|
||||
C<< { not => regex } >>: true if the regex is NOT found in any line of
|
||||
the output of C<@program_and_args>.
|
||||
|
||||
=back
|
||||
|
||||
=head3 Conditions for the output of C<query.py>
|
||||
|
||||
=over
|
||||
|
||||
=item *
|
||||
|
||||
C<< { def => regex } >>: true if the regex matches at least one line in
|
||||
the "Symbol Definitions" section of the output of C<@program_and_args>.
|
||||
|
||||
=item *
|
||||
|
||||
C<< { ref => regex } >>: true if the regex matches at least one line in
|
||||
the "Symbol References" section of the output of C<@program_and_args>.
|
||||
|
||||
=item *
|
||||
|
||||
C<< { doc => regex } >>: true if the regex matches at least one line in
|
||||
the "Documented in" section of the output of C<@program_and_args>.
|
||||
|
||||
=back
|
||||
|
||||
=cut
|
||||
|
||||
sub run_produces_ok {
|
||||
|
|
@ -211,20 +246,29 @@ EOT
|
|||
}
|
||||
|
||||
# Check regexes
|
||||
my %query_py_output; # filled in only if we see a def/ref/doc
|
||||
for my $entry (@$lrRegexes) {
|
||||
my ($re, $negated, $source) = _parse_condition($entry);
|
||||
|
||||
if(ref $entry eq 'Regexp') {
|
||||
eval line_mark_string 1,
|
||||
q(ok( (grep { m{$entry} } @outlines), "$desc: output includes $entry" ));
|
||||
|
||||
} elsif(ref $entry eq 'HASH' && ref $entry->{not} eq 'Regexp') {
|
||||
my $re = $entry->{not};
|
||||
eval line_mark_string 1,
|
||||
q(ok( !(grep { m{$re} } @outlines), "$desc: output excludes $re" ));
|
||||
# Parse query.py output if we need it and haven't done so
|
||||
%query_py_output = _parseq(@outlines)
|
||||
if $source ne 'output' && !%query_py_output;
|
||||
|
||||
# Build a line of test code to run
|
||||
my $test = 'ok( ';
|
||||
$test .= '!' if $negated;
|
||||
$test .= '(grep { m{$re} } ';
|
||||
if($source eq 'output') {
|
||||
$test .= '@outlines';
|
||||
} else {
|
||||
die "Invalid entry $entry";
|
||||
$test .= '@{$query_py_output{' . $source . '}}';
|
||||
}
|
||||
$test .= '), "$desc: ' . $source;
|
||||
$test .= ($negated ? ' excludes ' : ' includes ') . "\Q$re\E" . '");';
|
||||
|
||||
# Run it
|
||||
#diag "Running $test";
|
||||
eval line_mark_string 1, $test;
|
||||
} #foreach $entry
|
||||
|
||||
} #run_produces_ok()
|
||||
|
|
@ -233,16 +277,75 @@ EOT
|
|||
|
||||
These are ones you probably won't need to call.
|
||||
|
||||
=head2 _parseq
|
||||
|
||||
Parse the output of query.py. Usage:
|
||||
|
||||
%parsed = _parseq(@lines_of_output);
|
||||
|
||||
=cut
|
||||
|
||||
sub _parseq {
|
||||
my %retval = { def => [], ref => [], doc => [] };
|
||||
my $list;
|
||||
foreach(@_) {
|
||||
chomp;
|
||||
if($_ eq 'Symbol Definitions:') {
|
||||
$list = 'def';
|
||||
next;
|
||||
} elsif($_ eq 'Symbol References:') {
|
||||
$list = 'ref';
|
||||
next;
|
||||
} elsif($_ eq 'Documented in:') {
|
||||
$list = 'doc';
|
||||
next;
|
||||
}
|
||||
|
||||
#diag "Adding `$_' to list $list";
|
||||
push @{$retval{$list}}, $_;
|
||||
}
|
||||
return %retval;
|
||||
} #_parseq()
|
||||
|
||||
=head2 _parse_condition
|
||||
|
||||
Parse a condition for L</run_produces_ok>. Usage:
|
||||
|
||||
($regex, $negated, $source) = _parse_condition($entry[, $source]);
|
||||
|
||||
=cut
|
||||
|
||||
sub _parse_condition {
|
||||
my ($entry, $source_in) = @_;
|
||||
my ($regex, $negated, $source); # Return values
|
||||
|
||||
# Basic cases
|
||||
if(ref $entry eq 'Regexp') {
|
||||
return ($entry, 0, $source_in || 'output');
|
||||
} elsif(ref $entry eq 'HASH' && ref $entry->{not} eq 'Regexp') {
|
||||
return ($entry->{not}, 1, $source_in || 'output');
|
||||
}
|
||||
|
||||
# Sub-keys: chain
|
||||
if(ref $entry eq 'HASH' && scalar keys %{$entry} == 1) {
|
||||
return _parse_condition((values %{$entry})[0], (keys %{$entry})[0]);
|
||||
}
|
||||
|
||||
# If we get here, we don't know how to handle it
|
||||
die "Invalid entry $entry";
|
||||
} #_parse_condition()
|
||||
|
||||
|
||||
=head2 _croak
|
||||
|
||||
Lazy L<Carp/croak>
|
||||
Lazy invoker for L<Carp/croak>.
|
||||
|
||||
=cut
|
||||
|
||||
sub _croak {
|
||||
require Carp;
|
||||
goto &Carp::croak;
|
||||
}
|
||||
} #_croak()
|
||||
|
||||
=head2 line_mark_string
|
||||
|
||||
|
|
|
|||
|
|
@ -19,8 +19,6 @@
|
|||
#
|
||||
# This file uses core Perl modules only.
|
||||
|
||||
use autodie;
|
||||
|
||||
use FindBin '$Bin';
|
||||
use lib $Bin;
|
||||
|
||||
|
|
@ -32,7 +30,10 @@ use TestHelpers;
|
|||
|
||||
# These two lines are all that's required to set up for a test.
|
||||
my $tenv = TestEnvironment->new;
|
||||
$tenv->build_repo(sibling_abs_path('tree'))->build_db->update_env;
|
||||
$tenv->build_repo(sibling_abs_path('tree')); # dies on error
|
||||
eval { $tenv->build_db; };
|
||||
warn "Could not update database: $@" if $@;
|
||||
$tenv->update_env;
|
||||
|
||||
print($tenv->report);
|
||||
|
||||
|
|
@ -44,7 +45,9 @@ system(qw(ln -s), $tenv->update_py, 'update.py') == 0
|
|||
or warn "error creating ./update.py: $? $!";
|
||||
system(qw(ln -s), $tenv->query_py, 'query.py') == 0
|
||||
or warn "error creating ./query.py: $? $!";
|
||||
system(qw(ln -s), $tenv->find_doc, 'find-file-doc-comments.pl') == 0
|
||||
or warn "error creating ./find-file-doc-comments.pl: $? $!";
|
||||
|
||||
print("Exit when done, and the repository and database will be removed.\n");
|
||||
system($ENV{SHELL} || 'sh');
|
||||
|
||||
my $retval = system($ENV{SHELL} || 'sh');
|
||||
exit $retval>>8;
|
||||
|
|
|
|||
75
update.py
75
update.py
|
|
@ -18,6 +18,9 @@
|
|||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with Elixir. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# Throughout, an "idx" is the sequential number associated with a blob.
|
||||
# This is different from that blob's Git hash.
|
||||
|
||||
from sys import argv
|
||||
from lib import scriptLines
|
||||
import lib
|
||||
|
|
@ -25,6 +28,8 @@ import data
|
|||
import os
|
||||
from data import PathList
|
||||
|
||||
verbose = False
|
||||
|
||||
db = data.DB(lib.getDataDir(), readonly=False)
|
||||
|
||||
# Store new blobs hashed and file names (without path) for new tag
|
||||
|
|
@ -39,17 +44,19 @@ def updateBlobIDs(tag):
|
|||
# Get blob hashes and associated file names (without path)
|
||||
blobs = scriptLines('list-blobs', '-f', tag)
|
||||
|
||||
newBlobs = []
|
||||
newIdxes = []
|
||||
for blob in blobs:
|
||||
hash, filename = blob.split(b' ',maxsplit=1)
|
||||
if not db.blob.exists(hash):
|
||||
db.blob.put(hash, idx)
|
||||
db.hash.put(idx, hash)
|
||||
db.file.put(idx, filename)
|
||||
newBlobs.append(idx)
|
||||
newIdxes.append(idx)
|
||||
if verbose:
|
||||
print(f"New blob #{idx} {hash}:{filename}")
|
||||
idx += 1
|
||||
db.vars.put('numBlobs', idx)
|
||||
return newBlobs
|
||||
return newIdxes
|
||||
|
||||
def updateVersions(tag):
|
||||
|
||||
|
|
@ -66,13 +73,15 @@ def updateVersions(tag):
|
|||
obj = PathList()
|
||||
for idx, path in buf:
|
||||
obj.append(idx, path)
|
||||
if verbose:
|
||||
print(f"Tag {tag}: adding #{idx} {path}")
|
||||
db.vers.put(tag, obj, sync=True)
|
||||
|
||||
def updateDefinitions(blobs):
|
||||
for blob in blobs:
|
||||
if (blob % 1000 == 0): progress('defs: ' + str(blob))
|
||||
hash = db.hash.get(blob)
|
||||
filename = db.file.get(blob)
|
||||
def updateDefinitions(idxes):
|
||||
for idx in idxes:
|
||||
if (idx % 1000 == 0): progress('defs: ' + str(idx))
|
||||
hash = db.hash.get(idx)
|
||||
filename = db.file.get(idx)
|
||||
|
||||
if not lib.hasSupportedExt(filename): continue
|
||||
|
||||
|
|
@ -87,14 +96,16 @@ def updateDefinitions(blobs):
|
|||
else:
|
||||
obj = data.DefList()
|
||||
|
||||
obj.append(blob, type, line)
|
||||
obj.append(idx, type, line)
|
||||
if verbose:
|
||||
print(f"def {type} {ident} in #{idx} @ {line}");
|
||||
db.defs.put(ident, obj)
|
||||
|
||||
def updateReferences(blobs):
|
||||
for blob in blobs:
|
||||
if (blob % 1000 == 0): progress('refs: ' + str(blob))
|
||||
hash = db.hash.get(blob)
|
||||
filename = db.file.get(blob)
|
||||
def updateReferences(idxes):
|
||||
for idx in idxes:
|
||||
if (idx % 1000 == 0): progress('refs: ' + str(idx))
|
||||
hash = db.hash.get(idx)
|
||||
filename = db.file.get(idx)
|
||||
|
||||
if not lib.hasSupportedExt(filename): continue
|
||||
|
||||
|
|
@ -119,9 +130,34 @@ def updateReferences(blobs):
|
|||
else:
|
||||
obj = data.RefList()
|
||||
|
||||
obj.append(blob, lines)
|
||||
obj.append(idx, lines)
|
||||
if verbose:
|
||||
print(f"ref: {ident} in #{idx} @ {lines}");
|
||||
db.refs.put(ident, obj)
|
||||
|
||||
def updateDocComments(idxes):
|
||||
for idx in idxes:
|
||||
if (idx % 1000 == 0): progress('docs: ' + str(idx))
|
||||
hash = db.hash.get(idx)
|
||||
filename = db.file.get(idx)
|
||||
|
||||
if not lib.hasSupportedExt(filename): continue
|
||||
|
||||
lines = scriptLines('parse-docs', hash, filename)
|
||||
for l in lines:
|
||||
ident, line = l.split(b' ')
|
||||
line = int(line.decode())
|
||||
|
||||
if db.docs.exists(ident):
|
||||
obj = db.docs.get(ident)
|
||||
else:
|
||||
obj = data.RefList()
|
||||
|
||||
obj.append(idx, str(line))
|
||||
if verbose:
|
||||
print(f"doc: {ident} in #{idx} @ {line}");
|
||||
db.docs.put(ident, obj)
|
||||
|
||||
def progress(msg):
|
||||
print('{} - {} ({:.0%})'.format(project, msg, tagCount/numTags))
|
||||
|
||||
|
|
@ -140,8 +176,9 @@ print(project + ' - found ' + str(len(tagBuf)) + ' new tags')
|
|||
|
||||
for tag in tagBuf:
|
||||
tagCount +=1
|
||||
newBlobs = updateBlobIDs(tag)
|
||||
progress(tag.decode() + ': ' + str(len(newBlobs)) + ' new blobs')
|
||||
newIdxes = updateBlobIDs(tag)
|
||||
progress(tag.decode() + ': ' + str(len(newIdxes)) + ' new blobs')
|
||||
updateVersions(tag)
|
||||
updateDefinitions(newBlobs)
|
||||
updateReferences(newBlobs)
|
||||
updateDefinitions(newIdxes)
|
||||
updateReferences(newIdxes)
|
||||
updateDocComments(newIdxes)
|
||||
|
|
|
|||
Loading…
Reference in a new issue