diff --git a/librz/include/rz_diff.h b/librz/include/rz_diff.h index c28a547748..cafe0c47bb 100644 --- a/librz/include/rz_diff.h +++ b/librz/include/rz_diff.h @@ -67,7 +67,7 @@ RZ_API char *rz_diff_buffers_to_string(RzDiff *d, const ut8 *a, int la, const ut RZ_API int rz_diff_set_callback(RzDiff *d, RzDiffCallback callback, void *user); RZ_API bool rz_diff_buffers_distance(RzDiff *d, const ut8 *a, ut32 la, const ut8 *b, ut32 lb, ut32 *distance, double *similarity); RZ_API bool rz_diff_buffers_distance_myers(RzDiff *diff, const ut8 *a, ut32 la, const ut8 *b, ut32 lb, ut32 *distance, double *similarity); -RZ_API bool rz_diff_buffers_distance_levenstein(RzDiff *d, const ut8 *a, ut32 la, const ut8 *b, ut32 lb, ut32 *distance, double *similarity); +RZ_API bool rz_diff_buffers_distance_levenshtein(RzDiff *d, const ut8 *a, ut32 la, const ut8 *b, ut32 lb, ut32 *distance, double *similarity); RZ_API char *rz_diff_buffers_unified(RzDiff *d, const ut8 *a, int la, const ut8 *b, int lb); /* static method !??! */ RZ_API int rz_diff_lines(const char *file1, const char *sa, int la, const char *file2, const char *sb, int lb); diff --git a/librz/main/rz_diff.c b/librz/main/rz_diff.c index 4caf906ebe..a509a0d373 100644 --- a/librz/main/rz_diff.c +++ b/librz/main/rz_diff.c @@ -7,9 +7,8 @@ enum { MODE_DIFF, MODE_DIFF_STRS, MODE_DIFF_IMPORTS, - MODE_DIST, MODE_DIST_MYERS, - MODE_DIST_LEVENSTEIN, + MODE_DIST_LEVENSHTEIN, MODE_CODE, MODE_GRAPH, MODE_COLS, @@ -1038,12 +1037,10 @@ RZ_API int rz_main_rz_diff(int argc, const char **argv) { case 'h': return show_help(1); case 's': - if (ro.mode == MODE_DIST) { - ro.mode = MODE_DIST_LEVENSTEIN; - } else if (ro.mode == MODE_DIST_LEVENSTEIN) { - ro.mode = MODE_DIST_MYERS; + if (ro.mode == MODE_DIST_MYERS) { + ro.mode = MODE_DIST_LEVENSHTEIN; } else { - ro.mode = MODE_DIST; + ro.mode = MODE_DIST_MYERS; } break; case 'S': @@ -1278,18 +1275,15 @@ RZ_API int rz_main_rz_diff(int argc, const char **argv) { } rz_diff_free(d); break; - case MODE_DIST: case MODE_DIST_MYERS: - case MODE_DIST_LEVENSTEIN: { + case MODE_DIST_LEVENSHTEIN: { RzDiff *d = rz_diff_new(); if (d) { d->verbose = ro.verbose; - if (ro.mode == MODE_DIST_LEVENSTEIN) { - d->type = 'l'; - } else if (ro.mode == MODE_DIST_MYERS) { + if (ro.mode == MODE_DIST_MYERS) { d->type = 'm'; } else { - d->type = 0; + d->type = 'l'; } rz_diff_buffers_distance(d, bufa, (ut32)sza, bufb, (ut32)szb, &ro.count, &sim); rz_diff_free(d); diff --git a/librz/util/udiff.c b/librz/util/udiff.c index 38015a094d..6ea9e27580 100644 --- a/librz/util/udiff.c +++ b/librz/util/udiff.c @@ -166,218 +166,6 @@ RZ_API int rz_diff_buffers(RzDiff *d, const ut8 *a, ut32 la, const ut8 *b, ut32 : rz_diff_buffers_static(d, a, la, b, lb); } -RZ_API bool rz_diff_buffers_distance_levenstein(RzDiff *d, const ut8 *a, ut32 la, const ut8 *b, ut32 lb, ut32 *distance, double *similarity) { - rz_return_val_if_fail(a && b, false); - const bool verbose = d ? d->verbose : false; - /* - More memory efficient version on Levenshtein Distance from: - https://en.wikipedia.org/wiki/Levenshtein_distance - http://www.codeproject.com/Articles/13525/Fast-memory-efficient-Levenshtein-algorithm - ObM.. - - 8/July/2016 - More time efficient Levenshtein Distance. Now runs in about O(N*sum(MDistance)) instead of O(NM) - In real world testing the speedups for similar files are immense. Processing of - rz_diff -sV routerA/firmware_extract/bin/httpd routerB/firmware_extract/bin/httpd - reduced from 28 hours to about 13 minutes. - */ - int i, j; - const ut8 *aBufPtr; - const ut8 *bBufPtr; - ut32 aLen; - ut32 bLen; - - // temp pointer will be used to switch v0 and v1 after processing the inner loop. - int *temp; - int *v0, *v1; - - // We need these variables outside the context of the loops as we need to - // survive multiple loop iterations. - // start and stop are used in our inner loop - // colMin tells us the current 'best' edit distance. - // extendStop & extendStart are used when we get 'double up' edge conditions - // that require us to keep some more data. - int start = 0; - int stop = 0; - int smallest; - int colMin = 0; - int extendStop = 0; - int extendStart = 0; - - //we could move cost into the 'i' loop. - int cost = 0; - - // loops can get very big, this can be removed, but it's currently in there for debugging - // and optimisation testing. - ut64 loops = 0; - - // We need the longest file to be 'A' because our optimisation tries to stop and start - // around the diagonal. - // AAAAAAA - // B* - // B * - // B *____ - // if we have them the other way around and we terminate on the diagonal, we won't have - // inspected all the bytes of file B.. - // AAAA - // B* - // B * - // B * - // B * - // B ? - - if (la < lb) { - aBufPtr = b; - bBufPtr = a; - aLen = lb; - bLen = la; - } else { - aBufPtr = a; - bBufPtr = b; - aLen = la; - bLen = lb; - } - stop = bLen; - // Preliminary tests - - // one or both buffers empty? - if (aLen == 0 || bLen == 0) { - if (distance) { - *distance = RZ_MAX(aLen, bLen); - } - if (similarity) { - *similarity = aLen == bLen ? 1.0 : 0.0; - } - return true; - } - - //IF the files are the same size and are identical, then we have matching files - if (aLen == bLen && !memcmp(aBufPtr, bBufPtr, aLen)) { - if (distance) { - *distance = 0; - } - if (similarity) { - *similarity = 1.0; - } - return true; - } - // Only calloc if we have to do some processing - - // calloc v0 & v1 and check they initialised - v0 = (int *)calloc((bLen + 3), sizeof(int)); - if (!v0) { - eprintf("Error: cannot allocate %i bytes.", bLen + 3); - return false; - } - - v1 = (int *)calloc((bLen + 3), sizeof(int)); - if (!v1) { - eprintf("Error: cannot allocate %i bytes", 2 * (bLen + 3)); - free(v0); - return false; - } - - // initialise v0 and v1. - // With optimisiation we only strictly we only need to initialise v0[0..2]=0..2 & v1[0] = 1; - for (i = 0; i < bLen + 1; i++) { - v0[i] = i; - v1[i] = i + 1; - } - - // Outer loop = the length of the longest input file. - for (i = 0; i < aLen; i++) { - - // We're going to stop the inner loop at: - // bLen (so we don't run off the end of our array) - // or 'two below the diagonal' PLUS any extension we need for 'double up' edge values - // (see extendStop for logic) - stop = RZ_MIN((i + extendStop + 2), bLen); - - // We need a value in the result column (v1[start]). - // If you look at the loop below, we need it because we look at v1[j] as one of the - // potential shortest edit distances. - // In all cases where the edit distance can't 'reach', - // the value of v1[start] simply increments. - if (start > bLen) { - break; - } - v1[start] = v0[start] + 1; - - // need to have a bigger number in colMin than we'll ever encounter in the inner loop - colMin = aLen; - - // Inner loop does all the work: - for (j = start; j <= stop; j++) { - loops++; - - // The main levenshtein comparison: - cost = (aBufPtr[i] == bBufPtr[j]) ? 0 : 1; - smallest = RZ_MIN((v1[j] + 1), (v0[j + 1] + 1)); - smallest = RZ_MIN(smallest, (v0[j] + cost)); - - // populate the next two entries in v1. - // only really required if this is the last loop. - if (j + 2 > bLen + 3) { - break; - } - v1[j + 1] = smallest; - v1[j + 2] = smallest + 1; - - // If we have seen a smaller number, it's the new column Minimum - colMin = RZ_MIN((colMin), (smallest)); - } - - // We're going to start at i+1 next iteration - // The column minimum is the current edit distance - // This distance is the minimum 'search width' from the optimal 'i' diagonal - // The extendStart picks up an edge case where we have a match on the first iteration - // We update extendStart after we've set start for the next iteration. - start = i + 1 - colMin - extendStart; - - // If the last processed entry is a match, AND - // the current byte in 'a' and the previous processed entry in 'b' aren't a match - // then we need to extend our search below the optimal 'i' diagonal. because we'll - // have a vertical double up condition in our last two values of the results column. - // j-2 is used because j++ increments prior to loop exit in the processing loop above. - if (!cost && aBufPtr[i] != bBufPtr[j - 2]) { - extendStop++; - } - - // If new start would be a match then we have a horizontal 'double up' - // which means we need to keep an extra row of data - // so don't increment the start counter this time, BUT keep - // extendStart up our sleeves for next iteration. - if (i + 1 < aLen && start < bLen && aBufPtr[i + 1] == bBufPtr[start]) { - start--; - extendStart++; - } - //Switch v0 and v1 pointers via temp pointer - temp = v0; - v0 = v1; - v1 = temp; - - //Print a processing update every 10K of outer loop - if (verbose && i % 10000 == 0) { - eprintf("\rProcessing %d of %d\r", i, aLen); - } - } - //Clean up output on loop exit (purely aesthetic) - if (verbose) { - eprintf("\rProcessing %d of %d (loops=%" PFMT64d ")\n", i, aLen, loops); - } - if (distance) { - // the final distance is the last byte we processed in the inner loop. - // v0 is used instead of v1 because we switched the pointers before exiting the outer loop - *distance = v0[stop]; - } - if (similarity) { - double diff = (double)(v0[stop]) / (double)(RZ_MAX(aLen, bLen)); - *similarity = (double)1 - diff; - } - free(v0); - free(v1); - return true; -} - // Eugene W. Myers' O(ND) diff algorithm // Returns edit distance with costs: insertion=1, deletion=1, no substitution RZ_API bool rz_diff_buffers_distance_myers(RzDiff *diff, const ut8 *a, ut32 la, const ut8 *b, ut32 lb, ut32 *distance, double *similarity) { @@ -437,7 +225,7 @@ out: return true; } -RZ_API bool rz_diff_buffers_distance_original(RzDiff *diff, const ut8 *a, ut32 la, const ut8 *b, ut32 lb, ut32 *distance, double *similarity) { +RZ_API bool rz_diff_buffers_distance_levenstein(RzDiff *diff, const ut8 *a, ut32 la, const ut8 *b, ut32 lb, ut32 *distance, double *similarity) { if (!a || !b) { return false; } @@ -501,12 +289,11 @@ RZ_API bool rz_diff_buffers_distance(RzDiff *d, const ut8 *a, ut32 la, const ut8 case 'm': return rz_diff_buffers_distance_myers(d, a, la, b, lb, distance, similarity); case 'l': - return rz_diff_buffers_distance_levenstein(d, a, la, b, lb, distance, similarity); default: break; } } - return rz_diff_buffers_distance_original(d, a, la, b, lb, distance, similarity); + return rz_diff_buffers_distance_levenstein(d, a, la, b, lb, distance, similarity); } // Use Needleman–Wunsch to diffchar. diff --git a/man/rz-diff.1 b/man/rz-diff.1 index 90e8ac5656..b3ae361f01 100644 --- a/man/rz-diff.1 +++ b/man/rz-diff.1 @@ -58,9 +58,9 @@ Show two column hexdump diffing. .It Fl X Show two column hexII diffing. .It Fl s -Calculate text distance from two files. +Compute edit distance (no substitution, Eugene W. Myers' O(ND) diff algorithm) between two files. .It Fl ss -Same as before but using the Levenstein algorithm (faster but sometimes buggy) +Compute Levenshtein edit distance (substitution is allowed, O(N^2)) between two files. .It Fl S Ar [name, namelen, dist, size, ...] Specify which column of the code diffing algo use for diffing .It Fl t Ar 0\-100 diff --git a/test/unit/test_diff.c b/test/unit/test_diff.c index c9ff4b5dab..d59c1cf83f 100644 --- a/test/unit/test_diff.c +++ b/test/unit/test_diff.c @@ -33,23 +33,14 @@ bool test_rz_diff_buffers_distance(void) { int i; // Levenshtein edit distance (deletion/insertion/substitution) - diff->type = '\0'; + diff->type = 'l'; for (i = 0; tests[i].a; i++) { size_t la = strlen((const char *)tests[i].a), lb = strlen((const char *)tests[i].b); rz_diff_buffers_distance(diff, tests[i].a, la, tests[i].b, lb, &distance, NULL); - snprintf(msg, sizeof msg, "original %s/%s distance", tests[i].a, tests[i].b); + snprintf(msg, sizeof msg, "levenshtein %s/%s distance", tests[i].a, tests[i].b); mu_assert_eq(distance, tests[i].dis_distance, msg); } - // Broken rz_diff_buffers_distance_levenshtein, uncomment and see why it is incorrect - // diff->type = 'l'; - // for (i = 0; i < RZ_ARRAY_SIZE (tests); i++) { - // size_t la = strlen (tests[i].a), lb = strlen ((const char *)tests[i].b); - // rz_diff_buffers_distance (diff, tests[i].a, la, tests[i].b, lb, &distance, NULL); - // snprintf (msg, sizeof msg, "levenshtein %s/%s distance", tests[i].a, tests[i].b); - // mu_assert_eq (distance, tests[i].dis_distance, msg); - // } - // Eugene W. Myers' O(ND) diff algorithm, deletion/insertion edit distance diff->type = 'm'; for (i = 0; tests[i].a; i++) { @@ -68,4 +59,4 @@ int all_tests() { return tests_passed != tests_run; } -mu_main(all_tests) \ No newline at end of file +mu_main(all_tests)