diff --git a/inc/scan.h b/inc/scan.h index ae20de9..4404f7a 100644 --- a/inc/scan.h +++ b/inc/scan.h @@ -47,6 +47,9 @@ typedef struct scan_data_t uint32_t *hashes; /* pointer to wfp hashes*/ uint32_t *lines; /*pointer to line hashes */ uint32_t hash_count; /* count of wfp hashes */ + uint32_t *hashes2; /* pointer to wfp2 (token-aware) hashes, parsed from the wfp2= block */ + uint32_t *lines2; /* pointer to wfp2 line numbers */ + uint32_t hash_count2; /* count of wfp2 hashes */ long timer; /*timer for execution profile*/ bool preload; /*used in hash scanning */ int total_lines; /* total file lines */ diff --git a/inc/scanoss.h b/inc/scanoss.h index f4ab4b5..ef067b5 100644 --- a/inc/scanoss.h +++ b/inc/scanoss.h @@ -33,7 +33,7 @@ #define WFP_LN 4 #define WFP_REC_LN 18 -#define SCANOSS_VERSION "6.1.0-crc64" +#define SCANOSS_VERSION "6.2.0-crc64" /* Log files */ #define SCAN_LOG "/tmp/scanoss_scan.log" @@ -91,6 +91,7 @@ extern char * component_hint; extern struct ldb_table oss_url; extern struct ldb_table oss_file; extern struct ldb_table oss_wfp; +extern struct ldb_table oss_wfp2; extern struct ldb_table oss_purl; extern struct ldb_table oss_copyright; extern struct ldb_table oss_quality; @@ -104,6 +105,9 @@ extern struct ldb_table oss_notices; extern struct ldb_table oss_pivot; extern struct ldb_table oss_path; +/* True when the optional wfp2 table is present in the loaded KB. */ +extern bool wfp2_present; + /* Reconstruct a project's file structure (md5,path per file) from its url hash. */ void get_project_files(char * url_key); diff --git a/inc/wfp2.h b/inc/wfp2.h new file mode 100644 index 0000000..d03f9f4 --- /dev/null +++ b/inc/wfp2.h @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * inc/wfp2.h + * + * WFP2 (token-aware) fingerprint generation. + * + * Copyright (C) 2018-2025 SCANOSS.COM + */ + +#ifndef __WFP2_H +#define __WFP2_H + +#include + +/** + * @brief Compute the WFP2 (token-aware) snippet fingerprints of a source buffer. + * + * Mirrors the reference Go implementation (scanoss-go / go-minr snippetsx): + * language detection -> comment removal -> token-aware normalization -> winnowing + * (GRAM 20, WINDOW 40, CRC32C). The output hashes/lines must match, byte for byte, + * the fingerprints the KB was mined with, otherwise nothing will match the wfp2 table. + * + * @param src source buffer (text) + * @param src_len length of src in bytes + * @param hashes output array of wfp2 hashes + * @param lines output array of line numbers (paired with hashes) + * @param limit maximum number of hashes to emit + * @return number of hashes produced + */ +uint32_t winnowing_wfp2(char *src, long src_len, uint32_t *hashes, uint32_t *lines, long limit); + +#endif diff --git a/src/main.c b/src/main.c index 53eaeb2..ec3bd3d 100644 --- a/src/main.c +++ b/src/main.c @@ -53,6 +53,7 @@ struct ldb_table oss_url; struct ldb_table oss_file; struct ldb_table oss_wfp; +struct ldb_table oss_wfp2; struct ldb_table oss_purl; struct ldb_table oss_copyright; struct ldb_table oss_quality; @@ -66,6 +67,7 @@ struct ldb_table oss_notices; struct ldb_table oss_pivot; struct ldb_table oss_path; bool path_table_present = false; +bool wfp2_present = false; component_item *ignore_components; component_item *declared_components; @@ -144,6 +146,17 @@ void initialize_ldb_tables(char *name) snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "wfp"); oss_wfp = ldb_read_cfg(dbtable); + /* Optional wfp2 table: token-aware fingerprints mined into a separate table, + * leaving the classic wfp table untouched. When present, the engine can run a + * wfp2 snippet pass (fallback by default, additive when forced). */ + if (ldb_table_exists(oss_db_name, "wfp2")) + { + wfp2_present = true; + snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "wfp2"); + oss_wfp2 = ldb_read_cfg(dbtable); + scanlog("wfp2 table present: enabling wfp2 snippet matching\n"); + } + snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "purl"); oss_purl = ldb_read_cfg(dbtable); diff --git a/src/purl_scan.c b/src/purl_scan.c index 2c5026b..a9913da 100644 --- a/src/purl_scan.c +++ b/src/purl_scan.c @@ -454,7 +454,7 @@ int component_scan(char *url_hash_list) if (!quiet) { - printf("], \"server\": {\"version\": \"%s\", \"kb_version\": %s}}\n", SCANOSS_VERSION, kb_version); + printf("], \"server\": {\"version\": \"%s\", \"kb_version\": %s, \"wfp2\": %s}}\n", SCANOSS_VERSION, kb_version, wfp2_present ? "true" : "false"); fflush(stdout); } @@ -478,6 +478,8 @@ static int snippet_scan_stream(FILE *in) char file_path[MAX_FILE_PATH] = ""; uint64_t file_size = 0; bool got_file = false; + /* Snippet lines after a "wfp2=" marker belong to the token-aware wfp2 set */ + bool in_wfp2 = false; scanlog("--- SNIPPET SCAN ---\n"); @@ -492,12 +494,21 @@ static int snippet_scan_stream(FILE *in) bool is_fh2 = (memcmp(line, "fh2=", 4) == 0); bool is_hpsm = (memcmp(line, "hpsm=", 5) == 0); bool is_bin = (memcmp(line, "bin=", 4) == 0); - bool is_wfp = (!is_file && !is_fh2 && !is_hpsm && !is_bin); + bool is_wfp2_marker = (memcmp(line, "wfp2=", 5) == 0); + bool is_wfp = (!is_file && !is_fh2 && !is_hpsm && !is_bin && !is_wfp2_marker); /* Snippet-only mode: ignore hpsm/bin payloads */ if (is_hpsm || is_bin) continue; + /* wfp2 block delimiter: subsequent snippet lines are token-aware wfp2 + fingerprints, matched against the wfp2 table by ldb_scan_snippets */ + if (is_wfp2_marker) + { + in_wfp2 = true; + continue; + } + if (is_file) { if (got_file) @@ -569,6 +580,7 @@ static int snippet_scan_stream(FILE *in) strcpy(scan->source_md5, file_md5_hex); free(rec); got_file = true; + in_wfp2 = false; continue; } @@ -579,24 +591,50 @@ static int snippet_scan_stream(FILE *in) continue; } - if (is_wfp && scan && (scan->hash_count < MAX_HASHES_READ)) + if (is_wfp && scan) { - int line_ln = strlen(line); - for (int e = 0; e < line_ln; e++) - if (line[e] == '=' || line[e] == ',') line[e] = 0; - - int line_nr = atoi(line); - char *hexhash = line + strlen(line) + 1; + /* Route to the classic wfp buffers or, inside a wfp2= block, to the + wfp2 buffers (allocated lazily) — same as the -w (wfp_scan) path. */ + uint32_t *dst_hashes; + uint32_t *dst_lines; + uint32_t *dst_count; + if (in_wfp2) + { + if (!scan->hashes2) + { + scan->hashes2 = calloc(MAX_FILE_SIZE, 1); + scan->lines2 = malloc(MAX_FILE_SIZE); + } + dst_hashes = scan->hashes2; + dst_lines = scan->lines2; + dst_count = &scan->hash_count2; + } + else + { + dst_hashes = scan->hashes; + dst_lines = scan->lines; + dst_count = &scan->hash_count; + } - while (*hexhash) + if (*dst_count < MAX_HASHES_READ) { - ldb_hex_to_bin(hexhash, 8, (uint8_t *) &scan->hashes[scan->hash_count]); - uint32_reverse((uint8_t *) &scan->hashes[scan->hash_count]); - scan->lines[scan->hash_count] = line_nr; - hexhash += strlen(hexhash) + 1; - scan->hash_count++; - if (scan->hash_count >= MAX_HASHES_READ) - break; + int line_ln = strlen(line); + for (int e = 0; e < line_ln; e++) + if (line[e] == '=' || line[e] == ',') line[e] = 0; + + int line_nr = atoi(line); + char *hexhash = line + strlen(line) + 1; + + while (*hexhash) + { + ldb_hex_to_bin(hexhash, 8, (uint8_t *) &dst_hashes[*dst_count]); + uint32_reverse((uint8_t *) &dst_hashes[*dst_count]); + dst_lines[*dst_count] = line_nr; + hexhash += strlen(hexhash) + 1; + (*dst_count)++; + if (*dst_count >= MAX_HASHES_READ) + break; + } } } } @@ -608,14 +646,17 @@ static int snippet_scan_stream(FILE *in) return EXIT_FAILURE; } - if (scan->hash_count == 0) + if (scan->hash_count == 0 && scan->hash_count2 == 0) { fprintf(stderr, "snippet-scan: no WFP hashes in WFP input\n"); scan_data_free(scan); return EXIT_FAILURE; } - scan->total_lines = scan->lines[scan->hash_count - 1]; + /* ldb_scan_snippets recomputes total_lines across the combined wfp1+wfp2 + sources; this pre-sets it for the classic-only case. */ + if (scan->hash_count) + scan->total_lines = scan->lines[scan->hash_count - 1]; scan->timer = microseconds_now(); scan->match_type = ldb_scan_snippets(scan); diff --git a/src/report.c b/src/report.c index 0e980a4..92678ab 100644 --- a/src/report.c +++ b/src/report.c @@ -142,8 +142,10 @@ void print_server_stats(scan_data_t *scan) char hostname[MAX_ARGLN + 1]; printf(",\"server\": {"); printf("\"version\": \"%s\",", SCANOSS_VERSION); - printf("\"kb_version\": %s", kb_version); - + printf("\"kb_version\": %s,", kb_version); + /* Advertise wfp2 capability so clients/API can gate the opt-in by capability */ + printf("\"wfp2\": %s", wfp2_present ? "true" : "false"); + if (!(engine_flags & DISABLE_SERVER_INFO)) { gethostname(hostname, MAX_ARGLN + 1); diff --git a/src/scan.c b/src/scan.c index 54b366f..3d55419 100644 --- a/src/scan.c +++ b/src/scan.c @@ -33,6 +33,7 @@ #include "util.h" #include "versions.h" #include "winnowing.h" +#include "wfp2.h" #include "hpsm.h" #include "match_list.h" #include "report.h" @@ -61,6 +62,10 @@ scan_data_t * scan_data_init(char *target, int max_snippets, int max_components, scan->file_size = malloc(32); scan->hashes = calloc(MAX_FILE_SIZE,1); scan->lines = malloc(MAX_FILE_SIZE); + /* wfp2 hashes are only allocated lazily when a wfp2= block is parsed */ + scan->hashes2 = NULL; + scan->lines2 = NULL; + scan->hash_count2 = 0; scan->match_type = MATCH_NONE; scan->component_ranking_threshold = component_ranking_threshold; scan->snippet_adjust_tolerance = adjust_tolerance; @@ -88,6 +93,8 @@ void scan_data_free(scan_data_t * scan) free(scan->file_size); free(scan->hashes); free(scan->lines); + free(scan->hashes2); + free(scan->lines2); for (int i = 0; i < scan->matchmap_size; i++) { @@ -267,7 +274,11 @@ int wfp_scan(char * path, int scan_max_snippets, int scan_max_components, bool a size_t len = 0; ssize_t lineln; uint8_t *rec = NULL; - + /* When true, the snippet lines being read belong to the wfp2 (token-aware) + block of the current file. Toggled on by a "wfp2=" marker line and reset + on every new "file=" line. */ + bool in_wfp2 = false; + scanlog("--- WFP SCAN ---\n"); /* Open WFP file */ FILE *fp = fopen(path, "r"); @@ -290,7 +301,17 @@ int wfp_scan(char * path, int scan_max_snippets, int scan_max_components, bool a bool is_fh2 = (memcmp(line, "fh2=", 4) == 0); bool is_hpsm = (memcmp(line, "hpsm=", 5) == 0); bool is_bin = (memcmp(line, "bin=", 4) == 0); - bool is_wfp = (!is_file && !is_hpsm && !is_bin && !is_fh2); + bool is_wfp2_marker = (memcmp(line, "wfp2=", 5) == 0); + bool is_wfp = (!is_file && !is_hpsm && !is_bin && !is_fh2 && !is_wfp2_marker); + + /* wfp2 block delimiter: every snippet line after this (until the next + "file=") is a token-aware wfp2 fingerprint and must be matched against + the wfp2 table, not the classic wfp table. */ + if (is_wfp2_marker) + { + in_wfp2 = true; + continue; + } if (is_hpsm) { @@ -316,6 +337,9 @@ int wfp_scan(char * path, int scan_max_snippets, int scan_max_components, bool a /* Parse file information with format: file=MD5(32),file_size,file_path */ if (is_file) { + /* Each file starts in classic wfp mode until a wfp2= marker appears */ + in_wfp2 = false; + /* A scan data was fullfilled and is ready to be scanned */ if (scan) ldb_scan(scan); @@ -360,35 +384,60 @@ int wfp_scan(char * path, int scan_max_snippets, int scan_max_components, bool a /* Save hash/es to memory. Parse file information with format: linenr=wfp(6)[,wfp(6)]+ */ - if (is_wfp && scan && (scan->hash_count < MAX_HASHES_READ)) + if (is_wfp && scan) { - /* Split string by the equal and commas */ - int line_ln = strlen(line); - for (int e = 0; e < line_ln; e++) if (line[e]=='=' || line[e]==',') line[e] = 0; + /* Route the hashes to the classic wfp buffers or, when inside a + wfp2= block, to the wfp2 buffers (allocated lazily). */ + uint32_t *dst_hashes; + uint32_t *dst_lines; + uint32_t *dst_count; + if (in_wfp2) + { + if (!scan->hashes2) + { + scan->hashes2 = calloc(MAX_FILE_SIZE, 1); + scan->lines2 = malloc(MAX_FILE_SIZE); + } + dst_hashes = scan->hashes2; + dst_lines = scan->lines2; + dst_count = &scan->hash_count2; + } + else + { + dst_hashes = scan->hashes; + dst_lines = scan->lines; + dst_count = &scan->hash_count; + } - /* Extract line number */ - int line_nr = atoi(line); + if (*dst_count < MAX_HASHES_READ) + { + /* Split string by the equal and commas */ + int line_ln = strlen(line); + for (int e = 0; e < line_ln; e++) if (line[e]=='=' || line[e]==',') line[e] = 0; - /* Move pointer to the first hash */ - char *hexhash = line + strlen(line) + 1; + /* Extract line number */ + int line_nr = atoi(line); - /* Save all hashes in the present line */ - while (*hexhash) - { + /* Move pointer to the first hash */ + char *hexhash = line + strlen(line) + 1; - /* Convert hash to binary */ - ldb_hex_to_bin(hexhash, 8, (uint8_t *)&scan->hashes[scan->hash_count]); - uint32_reverse((uint8_t *)&scan->hashes[scan->hash_count]); + /* Save all hashes in the present line */ + while (*hexhash) + { + /* Convert hash to binary */ + ldb_hex_to_bin(hexhash, 8, (uint8_t *)&dst_hashes[*dst_count]); + uint32_reverse((uint8_t *)&dst_hashes[*dst_count]); - /* Save line number */ - scan->lines[scan->hash_count] = line_nr; + /* Save line number */ + dst_lines[*dst_count] = line_nr; - /* Move pointer to the next hash */ - hexhash += strlen(hexhash) + 1; + /* Move pointer to the next hash */ + hexhash += strlen(hexhash) + 1; - scan->hash_count++; - if (scan->hash_count > MAX_HASHES_READ) - break; + (*dst_count)++; + if (*dst_count > MAX_HASHES_READ) + break; + } } } } @@ -475,9 +524,9 @@ void output_matches_json(scan_data_t *scan) /** * @brief Scans a file and returns JSON matches via STDOUT. * scan structure can be already preloaded (.wfp scan) - * otherwise, it will be loaded here (scanning a physical file) - * - * @param scan + * otherwise, it will be loaded here (scanning a physical file) + * + * @param scan */ void ldb_scan(scan_data_t *scan) { @@ -561,16 +610,29 @@ void ldb_scan(scan_data_t *scan) scan->hash_count = winnowing(src, scan->hashes, scan->lines, MAX_FILE_SIZE); if (scan->hash_count) scan->total_lines = scan->lines[scan->hash_count - 1]; + + /* When the wfp2 table is available, also compute the file's + token-aware wfp2 fingerprints so the wfp2 snippet pass can + run (fallback by default, additive with --force-wfp2). */ + if (wfp2_present) + { + if (!scan->hashes2) + { + scan->hashes2 = calloc(MAX_FILE_SIZE, 1); + scan->lines2 = malloc(MAX_FILE_SIZE); + } + scan->hash_count2 = winnowing_wfp2(src, file_size, scan->hashes2, scan->lines2, MAX_FILE_SIZE / sizeof(uint32_t)); + } } free(src); } else if (scan->hash_count) scan->total_lines = scan->lines[scan->hash_count - 1]; - /* Perform snippet scan */ - if (scan->total_lines) + /* Perform snippet scan. ldb_scan_snippets accumulates wfp1 and, when + present, wfp2 hashes into a single matchmap. */ + if (scan->total_lines || (wfp2_present && scan->hash_count2)) scan->match_type = ldb_scan_snippets(scan); - else scanlog("File skipped\n"); } diff --git a/src/snippets.c b/src/snippets.c index a559700..eefa72b 100644 --- a/src/snippets.c +++ b/src/snippets.c @@ -50,8 +50,9 @@ int matchmap_max_files = DEFAULT_MATCHMAP_FILES; #define MATCHMAP_ITEM_SIZE (matchmap_max_files * 2) /* Length of a WFP table record: file id (oss_file.key_ln) + 2 bytes line number. - Taken from the table definition at runtime, since the KB may be MD5 (18) or CRC64 (10) based */ -#define wfp_rec_ln() (oss_wfp.rec_ln > 0 ? oss_wfp.rec_ln : WFP_REC_LN) + Taken from the table definition at runtime, since the KB may be MD5 (18) or CRC64 (10) based. + Parameterized by table so the same logic serves the classic wfp and the wfp2 table. */ +#define wfp_rec_ln(t) ((t).rec_ln > 0 ? (t).rec_ln : WFP_REC_LN) static bool get_all_file_ids(struct ldb_table *table, uint8_t *key, uint8_t *subkey, uint8_t *data, uint32_t datalen, int iteration, void *ptr) { uint8_t *record = (uint8_t *)ptr; @@ -106,15 +107,14 @@ bool skip_snippets(char *src, uint64_t srcln) * @brief Ajust snippet match tolerance * @param scan pointer to scan data struct */ -static void adjust_tolerance(scan_data_t *scan) +static void adjust_tolerance(scan_data_t *scan, uint32_t wfpcount, uint32_t last_line) { - uint32_t wfpcount = scan->hash_count; int range_tolerance = SNIPPETS_DEFAULT_RANGE_TOLERANCE; /** A maximum number of non-matched lines tolerated inside a matching range */ int min_match_lines = SNIPPETS_DEFAULT_MIN_MATCH_LINES; /** Minimum number of lines matched for a match range to be acepted */ int min_match_hits = SNIPPETS_DEFAULT_MIN_MATCH_HITS; /** Minimum number of snippet ID hits to produce a snippet match*/ - - if (wfpcount && scan->lines[wfpcount - 1] > SNIPPETS_DEFAULT_MIN_MATCH_LINES * 2) + + if (wfpcount && last_line > SNIPPETS_DEFAULT_MIN_MATCH_LINES * 2) { /* Range tolerance is the maximum amount of non-matched lines accepted within a matched range. This goes from 21 in small files to 5 in large files */ @@ -187,8 +187,21 @@ typedef struct matchmap_entry_t uint8_t * md5_set; uint32_t line; uint8_t wfp[WFP_LN]; - uint32_t size; + uint32_t size; + uint32_t rec_ln; /* record length of the table this hash was fetched from */ } matchmap_entry_t; + +/* One fingerprint source feeding the snippet scan: a WFP table plus the hash/line + arrays to query it with. The classic wfp1 scan uses a single source (oss_wfp); + the wfp2 opt-in adds a second one (oss_wfp2) so both are accumulated into the + SAME matchmap (hits summed and ranges unified per file by add_file_to_matchmap). */ +typedef struct +{ + struct ldb_table *table; + uint32_t *hashes; + uint32_t *lines; + uint32_t count; +} wfp_source_t; /** * @brief Add one new md5 to the matchmap * @param scan pointer to scan object @@ -314,15 +327,49 @@ int add_file_to_matchmap(scan_data_t *scan, matchmap_entry_t *item, uint8_t *md5 } /** - * @brief Main function of snippet scanning. Produce the matchmap processing the incoming wfps + * @brief Run the snippet-matching pass over one or more WFP sources. + * + * Each source pairs a WFP table with its hash/line arrays. All sources are + * fetched into a single combined map and accumulated into ONE matchmap: a file + * matched by several sources (e.g. wfp1 and wfp2) is deduplicated by file id by + * add_file_to_matchmap, so its hits are summed and its ranges unified. The + * min-hits threshold is applied later (biggest_snippet) over that combined + * evidence, so a file that each source only partially supports can still cross + * the threshold once. + * * @param scan current scan structure + * @param sources array of WFP sources (table + hashes + lines + count) + * @param n_sources number of sources * @return match_t returns the type of scan result. */ -match_t ldb_scan_snippets(scan_data_t *scan) +static match_t snippet_scan_pass(scan_data_t *scan, wfp_source_t *sources, int n_sources) { + /* Total hashes across every source, and the highest line number (used to + size the line indirection table and as the matched-percent denominator). */ + uint32_t hash_count = 0; + uint32_t max_line = 0; + for (int s = 0; s < n_sources; s++) + { + hash_count += sources[s].count; + if (sources[s].count && sources[s].lines[sources[s].count - 1] > max_line) + max_line = sources[s].lines[sources[s].count - 1]; + } - scanlog("ldb_scan_snippets\n"); - if (!scan->hash_count) + /* The match threshold (min_match_hits / min_match_lines / range_tolerance) + must follow the ORIGINAL rule: it scales with the file size, proxied by the + classic wfp1 fingerprint count — exactly as before wfp2 existed. sources[0] + is that classic fingerprint (ldb_scan_snippets always adds oss_wfp first). + The wfp2 opt-in must NOT move the threshold; it only ADDS hits into the same + per-file matchmap (hits summed below), and THAT is where combined detection + is gained: a file whose wfp1 hits alone fall short can cross the unchanged + bar once wfp2 hits are added. Passing the summed wfp1+wfp2 count here would + instead inflate the threshold above the original rule and eat that gain (it + could even push the combined result below the classic wfp1 one). With a + single source this is the classic count, so classic scans are unchanged. */ + uint32_t file_wfpcount = (n_sources > 0) ? sources[0].count : 0; + + scanlog("snippet_scan_pass (%d source(s), %u hashes)\n", n_sources, hash_count); + if (!hash_count) { scanlog("No hashes return NONE\n"); return MATCH_NONE; @@ -331,44 +378,52 @@ match_t ldb_scan_snippets(scan_data_t *scan) if (engine_flags & DISABLE_SNIPPET_MATCHING) return MATCH_NONE; + /* The matched percentage is computed later against the combined line span */ + scan->total_lines = max_line; + matchmap_setup(scan); if (scan->snippet_adjust_tolerance) - adjust_tolerance(scan); + adjust_tolerance(scan, file_wfpcount, max_line); /* First build a map with all the MD5s related with each WFP from the source file*/ - matchmap_entry_t map[scan->hash_count]; + matchmap_entry_t map[hash_count]; /* map_lines_indirection will be used to keep track of the porcessed lines*/ - int8_t map_lines_indirection[scan->lines[scan->hash_count -1] + 1]; + int8_t map_lines_indirection[max_line + 1]; memset(map_lines_indirection, -1, sizeof(map_lines_indirection)); int lines_coverage = 0; int map_max_size = 0; - /*Fill up the map with the md5s related with each wfp*/ - for (long i = 0; i < scan->hash_count; i++) + /*Fill up the map with the md5s related with each wfp, across every source*/ + long idx = 0; + for (int s = 0; s < n_sources; s++) { - /* Get all file IDs for given wfp */ - map[i].md5_set = malloc(WFP_REC_LN * MATCHMAP_ITEM_SIZE); - /* WFP_REC_LN (18) is the largest possible record, so the buffer above always fits */ - wfp_invert(scan->hashes[i], map[i].wfp); - //scanlog(" Add wfp %02x%02x%02x%02x to map\n",map[i].wfp[0], map[i].wfp[1],map[i].wfp[2],map[i].wfp[3]); - uint32_write(map[i].md5_set, 0); - map[i].line = scan->lines[i]; - ldb_fetch_recordset(NULL, oss_wfp, map[i].wfp, false, get_all_file_ids, (void *)map[i].md5_set); - map[i].size = uint32_read(map[i].md5_set) / wfp_rec_ln(); - //Initializate the lines indirection when a wfp from a line has at least one md5 linked - if (map[i].size) - map_lines_indirection[scan->lines[i]] = 0; - - if (map[i].size > map_max_size) - map_max_size = map[i].size; - + struct ldb_table *wfp_table = sources[s].table; + uint32_t rec_ln = wfp_rec_ln((*wfp_table)); + for (uint32_t j = 0; j < sources[s].count; j++, idx++) + { + /* Get all file IDs for given wfp */ + map[idx].md5_set = malloc(WFP_REC_LN * MATCHMAP_ITEM_SIZE); + /* WFP_REC_LN (18) is the largest possible record, so the buffer above always fits */ + wfp_invert(sources[s].hashes[j], map[idx].wfp); + uint32_write(map[idx].md5_set, 0); + map[idx].line = sources[s].lines[j]; + map[idx].rec_ln = rec_ln; + ldb_fetch_recordset(NULL, *wfp_table, map[idx].wfp, false, get_all_file_ids, (void *)map[idx].md5_set); + map[idx].size = uint32_read(map[idx].md5_set) / rec_ln; + //Initializate the lines indirection when a wfp from a line has at least one md5 linked + if (map[idx].size) + map_lines_indirection[map[idx].line] = 0; + + if (map[idx].size > map_max_size) + map_max_size = map[idx].size; + } } /* Classify the WFPs in cathegories depending on popularity Each cathegoy will contain a sub set of index refered to map rows*/ #define MAP_INDIRECTION_CAT_NUMBER 1000 #define MAP_INDIRECTION_CAT_SIZE (map_max_size / MAP_INDIRECTION_CAT_NUMBER) == 0 ? 1 : (map_max_size / MAP_INDIRECTION_CAT_NUMBER) - int map_indedirection_items_size = (scan->hash_count / (MAP_INDIRECTION_CAT_NUMBER))/ 10 < 10 ? - scan->hash_count : - (scan->hash_count / (MAP_INDIRECTION_CAT_NUMBER))/ 10; + int map_indedirection_items_size = (hash_count / (MAP_INDIRECTION_CAT_NUMBER))/ 10 < 10 ? + hash_count : + (hash_count / (MAP_INDIRECTION_CAT_NUMBER))/ 10; int * map_indirection[MAP_INDIRECTION_CAT_NUMBER]; //define the cathegories int map_indirection_index[MAP_INDIRECTION_CAT_NUMBER]; //index for each cathegory @@ -379,7 +434,7 @@ match_t ldb_scan_snippets(scan_data_t *scan) scanlog ("< Snippet scan setup: Total lines: %d ,Matchmap size: %d, Min hits: %d, Min lines: %d, Map max size = %d, Cat N = %d x %d, Cat size = %d >\n", scan->total_lines, scan->max_matchmap_size, scan->snippet_min_hits, scan->snippet_min_lines, map_max_size, MAP_INDIRECTION_CAT_NUMBER, map_indedirection_items_size, MAP_INDIRECTION_CAT_SIZE); - for (int i =0; i < scan->hash_count; i++) + for (int i =0; i < hash_count; i++) { if ( map[i].size < 1) continue; @@ -428,7 +483,7 @@ match_t ldb_scan_snippets(scan_data_t *scan) } if (cat_limit > scan->max_matchmap_size) { - if ((hashes_to_process < scan->hash_count / 10 || (float) lines_coverage / scan->hash_count < MIN_LINES_COVERAGE) && cat_limit < MAX_MATCHMAP_FILES) + if ((hashes_to_process < hash_count / 10 || (float) lines_coverage / hash_count < MIN_LINES_COVERAGE) && cat_limit < MAX_MATCHMAP_FILES) { scan->max_matchmap_size += map[map_indirection[i][j]].size; } @@ -459,7 +514,7 @@ match_t ldb_scan_snippets(scan_data_t *scan) } } - for (int i = 0; i <= scan->lines[scan->hash_count - 1]; i++) + for (uint32_t i = 0; i <= max_line; i++) { if (map_lines_indirection[i] > -1 && map_lines_indirection[i] == 0) { @@ -469,10 +524,10 @@ match_t ldb_scan_snippets(scan_data_t *scan) } scan->max_matchmap_size = cat_limit; scanlog("Map limit on %d MD5s at %d of %d caths. Selected hashes: %d/%d - lines coverage %d\n", - scan->max_matchmap_size, cat_limit_index, MAP_INDIRECTION_CAT_NUMBER, hashes_to_process, scan->hash_count, (lines_coverage * 100) / scan->total_lines); + scan->max_matchmap_size, cat_limit_index, MAP_INDIRECTION_CAT_NUMBER, hashes_to_process, hash_count, (lines_coverage * 100) / scan->total_lines); scan->matchmap = calloc(scan->max_matchmap_size, sizeof(matchmap_entry)); - int map_indexes[scan->hash_count]; + int map_indexes[hash_count]; memset(map_indexes, 0, sizeof(map_indexes)); /*Add MD5s to the matchmap, sorting by sector. First add the MD5s starting with 00, then with 01 and so on*/ @@ -491,7 +546,7 @@ match_t ldb_scan_snippets(scan_data_t *scan) /* Add each item to the matchmap*/ for (int wfp_index = map_indexes[i]; wfp_index < map[i].size; wfp_index++) { - int wfp_p = wfp_index * wfp_rec_ln(); + int wfp_p = wfp_index * map[i].rec_ln; /*Stop when a new sector appers*/ if (md5s[wfp_p] != sector) { @@ -549,7 +604,7 @@ match_t ldb_scan_snippets(scan_data_t *scan) /* Add each item to the matchmap*/ for (int wfp_index = map_indexes[i]; wfp_index < map[i].size; wfp_index++) { - int wfp_p = wfp_index * wfp_rec_ln(); + int wfp_p = wfp_index * map[i].rec_ln; int sector = md5s[wfp_p]; int sector_max = scan->snippet_min_hits; @@ -583,12 +638,12 @@ match_t ldb_scan_snippets(scan_data_t *scan) } //Free memory - for (int i = 0; i < scan->hash_count; i++) + for (int i = 0; i < hash_count; i++) { free(map[i].md5_set); } - + for (int i = 0; i < MAP_INDIRECTION_CAT_NUMBER; i++) { free(map_indirection[i]); @@ -596,9 +651,38 @@ match_t ldb_scan_snippets(scan_data_t *scan) if (scan->matchmap_size) return MATCH_SNIPPET; - + scanlog("Snippet scan has no matches\n"); return MATCH_NONE; } + +/** + * @brief Snippet scan. Always queries the classic wfp table (oss_wfp); when the + * wfp2 table is present and the scan carries wfp2 hashes (parsed from a + * wfp2= block or computed from the source), oss_wfp2 is added as a second + * source and both are accumulated into the same matchmap. Without wfp2 + * hashes the behaviour is identical to a classic wfp1-only scan. + * @param scan current scan structure + * @return match_t scan result type + */ +match_t ldb_scan_snippets(scan_data_t *scan) +{ + wfp_source_t sources[2]; + int n = 0; + + if (scan->hash_count > 0) + sources[n++] = (wfp_source_t){ &oss_wfp, scan->hashes, scan->lines, scan->hash_count }; + + if (wfp2_present && scan->hash_count2 > 0) + sources[n++] = (wfp_source_t){ &oss_wfp2, scan->hashes2, scan->lines2, scan->hash_count2 }; + + if (!n) + { + scanlog("No hashes return NONE\n"); + return MATCH_NONE; + } + + return snippet_scan_pass(scan, sources, n); +} diff --git a/src/wfp2.c b/src/wfp2.c new file mode 100644 index 0000000..99330bf --- /dev/null +++ b/src/wfp2.c @@ -0,0 +1,624 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * src/wfp2.c + * + * WFP2 (token-aware) fingerprint generation. + * + * Copyright (C) 2018-2025 SCANOSS.COM + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/** + * @file wfp2.c + * @brief Token-aware (wfp2) fingerprint generation. + * + * This is a faithful C port of the reference Go implementation + * (scanoss-go pkg/fingerprint/wfp2, mirrored in wfp2_wrapper/internal) used by + * go-minr to mine the wfp2 table. The pipeline is: + * DetectLanguage -> RemoveCommentsFromSource -> Normalizex -> HashSourceX + * The normalization collapses identifiers (0xff) and consecutive types (0x00), + * maps reserved words/punctuation to fixed byte codes, drops comments and + * whitespace, and then winnows (GRAM 20 / WINDOW 40, CRC32C) the result. + * + * The output MUST match the mined fingerprints byte for byte; any divergence in + * the tables or the algorithm makes the wfp2 snippet scan miss. + */ + +#include +#include +#include +#include +#include + +#include "crc32c.h" +#include "wfp2.h" + +#define GRAM_WFP2 20 +#define WINDOW_WFP2 40 + +/* ------------------------------------------------------------------ */ +/* Reserved words / punctuation -> byte code. Generated from the Go + elementsMap. Reserved codes: 0 (type marker), 0xff (other), plus the + skipped 1,9,10,32,255. elementsMap is checked before typesMap. */ +typedef struct { const char *key; uint8_t val; } wfp2_elem_t; + +static wfp2_elem_t elements[] = { + {"alias", 2}, {"alignas", 3}, {"alignof", 4}, {"and", 5}, {"as", 6}, {"asm", 7}, + {"assert", 8}, {"async", 11}, {"await", 12}, {"base", 13}, {"become", 14}, + {"begin", 15}, {"bindattr", 16}, {"bindingattr", 17}, {"bitand", 18}, {"bitor", 19}, + {"break", 20}, {"case", 21}, {"catch", 22}, {"cdecl", 23}, {"?", 24}, + {"checked", 25}, {"compl", 26}, {"concept", 27}, {"continue", 28}, {"crate", 29}, {"debugger", 30}, + {"declare", 31}, {"declspec", 33}, {"def", 34}, {"default", 35}, {"defer", 36}, + {"defined", 37}, {"del", 38}, {"delete", 39}, {"deriving", 40}, {"do", 41}, + {"@", 42}, {"dyn", 43}, {"dynamic", 44}, {"elif", 45}, {"else", 46}, {"elseif", 47}, {"elsif", 48}, + {"end", 49}, {"endfor", 50}, {"endforeach", 51}, {"endif", 52}, {"endswitch", 53}, {"endwhile", 54}, + {"eq", 55}, {"esac", 56}, {"except", 57}, {"explicit", 58}, {"export", 59}, + {"extends", 60}, {"fallthrough", 61}, {"far", 62}, {"fastcall", 63}, + {"feature", 64}, {"fi", 65}, {"finally", 66}, {"fixed", 67}, {"owns", 68}, {"fn", 69}, + {"for", 70}, {"foreach", 71}, {"forever", 72}, {"forsome", 73}, {"friend", 74}, {"from", 75}, + {"function", 76}, {"get", 77}, {"global", 78}, {"goto", 79}, {"has", 80}, {"id", 81}, + {"if", 82}, {"impl", 83}, {"implements", 84}, {"implicit", 85}, {"import", 86}, {"in", 87}, + {"include", 88}, {"infix", 89}, {"inherit", 90}, {"inout", 91}, + {"instanceof", 92}, {"spawn", 93}, {"deref", 94}, {"offsetof", 95}, + {"is", 96}, {"lambda", 97}, {"lateinit", 98}, + {"lazy", 99}, {"leave", 100}, {"let", 101}, {"local", 102}, {"lock", 103}, + {"loop", 104}, {"macro", 105}, {"match", 106}, {"me", 107}, {"module", 108}, + {"namespace", 109}, {"new", 110}, {"next", 111}, {"nil", 112}, {"no", 113}, + {"nocopy", 114}, {"none", 115}, {"nonlocal", 116}, {"not", 117}, + {"of", 118}, {"on", 119}, {"open", 120}, {"operator", 121}, + {"or", 122}, {"out", 123}, {"override", 124}, {"package", 125}, {"parallel", 126}, + {"params", 127}, {"partial", 128}, {"pass", 129}, {"past", 130}, {"print", 131}, + {"printf", 132}, {"procedure", 133}, {"pure", 134}, {"raise", 135}, {"rec", 136}, + {"receiver", 137}, {"redim", 138}, {"reified", 139}, + {"remove", 140}, {"repeat", 141}, {"require", 142}, {"required", 143}, {"resource", 144}, + {"resume", 145}, {"return", 146}, {"safe", 147}, {"sealed", 148}, + {"select", 149}, {"self", 150}, {"send", 151}, {"set", 152}, {"setparam", 153}, {"semaphore", 154}, + {"sizeof", 155}, {"stdcall", 156}, {"step", 157}, + {"sub", 158}, {"super", 159}, + {"suspend", 160}, {"switch", 161}, {"tailrec", 162}, {"then", 163}, + {"this", 164}, {"thread", 165}, {"throw", 166}, {"throws", 167}, {"to", 168}, + {"true", 169}, {"try", 170}, + {"unchecked", 171}, {"undef", 172}, + {"unless", 173}, {"unsized", 174}, + {"until", 175}, {"use", 176}, {"using", 177}, {"val", 178}, {"value", 179}, {"var", 180}, + {"wasm", 181}, + {"weak", 182}, {"when", 183}, {"where", 184}, {"while", 185}, {"with", 186}, {"xor", 187}, + {"/", 188}, {"yield", 189}, {"yieldto", 190}, {"subroutine", 191}, {"ensure", 192}, {"behind", 193}, {"exec", 194}, + {"!", 195}, {"\"", 196}, {"#", 197}, {"$", 198}, {"%", 199}, + {"&", 200}, {"'", 201}, {"(", 202}, {")", 203}, {"*", 204}, + {"+", 205}, {",", 206}, {"-", 207}, {".", 208}, + {":", 210}, {";", 211}, {"<", 212}, {"=", 213}, {">", 214}, + {"[", 217}, {"\\", 218}, {"]", 219}, + {"^", 220}, {"`", 221}, {"{", 222}, {"|", 223}, {"}", 224}, + {"~", 225}, +}; +#define N_ELEM (sizeof(elements) / sizeof(elements[0])) + +/* Type keywords -> collapsed to a single 0x00 run. Generated from the Go + typesMap. Entries containing non-alnum chars (e.g. "interface{}", "&str", + "real*4", "comp-1") are kept for fidelity but never match a word token. */ +static const char *types[] = { + "char", "short", "int", "long", "float", "double", "void", "signed", "unsigned", + "bool", "true", "false", "wchar", "const", "volatile", "static", "extern", "auto", + "register", "inline", "mutable", "restrict", "constexpr", "consteval", "constinit", + "noexcept", "class", "struct", "union", "enum", "interface", "typedef", "typename", + "typealias", "type", "typeid", "typeof", "decltype", "trait", "protocol", "delegate", + "event", "public", "private", "protected", "internal", "pub", "abstract", "virtual", + "final", "sealed", "readonly", "synchronized", "transient", "strictfp", "native", + "unsafe", "ptr", "ref", "byte", "boolean", "integer", "character", "string", "object", + "sbyte", "ushort", "uint", "ulong", "decimal", "int16", "int32", "int64", "uint16", + "uint32", "uint64", "single", "number", "undefined", "null", "symbol", "bigint", + "str", "list", "dict", "tuple", "set", "frozenset", "bytes", "bytearray", "int8", + "uint8", "float32", "float64", "complex64", "complex128", "rune", "uintptr", + "interface{}", "chan", "func", "map", "slice", "i8", "i16", "i32", "i64", "i128", + "isize", "u8", "u16", "u32", "u64", "u128", "usize", "f32", "f64", "&str", "vec", + "option", "result", "box", "rc", "arc", "varchar", "real", "text", "blob", "numeric", + "timestamp", "datetime", "date", "time", "smallint", "tinyint", "mediumint", "binary", + "varbinary", "cardinal", "shortint", "longint", "comp", "currency", "extended", + "real48", "ansichar", "widechar", "ansistring", "widestring", "shortstring", "pchar", + "pwidechar", "logical", "complex", "real*4", "real*8", "integer*2", "integer*4", + "pic", "picture", "comp-1", "comp-2", "comp-3", "comp-4", "comp-5", "packed-decimal", + "display", "array", "dictionary", "shortarray", "intarray", "longarray", "floatarray", + "doublearray", "booleanarray", "vhararray", +}; +#define N_TYPE (sizeof(types) / sizeof(types[0])) + +static int elem_cmp(const void *a, const void *b) +{ + return strcmp(((const wfp2_elem_t *)a)->key, ((const wfp2_elem_t *)b)->key); +} +static int str_cmp(const void *a, const void *b) +{ + return strcmp(*(const char *const *)a, *(const char *const *)b); +} + +/* Tables are sorted once (lazily) so lookups can use bsearch. The engine scans + files sequentially within a single process, so no locking is required. */ +static bool tables_ready = false; +static void ensure_tables(void) +{ + if (tables_ready) return; + qsort(elements, N_ELEM, sizeof(elements[0]), elem_cmp); + qsort(types, N_TYPE, sizeof(types[0]), str_cmp); + tables_ready = true; +} + +/* Lookup a NUL-terminated token in the elements table. */ +static bool elem_lookup(const char *word, uint8_t *out) +{ + wfp2_elem_t key = { word, 0 }; + wfp2_elem_t *r = bsearch(&key, elements, N_ELEM, sizeof(elements[0]), elem_cmp); + if (r) { *out = r->val; return true; } + return false; +} + +/* True if the NUL-terminated token is a type keyword. */ +static bool type_lookup(const char *word) +{ + const char *kp = word; + return bsearch(&kp, types, N_TYPE, sizeof(types[0]), str_cmp) != NULL; +} + +/* ------------------------------------------------------------------ */ +/* Language detection and comment removal. + Only the comment style actually matters downstream; languages whose comment + patterns are identical (c/cpp/go/java/javascript) are interchangeable here. */ + +enum { L_NONE = 0, L_PYTHON, L_JAVA, L_GO, L_C, L_CPP, L_JS, L_RUBY, L_PHP }; + +static const char *kw_python[] = {"def", "import", "self", "print", "None", "True", "False", "elif", "except"}; +static const char *kw_java[] = {"public", "static", "void", "class", "implements", "extends", "null", "package"}; +static const char *kw_go[] = {"func", "package", "import", "defer", "chan", "go", "select", "map", "range"}; +static const char *kw_c[] = {"#include", "printf", "scanf", "malloc", "free", "NULL", "int", "char"}; +static const char *kw_cpp[] = {"#include", "cout", "cin", "std", "new", "delete", "namespace", "class"}; +static const char *kw_js[] = {"function", "var", "let", "const", "console", "undefined", "require", "export"}; +static const char *kw_ruby[] = {"def", "end", "puts", "require", "class", "module", "elsif", "unless"}; +static const char *kw_php[] = {" n) return -1; + for (int i = 0; i + nl <= n; i++) + if (memcmp(s + i, needle, nl) == 0) return i; + return -1; +} + +/* strings.TrimSpace: return the trimmed [*op, *op+*on) span of s[0:n]. */ +static void trim_space(const char *s, int n, const char **op, int *on) +{ + int a = 0, b = n; + while (a < b && is_ws((unsigned char)s[a])) a++; + while (b > a && is_ws((unsigned char)s[b - 1])) b--; + *op = s + a; + *on = b - a; +} + +/* strings.TrimRight(x, " \t"): reduce *n to drop trailing spaces/tabs. */ +static void trim_right_ws(const char *s, int *n) +{ + int b = *n; + while (b > 0 && (s[b - 1] == ' ' || s[b - 1] == '\t')) b--; + *n = b; +} + +/* strings.TrimLeft(x, " \t"). */ +static void trim_left_ws(const char *s, int n, const char **op, int *on) +{ + int a = 0; + while (a < n && (s[a] == ' ' || s[a] == '\t')) a++; + *op = s + a; + *on = n - a; +} + +/* + * DetectLanguage: split content on whitespace, and for every token count a hit + * for each (language, keyword) whose keyword is a prefix of the token. Returns + * the highest-scoring language id (L_NONE if nothing matched). + * + * WARNING (determinism): the reference Go implementation (go-minr / scanoss-go) + * resolves ties via random map iteration, so for files where several languages + * tie on score it is NON-deterministic, and the mined KB inherits that: the same + * file can be normalized with different comment rules across mined files. This + * port is deterministic (fixed order, first max wins), but for exact matching the + * miner AND every scanner must share ONE deterministic tie rule and the KB be + * re-mined with it. Until then, tie-files may match inconsistently. + */ +static int detect_language(const char *src, long n) +{ + int scores[N_LANG]; + memset(scores, 0, sizeof(scores)); + + long i = 0; + while (i < n) + { + while (i < n && is_ws((unsigned char)src[i])) i++; + long start = i; + while (i < n && !is_ws((unsigned char)src[i])) i++; + int tlen = (int)(i - start); + if (tlen <= 0) continue; + const char *tok = src + start; + + for (unsigned int L = 0; L < N_LANG; L++) + { + for (int k = 0; k < langs[L].nkw; k++) + { + const char *kw = langs[L].kw[k]; + int kl = (int)strlen(kw); + if (tlen >= kl && memcmp(tok, kw, kl) == 0) + scores[L]++; + } + } + } + + int best = -1, best_score = 0; + for (unsigned int L = 0; L < N_LANG; L++) + { + if (scores[L] > best_score) + { + best_score = scores[L]; + best = (int)L; + } + } + return best < 0 ? L_NONE : langs[best].id; +} + +static lang_def_t *lang_by_id(int id) +{ + for (unsigned int L = 0; L < N_LANG; L++) + if (langs[L].id == id) return &langs[L]; + return NULL; +} + +/* + * Process a single input line into out-buffer lb, mirroring the per-line logic + * of RemoveCommentsFromSource. Returns the output line length; updates the + * multiline-comment state across lines. + */ +static int strip_line(const char *line, int n, lang_def_t *L, + bool *in_multi, const char **multi_end, char *lb) +{ + if (*in_multi) + { + int idx = idx_of(line, n, *multi_end); + if (idx >= 0) + { + *in_multi = false; + int me = (int)strlen(*multi_end); + const char *after = line + idx + me; + int an = n - (idx + me); + const char *ta; int tan; + trim_space(after, an, &ta, &tan); + if (tan > 0) { memcpy(lb, ta, tan); return tan; } + return 0; + } + return 0; /* still inside the comment: emit an empty line */ + } + + /* Multiline comment start? */ + for (int m = 0; m < L->nmulti; m++) + { + const char *beg = L->mbeg[m]; + const char *end = L->mend[m]; + int beg_idx = idx_of(line, n, beg); + if (beg_idx < 0) continue; + + int bl = (int)strlen(beg); + const char *before = line; + int before_n = beg_idx; + trim_right_ws(before, &before_n); + + int search_start = beg_idx + bl; + int rest_n = n - search_start; + int end_idx = idx_of(line + search_start, rest_n, end); + + if (end_idx >= 0) + { + /* Comment opens and closes on the same line */ + int el = (int)strlen(end); + const char *after = line + search_start + end_idx + el; + int an = n - (search_start + end_idx + el); + + int out = 0; + memcpy(lb, before, before_n); + out = before_n; + + const char *ta; int tan; + trim_space(after, an, &ta, &tan); + if (tan > 0) + { + const char *la; int lan; + trim_left_ws(after, an, &la, &lan); + if (out > 0) { lb[out++] = ' '; memcpy(lb + out, la, lan); out += lan; } + else { memcpy(lb, la, lan); out = lan; } + } + + const char *tc; int tcn; + trim_space(lb, out, &tc, &tcn); + return tcn > 0 ? out : 0; + } + + /* Comment continues on following lines */ + *in_multi = true; + *multi_end = end; + const char *tb; int tbn; + trim_space(before, before_n, &tb, &tbn); + if (tbn > 0) { memcpy(lb, before, before_n); return before_n; } + return 0; + } + + /* Line comment */ + const char *clean = line; + int clean_n = n; + for (int c = 0; c < L->nline; c++) + { + int idx = idx_of(line, n, L->line[c]); + if (idx >= 0) + { + clean = line; + clean_n = idx; + trim_right_ws(clean, &clean_n); + break; + } + } + + const char *tcl; int tcln; + trim_space(clean, clean_n, &tcl, &tcln); + if (tcln > 0) { memcpy(lb, clean, clean_n); return clean_n; } + return 0; +} + +/* + * RemoveCommentsFromSource: strip comments line by line, preserving the line + * count (removed lines become empty). Writes into out (caller-owned, must hold + * at least 2*n+2 bytes). Returns the output length. If the language has no known + * comment pattern, the source is copied unchanged. + */ +static long remove_comments(const char *src, long n, int lang_id, char *out) +{ + lang_def_t *L = lang_by_id(lang_id); + if (!L) + { + memcpy(out, src, n); + return n; + } + + char *lb = malloc(n + 2); + if (!lb) + { + memcpy(out, src, n); + return n; + } + + bool in_multi = false; + const char *multi_end = ""; + long o = 0; + long ls = 0; + bool first = true; + + while (ls <= n) + { + long le = ls; + while (le < n && src[le] != '\n') le++; + int line_n = (int)(le - ls); + + if (!first) out[o++] = '\n'; + first = false; + + int outn = strip_line(src + ls, line_n, L, &in_multi, &multi_end, lb); + if (outn > 0) { memcpy(out + o, lb, outn); o += outn; } + + if (le >= n) break; + ls = le + 1; + } + + free(lb); + return o; +} + +/* ------------------------------------------------------------------ */ +/* Token-aware normalization (Normalizex). */ + +static unsigned char lc(unsigned char c) +{ + if (c >= 'A' && c <= 'Z') return c + 32; + return c; +} +static bool is_alnum_(unsigned char c) +{ + return (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '_'; +} + +/* + * Normalizex: lowercase, then per line emit one byte per token: + * reserved word / punctuation -> its element code + * type keyword -> a single 0x00 for a run of consecutive types + * any other identifier -> 0xff + * whitespace / unmapped char -> dropped + * Lines are preserved (separated by '\n'). Writes into out (>= n+2 bytes). + * Returns the output length. + */ +static long normalizex(const char *in, long n, char *out) +{ + long o = 0; + long ls = 0; + bool first = true; + + while (ls <= n) + { + long le = ls; + while (le < n && in[le] != '\n') le++; + + if (!first) out[o++] = '\n'; + first = false; + + bool last_was_type = false; + long j = ls; + while (j < le) + { + unsigned char rl = lc((unsigned char)in[j]); + if (is_alnum_(rl)) + { + char wbuf[256]; + int wl = 0; + while (j < le && is_alnum_(lc((unsigned char)in[j]))) + { + if (wl < (int)sizeof(wbuf) - 1) wbuf[wl++] = (char)lc((unsigned char)in[j]); + j++; + } + wbuf[wl] = 0; + + uint8_t ev; + if (elem_lookup(wbuf, &ev)) { out[o++] = (char)ev; last_was_type = false; } + else if (type_lookup(wbuf)) { if (!last_was_type) out[o++] = 0x00; last_was_type = true; } + else { out[o++] = (char)0xff; last_was_type = false; } + } + else + { + char cs[2] = { (char)rl, 0 }; + uint8_t ev; + if (elem_lookup(cs, &ev)) out[o++] = (char)ev; + last_was_type = false; + j++; + } + } + + if (le >= n) break; + ls = le + 1; + } + return o; +} + +/* ------------------------------------------------------------------ */ +/* Winnowing over the normalized byte stream (HashSourceX). */ + +static uint32_t min_hash(const uint32_t *h, int n) +{ + int idx = 0; + for (int r = 0; r < n; r++) + if (h[r] <= h[idx]) idx = r; + return h[idx]; +} + +uint32_t winnowing_wfp2(char *src, long src_len, uint32_t *hashes, uint32_t *lines, long limit) +{ + if (!src || src_len <= 0 || limit <= 0) return 0; + + ensure_tables(); + + int lang = detect_language(src, src_len); + + /* Comment removal, then normalization. Both outputs are <= 2*len. */ + char *no_com = malloc(2 * src_len + 2); + char *norm = malloc(2 * src_len + 2); + if (!no_com || !norm) + { + free(no_com); + free(norm); + return 0; + } + + long nc_len = remove_comments(src, src_len, lang, no_com); + long norm_len = normalizex(no_com, nc_len, norm); + + uint8_t window[GRAM_WFP2]; + int wlen = 0; + uint32_t ghashes[WINDOW_WFP2]; + int hlen = 0; + uint32_t last = 0; + int cur_line = 1; + uint32_t count = 0; + + for (long i = 0; i < norm_len; i++) + { + uint8_t ch = (uint8_t)norm[i]; + if (ch == '\n') + { + cur_line++; + if (cur_line >= 0xffff) break; + continue; + } + + window[wlen++] = ch; + if (wlen >= GRAM_WFP2) + { + ghashes[hlen++] = calc_crc32c((char *)window, GRAM_WFP2); + if (hlen >= WINDOW_WFP2) + { + uint32_t a = min_hash(ghashes, WINDOW_WFP2); + if (a != last) + { + last = a; + uint8_t minbuf[4]; + minbuf[0] = (uint8_t)(a & 0xff); + minbuf[1] = (uint8_t)((a >> 8) & 0xff); + minbuf[2] = (uint8_t)((a >> 16) & 0xff); + minbuf[3] = (uint8_t)((a >> 24) & 0xff); + if ((long)count < limit) + { + hashes[count] = calc_crc32c((char *)minbuf, 4); + lines[count] = (uint32_t)cur_line; + count++; + } + else + { + break; + } + } + /* hashes = hashes[1:WINDOW] */ + memmove(ghashes, ghashes + 1, (WINDOW_WFP2 - 1) * sizeof(uint32_t)); + hlen = WINDOW_WFP2 - 1; + } + /* window = window[1:GRAM] */ + memmove(window, window + 1, GRAM_WFP2 - 1); + wlen = GRAM_WFP2 - 1; + } + } + + free(no_com); + free(norm); + return count; +}