Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions inc/scan.h
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@ typedef struct scan_data_t
uint32_t *hashes; /* pointer to wfp hashes*/
uint32_t *lines; /*pointer to line hashes */
uint32_t hash_count; /* count of wfp hashes */
uint32_t *hashes2; /* pointer to wfp2 (token-aware) hashes, parsed from the wfp2= block */
uint32_t *lines2; /* pointer to wfp2 line numbers */
uint32_t hash_count2; /* count of wfp2 hashes */
long timer; /*timer for execution profile*/
bool preload; /*used in hash scanning */
int total_lines; /* total file lines */
Expand Down
6 changes: 5 additions & 1 deletion inc/scanoss.h
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
#define WFP_LN 4
#define WFP_REC_LN 18

#define SCANOSS_VERSION "6.1.0-crc64"
#define SCANOSS_VERSION "6.2.0-crc64"

/* Log files */
#define SCAN_LOG "/tmp/scanoss_scan.log"
Expand Down Expand Up @@ -91,6 +91,7 @@ extern char * component_hint;
extern struct ldb_table oss_url;
extern struct ldb_table oss_file;
extern struct ldb_table oss_wfp;
extern struct ldb_table oss_wfp2;
extern struct ldb_table oss_purl;
extern struct ldb_table oss_copyright;
extern struct ldb_table oss_quality;
Expand All @@ -104,6 +105,9 @@ extern struct ldb_table oss_notices;
extern struct ldb_table oss_pivot;
extern struct ldb_table oss_path;

/* True when the optional wfp2 table is present in the loaded KB. */
extern bool wfp2_present;

/* Reconstruct a project's file structure (md5,path per file) from its url hash. */
void get_project_files(char * url_key);

Expand Down
32 changes: 32 additions & 0 deletions inc/wfp2.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* inc/wfp2.h
*
* WFP2 (token-aware) fingerprint generation.
*
* Copyright (C) 2018-2025 SCANOSS.COM
*/

#ifndef __WFP2_H
#define __WFP2_H

#include <stdint.h>

/**
* @brief Compute the WFP2 (token-aware) snippet fingerprints of a source buffer.
*
* Mirrors the reference Go implementation (scanoss-go / go-minr snippetsx):
* language detection -> comment removal -> token-aware normalization -> winnowing
* (GRAM 20, WINDOW 40, CRC32C). The output hashes/lines must match, byte for byte,
* the fingerprints the KB was mined with, otherwise nothing will match the wfp2 table.
*
* @param src source buffer (text)
* @param src_len length of src in bytes
* @param hashes output array of wfp2 hashes
* @param lines output array of line numbers (paired with hashes)
* @param limit maximum number of hashes to emit
* @return number of hashes produced
*/
uint32_t winnowing_wfp2(char *src, long src_len, uint32_t *hashes, uint32_t *lines, long limit);

#endif
13 changes: 13 additions & 0 deletions src/main.c
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
struct ldb_table oss_url;
struct ldb_table oss_file;
struct ldb_table oss_wfp;
struct ldb_table oss_wfp2;
struct ldb_table oss_purl;
struct ldb_table oss_copyright;
struct ldb_table oss_quality;
Expand All @@ -66,6 +67,7 @@ struct ldb_table oss_notices;
struct ldb_table oss_pivot;
struct ldb_table oss_path;
bool path_table_present = false;
bool wfp2_present = false;
component_item *ignore_components;
component_item *declared_components;

Expand Down Expand Up @@ -144,6 +146,17 @@ void initialize_ldb_tables(char *name)
snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "wfp");
oss_wfp = ldb_read_cfg(dbtable);

/* Optional wfp2 table: token-aware fingerprints mined into a separate table,
* leaving the classic wfp table untouched. When present, the engine can run a
* wfp2 snippet pass (fallback by default, additive when forced). */
if (ldb_table_exists(oss_db_name, "wfp2"))
{
wfp2_present = true;
snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "wfp2");
oss_wfp2 = ldb_read_cfg(dbtable);
scanlog("wfp2 table present: enabling wfp2 snippet matching\n");
}

snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "purl");
oss_purl = ldb_read_cfg(dbtable);

Expand Down
79 changes: 60 additions & 19 deletions src/purl_scan.c
Original file line number Diff line number Diff line change
Expand Up @@ -454,7 +454,7 @@ int component_scan(char *url_hash_list)

if (!quiet)
{
printf("], \"server\": {\"version\": \"%s\", \"kb_version\": %s}}\n", SCANOSS_VERSION, kb_version);
printf("], \"server\": {\"version\": \"%s\", \"kb_version\": %s, \"wfp2\": %s}}\n", SCANOSS_VERSION, kb_version, wfp2_present ? "true" : "false");
fflush(stdout);
}

Expand All @@ -478,6 +478,8 @@ static int snippet_scan_stream(FILE *in)
char file_path[MAX_FILE_PATH] = "";
uint64_t file_size = 0;
bool got_file = false;
/* Snippet lines after a "wfp2=" marker belong to the token-aware wfp2 set */
bool in_wfp2 = false;

scanlog("--- SNIPPET SCAN ---\n");

Expand All @@ -492,12 +494,21 @@ static int snippet_scan_stream(FILE *in)
bool is_fh2 = (memcmp(line, "fh2=", 4) == 0);
bool is_hpsm = (memcmp(line, "hpsm=", 5) == 0);
bool is_bin = (memcmp(line, "bin=", 4) == 0);
bool is_wfp = (!is_file && !is_fh2 && !is_hpsm && !is_bin);
bool is_wfp2_marker = (memcmp(line, "wfp2=", 5) == 0);
bool is_wfp = (!is_file && !is_fh2 && !is_hpsm && !is_bin && !is_wfp2_marker);

/* Snippet-only mode: ignore hpsm/bin payloads */
if (is_hpsm || is_bin)
continue;

/* wfp2 block delimiter: subsequent snippet lines are token-aware wfp2
fingerprints, matched against the wfp2 table by ldb_scan_snippets */
if (is_wfp2_marker)
{
in_wfp2 = true;
continue;
}

if (is_file)
{
if (got_file)
Expand Down Expand Up @@ -569,6 +580,7 @@ static int snippet_scan_stream(FILE *in)
strcpy(scan->source_md5, file_md5_hex);
free(rec);
got_file = true;
in_wfp2 = false;
continue;
}

Expand All @@ -579,24 +591,50 @@ static int snippet_scan_stream(FILE *in)
continue;
}

if (is_wfp && scan && (scan->hash_count < MAX_HASHES_READ))
if (is_wfp && scan)
{
int line_ln = strlen(line);
for (int e = 0; e < line_ln; e++)
if (line[e] == '=' || line[e] == ',') line[e] = 0;

int line_nr = atoi(line);
char *hexhash = line + strlen(line) + 1;
/* Route to the classic wfp buffers or, inside a wfp2= block, to the
wfp2 buffers (allocated lazily) — same as the -w (wfp_scan) path. */
uint32_t *dst_hashes;
uint32_t *dst_lines;
uint32_t *dst_count;
if (in_wfp2)
{
if (!scan->hashes2)
{
scan->hashes2 = calloc(MAX_FILE_SIZE, 1);
scan->lines2 = malloc(MAX_FILE_SIZE);
}
dst_hashes = scan->hashes2;
dst_lines = scan->lines2;
dst_count = &scan->hash_count2;
}
else
{
dst_hashes = scan->hashes;
dst_lines = scan->lines;
dst_count = &scan->hash_count;
}

while (*hexhash)
if (*dst_count < MAX_HASHES_READ)
{
ldb_hex_to_bin(hexhash, 8, (uint8_t *) &scan->hashes[scan->hash_count]);
uint32_reverse((uint8_t *) &scan->hashes[scan->hash_count]);
scan->lines[scan->hash_count] = line_nr;
hexhash += strlen(hexhash) + 1;
scan->hash_count++;
if (scan->hash_count >= MAX_HASHES_READ)
break;
int line_ln = strlen(line);
for (int e = 0; e < line_ln; e++)
if (line[e] == '=' || line[e] == ',') line[e] = 0;

int line_nr = atoi(line);
char *hexhash = line + strlen(line) + 1;

while (*hexhash)
{
ldb_hex_to_bin(hexhash, 8, (uint8_t *) &dst_hashes[*dst_count]);
uint32_reverse((uint8_t *) &dst_hashes[*dst_count]);
dst_lines[*dst_count] = line_nr;
hexhash += strlen(hexhash) + 1;
(*dst_count)++;
if (*dst_count >= MAX_HASHES_READ)
break;
}
}
}
}
Expand All @@ -608,14 +646,17 @@ static int snippet_scan_stream(FILE *in)
return EXIT_FAILURE;
}

if (scan->hash_count == 0)
if (scan->hash_count == 0 && scan->hash_count2 == 0)
{
fprintf(stderr, "snippet-scan: no WFP hashes in WFP input\n");
scan_data_free(scan);
return EXIT_FAILURE;
}

scan->total_lines = scan->lines[scan->hash_count - 1];
/* ldb_scan_snippets recomputes total_lines across the combined wfp1+wfp2
sources; this pre-sets it for the classic-only case. */
if (scan->hash_count)
scan->total_lines = scan->lines[scan->hash_count - 1];
scan->timer = microseconds_now();
scan->match_type = ldb_scan_snippets(scan);

Expand Down
6 changes: 4 additions & 2 deletions src/report.c
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,10 @@ void print_server_stats(scan_data_t *scan)
char hostname[MAX_ARGLN + 1];
printf(",\"server\": {");
printf("\"version\": \"%s\",", SCANOSS_VERSION);
printf("\"kb_version\": %s", kb_version);

printf("\"kb_version\": %s,", kb_version);
/* Advertise wfp2 capability so clients/API can gate the opt-in by capability */
printf("\"wfp2\": %s", wfp2_present ? "true" : "false");

if (!(engine_flags & DISABLE_SERVER_INFO))
{
gethostname(hostname, MAX_ARGLN + 1);
Expand Down
Loading
Loading