Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions src/main/java/com/scanoss/Scanner.java
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import com.scanoss.processor.*;
import com.scanoss.rest.ScanApi;
import com.scanoss.settings.Bom;
import com.scanoss.settings.ScanConfig;
import com.scanoss.settings.ScanossSettings;
import com.scanoss.utils.JsonUtils;
import lombok.*;
Expand Down Expand Up @@ -104,6 +105,8 @@ public class Scanner {
private final ScanFileProcessor scanFileProcessor;
private final WfpFileProcessor wfpFileProcessor;
private final ScanossSettings settings;
private final ScanConfig cliScanConfig; // CLI-provided scan config (lowest priority)
private final ScanConfig scanConfig; // Resolved scan config (after priority merge)
private final ScannerPostProcessor postProcessor;
private final FilterConfig filterConfig;
private Predicate<Path> fileFilter;
Expand All @@ -116,7 +119,8 @@ private Scanner(Boolean skipSnippets, Boolean allExtensions, Boolean obfuscate,
Integer snippetLimit, String customCert, Proxy proxy,
Winnowing winnowing, ScanApi scanApi,
ScanFileProcessor scanFileProcessor, WfpFileProcessor wfpFileProcessor,
ScanossSettings settings, ScannerPostProcessor postProcessor, FilterConfig filterConfig,
ScanossSettings settings, ScanConfig cliScanConfig, ScanConfig scanConfig,
ScannerPostProcessor postProcessor, FilterConfig filterConfig,
Predicate<Path> fileFilter,
Predicate<Path> folderFilter
) {
Expand All @@ -137,20 +141,27 @@ private Scanner(Boolean skipSnippets, Boolean allExtensions, Boolean obfuscate,
this.snippetLimit = snippetLimit;
this.customCert = customCert;
this.proxy = proxy;
this.settings = Objects.requireNonNullElseGet(settings, () -> ScanossSettings.builder().build());
this.cliScanConfig = cliScanConfig;
// Resolve scan config: file_snippet (highest) > settings (middle) > CLI (lowest)
this.scanConfig = this.settings.getResolvedScanConfig(
Objects.requireNonNullElseGet(cliScanConfig, () -> ScanConfig.builder().build()));
this.winnowing = Objects.requireNonNullElseGet(winnowing, () ->
Winnowing.builder().skipSnippets(skipSnippets).allExtensions(allExtensions).obfuscate(obfuscate)
.hpsm(hpsm).snippetLimit(snippetLimit)
.skipHeaders(this.scanConfig.getSkipHeaders() != null && this.scanConfig.getSkipHeaders())
.skipHeadersLimit(this.scanConfig.getSkipHeadersLimit() != null ? this.scanConfig.getSkipHeadersLimit() : 0)
.build());
this.scanApi = Objects.requireNonNullElseGet(scanApi, () ->
ScanApi.builder().url(url).apiKey(apiKey).timeout(timeout).retryLimit(retryLimit).flags(scanFlags)
.sbomType(sbomType).sbom(sbom).customCert(customCert).proxy(proxy).settings(settings)
.sbomType(sbomType).sbom(sbom).customCert(customCert).proxy(proxy).settings(this.settings)
.scanConfig(this.scanConfig)
.build());
this.scanFileProcessor = Objects.requireNonNullElseGet(scanFileProcessor, () ->
ScanFileProcessor.builder().winnowing(this.winnowing).scanApi(this.scanApi).build());
this.wfpFileProcessor = Objects.requireNonNullElseGet(wfpFileProcessor, () -> WfpFileProcessor.builder()
.winnowing(this.winnowing)
.build());
this.settings = Objects.requireNonNullElseGet(settings, () -> ScanossSettings.builder().build());
this.postProcessor = Objects.requireNonNullElseGet(postProcessor, () ->
ScannerPostProcessor.builder().build());

Expand Down
75 changes: 74 additions & 1 deletion src/main/java/com/scanoss/Winnowing.java
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,10 @@ public class Winnowing {
@Builder.Default
private int snippetLimit = MAX_LONG_LINE_CHARS; // Enable limiting of size of a single line of snippet generation
@Builder.Default
private boolean skipHeaders = false; // Skip license headers, comments and imports at the beginning of files
@Builder.Default
private int skipHeadersLimit = 0; // Maximum number of header lines to skip (0 = auto-detect)
@Builder.Default
private Map<String, String> obfuscationMap = new ConcurrentHashMap<>();

/**
Expand Down Expand Up @@ -168,6 +172,12 @@ public String wfpForContents(@NonNull String filename, Boolean binFile, byte[] c
wfpBuilder.append(String.format("hpsm=%s\n", Hpsm.calcHpsm(contents)));
}

int skipLines = 0;
if (this.skipHeaders) {
skipLines = detectHeaderLines(fileContents, this.skipHeadersLimit);
log.trace("Skipping {} header lines for snippet generation: {}", skipLines, filename);
}

String gram = "";
List<Long> window = new ArrayList<>();
char normalized;
Expand All @@ -183,7 +193,7 @@ public String wfpForContents(@NonNull String filename, Boolean binFile, byte[] c
} else {
normalized = WinnowingUtils.normalize(c);
}
if (normalized > 0) {
if (normalized > 0 && line > skipLines) {
gram += normalized;
if (gram.length() >= ScanossConstants.GRAM) {
Long gramCRC32 = crc32c(gram);
Expand Down Expand Up @@ -312,6 +322,69 @@ private Boolean skipSnippets(@NonNull String filename, char[] contents) {
return false;
}

/**
* Detect the number of header lines at the beginning of a file.
* Header lines include license comment blocks, single-line comments,
* blank lines, and import/package statements.
*
* @param contents file contents as char array
* @param maxLines maximum number of header lines to detect (0 = no limit)
* @return number of header lines detected
*/
int detectHeaderLines(char[] contents, int maxLines) {
int headerLines = 0;
boolean inBlockComment = false;
int lineStart = 0;

for (int i = 0; i <= contents.length; i++) {
if (i == contents.length || contents[i] == '\n') {
String line = new String(contents, lineStart, i - lineStart).trim();

if (inBlockComment) {
headerLines++;
if (line.contains("*/")) {
inBlockComment = false;
}
} else if (line.isEmpty()) {
headerLines++;
} else if (line.startsWith("//") || line.startsWith("#!") || line.startsWith("# ")) {
headerLines++;
} else if (line.startsWith("/*")) {
headerLines++;
if (!line.contains("*/")) {
inBlockComment = true;
}
} else if (line.startsWith("*") || line.startsWith("* ")) {
headerLines++;
} else if (isImportOrPackageLine(line)) {
headerLines++;
} else {
break; // Non-header line found
}

if (maxLines > 0 && headerLines >= maxLines) {
break;
}

lineStart = i + 1;
}
}

return headerLines;
}

/**
* Check if a line is an import or package declaration.
*
* @param line trimmed source line
* @return true if the line is an import/package/include statement
*/
private boolean isImportOrPackageLine(String line) {
return line.startsWith("import ") || line.startsWith("package ") ||
line.startsWith("from ") || line.startsWith("#include ") ||
line.startsWith("using ") || line.startsWith("require ");
}

/**
* Try to detect if this is a text file or not
*
Expand Down
57 changes: 56 additions & 1 deletion src/main/java/com/scanoss/cli/ScanCommandLine.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.scanoss.Scanner;
import com.scanoss.exceptions.ScannerException;
import com.scanoss.exceptions.WinnowingException;
import com.scanoss.settings.ScanConfig;
import com.scanoss.settings.ScanossSettings;
import com.scanoss.utils.JsonUtils;
import com.scanoss.utils.ProxyUtils;
Expand Down Expand Up @@ -105,6 +106,27 @@ class ScanCommandLine implements Runnable {
@picocli.CommandLine.Option(names = {"-H", "--hpsm"}, description = "Use High Precision Snippet Matching algorithm")
private boolean enableHpsm = false;

@picocli.CommandLine.Option(names = {"--min-snippet-hits"}, description = "Minimum snippet hits required (0 = unset, uses server config)")
private int minSnippetHits = 0;

@picocli.CommandLine.Option(names = {"--min-snippet-lines"}, description = "Minimum snippet lines required (0 = unset, uses server config)")
private int minSnippetLines = 0;

@picocli.CommandLine.Option(names = {"--honour-file-exts"}, description = "Honour file extensions (true|false|unset)", arity = "1")
private String honourFileExts = null;

@picocli.CommandLine.Option(names = {"--ranking"}, description = "Enable/disable ranking (true|false|unset)", arity = "1")
private String ranking = null;

@picocli.CommandLine.Option(names = {"--ranking-threshold"}, description = "Ranking threshold value (-1 = unset, uses server config)")
private int rankingThreshold = -1;

@picocli.CommandLine.Option(names = {"--skip-headers"}, description = "Skip license headers, comments and imports at the beginning of files (applies locally)")
private boolean skipHeaders = false;

@picocli.CommandLine.Option(names = {"--skip-headers-limit"}, description = "Skip limit for license headers (0 = unset, applies locally)")
private int skipHeadersLimit = 0;

@picocli.CommandLine.Parameters(arity = "1", description = "file/folder to scan")
private String fileFolder;

Expand Down Expand Up @@ -160,11 +182,12 @@ public void run() {
printMsg(err, String.format("Using flags %s", scanFlags));
}
}
ScanConfig cliScanConfig = buildCliScanConfig();
scanner = Scanner.builder().skipSnippets(skipSnippets).allFolders(allFolders).allExtensions(allExtensions)
.hiddenFilesFolders(allHidden).numThreads(numThreads).url(apiUrl).apiKey(apiKey)
.retryLimit(retryLimit).timeout(Duration.ofSeconds(timeoutLimit)).scanFlags(scanFlags)
.snippetLimit(snippetLimit).customCert(caCertPem).proxy(proxy).hpsm(enableHpsm)
.settings(settings).obfuscate(obfuscate)
.settings(settings).obfuscate(obfuscate).cliScanConfig(cliScanConfig)
.build();

File f = new File(fileFolder);
Expand Down Expand Up @@ -198,6 +221,38 @@ private String loadFileToString(@NonNull String filename) {
}
}

/**
* Build a ScanConfig from CLI arguments.
*
* @return ScanConfig populated with CLI-provided values
*/
private ScanConfig buildCliScanConfig() {
ScanConfig.ScanConfigBuilder builder = ScanConfig.builder()
.minSnippetHits(minSnippetHits)
.minSnippetLines(minSnippetLines)
.rankingThreshold(rankingThreshold)
.skipHeaders(skipHeaders)
.skipHeadersLimit(skipHeadersLimit);

builder.honourFileExts(parseTriStateBoolean(honourFileExts));
builder.rankingEnabled(parseTriStateBoolean(ranking));

return builder.build();
}

/**
* Parse a tri-state boolean string value.
*
* @param value the string value ("true", "false", "unset", or null)
* @return Boolean.TRUE, Boolean.FALSE, or null for unset
*/
private static Boolean parseTriStateBoolean(String value) {
if (value == null || value.equalsIgnoreCase("unset")) {
return null;
}
return Boolean.parseBoolean(value);
}

/**
* Scan the specified file and output the results
*
Expand Down
23 changes: 22 additions & 1 deletion src/main/java/com/scanoss/rest/ScanApi.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.scanoss.dto.SbomLegacy;
import com.scanoss.exceptions.ScanApiException;
import com.scanoss.settings.Rule;
import com.scanoss.settings.ScanConfig;
import com.scanoss.settings.ScanossSettings;
import com.scanoss.utils.JsonUtils;
import com.scanoss.utils.PackageDetails;
Expand Down Expand Up @@ -72,12 +73,14 @@ public class ScanApi {
private Proxy proxy; // Proxy configuration
private String baseUrl; // SCANOSS base API URI (to used instead of url)
private ScanossSettings settings;
private ScanConfig scanConfig; // Resolved scan configuration parameters
@SuppressWarnings("unused")
private ScanApi(String scanType, Duration timeout, Integer retryLimit, String url, String apiKey, String flags,
String sbomType, String sbom,
OkHttpClient okHttpClient, Map<String, String> headers, String customCert,
Proxy proxy, String baseUrl, ScanossSettings settings) {
Proxy proxy, String baseUrl, ScanossSettings settings, ScanConfig scanConfig) {
this.settings = settings;
this.scanConfig = scanConfig;
this.scanType = scanType;
this.timeout = timeout;
this.retryLimit = retryLimit;
Expand Down Expand Up @@ -178,6 +181,24 @@ public String scan(String wfp, String context, int scanID) throws ScanApiExcepti
data.put("type", "identify");
}

// Add scan configuration parameters (only when set/non-default)
if (scanConfig != null) {
if (scanConfig.isMinSnippetHitsSet()) {
data.put("min_snippet_hits", String.valueOf(scanConfig.getMinSnippetHits()));
}
if (scanConfig.isMinSnippetLinesSet()) {
data.put("min_snippet_lines", String.valueOf(scanConfig.getMinSnippetLines()));
}
if (scanConfig.isHonourFileExtsSet()) {
data.put("honour_file_exts", String.valueOf(scanConfig.getHonourFileExts()));
}
if (scanConfig.isRankingEnabledSet()) {
data.put("ranking_enabled", String.valueOf(scanConfig.getRankingEnabled()));
}
if (scanConfig.isRankingThresholdSet()) {
data.put("ranking_threshold", String.valueOf(scanConfig.getRankingThreshold()));
}
}

Request request; // Create multipart request
try {
Expand Down
Loading
Loading