diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..917e4cb --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,38 @@ +name: Java with Nexus Repository + +on: + release: + types: [published] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + - name: Set up Apache Maven Central + uses: actions/setup-java@v1 + with: # running setup-java again overwrites the settings.xml + java-version: 11 + server-id: ossrh + server-username: OSSRH_USERNAME + server-password: OSSRH_TOKEN + gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Build with Maven + run: mvn package -Dgpg.skip=false --file pom.xml + - name: Get package version + run: echo ::set-output name=PACKAGE_VERSION::$(mvn -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec) + id: scanner-version + - name: Get Version + run: echo "Current package version is ${{ steps.scanner-version.outputs.PACKAGE_VERSION }}" + - name: Release to Central Repository + uses: samuelmeuli/action-maven-publish@master + with: + gpg_private_key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} + gpg_passphrase: ${{ secrets.GPG_PASSPHRASE }} + nexus_username: ${{ secrets.OSSRH_USERNAME }} + nexus_password: ${{ secrets.OSSRH_TOKEN }} + server_id: ossrh + diff --git a/README.md b/README.md index 6d83aa6..0adb312 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,20 @@ -# scanner.java +# scanner.java - Deprecated Simple Java implementation of a scanner for the OSSKB (Open Source KB). +** Deprecated ** + +This project is not actively maintained. + +For an up-to-date API/SDK see the [SCANOSS Java Library](https://github.com/scanoss/scanoss.java) or the [Python CLI](https://github.com/scanoss/scanoss.py). + # Usage +The code can be used as a library, or as a CLI. ## Standalone - -A jar with dependencies is generated by Maven Assembly plugin using `mvn package`. +A jar with dependencies is generated by Maven Assembly plugin using `mvn package`. Usage: - ```% java -jar target/scanner-*-jar-with-dependencies.jar usage: scanner -blacklist Scan and blacklist components in SBOM file @@ -21,11 +26,25 @@ usage: scanner -o,--output Save output to file ``` - ## Maven library +To build the code please run: +``` +mvn clean package +``` +And to test run: +``` +java -jar target/scanner-*-jar-with-dependencies.jar +``` +To publish please run: +``` +mvn deploy -Dmaven.test.skip=true +``` + +It should then be possible to find it on [Maven Central](https://oss.sonatype.org/#nexus-search;quick~com.scanoss). + +If the staging environment is used, then details for releasing it can be found [here](https://central.sonatype.org/publish/release/). # How does it work - This reference code illustrates the usage of the SCANOSS API to obtain file identification against the OSSKB without sending the actual code, but instead the WFP hashes The provided FILE is read into memory, WFP fingerprints are calculated and sent to the [OSSKB API](https://osskb.org) @@ -33,5 +52,4 @@ The provided FILE is read into memory, WFP fingerprints are calculated and sent Results are printed via STDOUT. # License - -scanner.java is released under the Apache 2.0 license. Please check the LICENSE file for further details. \ No newline at end of file +scanner.java is released under the Apache 2.0 license. Please check the LICENSE file for further details. diff --git a/pom.xml b/pom.xml index 172139f..5e9c987 100644 --- a/pom.xml +++ b/pom.xml @@ -4,9 +4,9 @@ 4.0.0 com.scanoss scanner - 1.1.1 + 1.1.6 scanner.java - Java implementation of a scanner that works with SCANOSS API and OSSKB. + Java implementation of a scanner that works with SCANOSS API and OSSKB. SCANOSS https://www.scanoss.com @@ -23,6 +23,7 @@ ossrh https://oss.sonatype.org/service/local/staging/deploy/maven2/ + @@ -34,8 +35,8 @@ - Juan M Salamanca - juan.m.salamanca@scanoss.com + SCANOSS Dev + infra@scanoss.com SCANOSS https://www.scanoss.com @@ -48,7 +49,9 @@ maven-compiler-plugin 3.8.1 - 12 + 11 + 11 + 11 @@ -78,6 +81,7 @@ org.apache.maven.plugins maven-source-plugin + 3.2.1 attach-sources @@ -99,6 +103,24 @@ + + + + + + + + + + + + + + + + + + org.apache.maven.plugins maven-deploy-plugin @@ -111,12 +133,14 @@ - org.apache.maven.plugins maven-gpg-plugin - 1.5 + + false + + 3.0.1 sign-artifacts @@ -124,9 +148,13 @@ sign - - ${gpg.keyname} - ${gpg.keyname} + + ${gpg.keyname} + ${gpg.keyname} + + --pinentry-mode + loopback + @@ -159,16 +187,12 @@ slf4j-api 1.7.30 - junit junit 4.13.1 test - - - https://github.com/scanoss/scanner.java - \ No newline at end of file + diff --git a/src/main/java/com/scanoss/scanner/BlacklistRules.java b/src/main/java/com/scanoss/scanner/BlacklistRules.java index 21150c2..36d80f6 100644 --- a/src/main/java/com/scanoss/scanner/BlacklistRules.java +++ b/src/main/java/com/scanoss/scanner/BlacklistRules.java @@ -21,6 +21,7 @@ import java.util.Arrays; import java.util.List; +import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; public class BlacklistRules { @@ -28,7 +29,7 @@ public class BlacklistRules { public static final List BLACKLIST_EXTS; static { - BLACKLIST_EXTS = Arrays.asList(new String[]{"1", "2", "3", "4", "5", "6", "7", "8", "9", "ac", + BLACKLIST_EXTS = Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9", "ac", "am", "bmp", "build", "cfg", "chm", "changelog", "class", "cmake", "conf", "config", "contributors", "copying", "csproj", "css", "csv", "cvsignore", "dat", "data", "dtd", "dts", "dtsi", "eps", "geojson", "gif", "gitignore", "glif", "gmo", "guess", "hex", "html", "htm", "ico", "in", "inc", "info", "ini", @@ -36,15 +37,15 @@ public class BlacklistRules { "meta", "mxml", "notice", "out", "pdf", "pem", "phtml", "png", "po", "prefs", "properties", "readme", "result", "rst", "scss", "sha", "sha1", "sha2", "sha256", "sln", "spec", "sub", "svg", "svn-base", "tab", "template", "test", "tex", "todo", "txt", "utf-8", "version", "vim", "wav", "xht", "xhtml", "xml", "xpm", - "xsd", "xul", "yaml", "yml"}); + "xsd", "xul", "yaml", "yml"); } public static boolean isMarkupOrJSON(String src) { - return (src.charAt(0) == '{' || src.startsWith(" 0 ) && (src.charAt(0) == '{' || src.startsWith("() { + Files.walkFileTree(Paths.get(dir), new SimpleFileVisitor<>() { @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - throws IOException { + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { if (!Files.isDirectory(file) && !BlacklistRules.hasBlacklistedExt(file.toString())) { try { - wfp.append(Winnowing.wfpForFile(file.toString(), file.toString())); - } catch (NoSuchAlgorithmException | IOException e) { - log.warn("Exception while creating wfp for file: {}",file.toString(),e); + String wfpString = Winnowing.wfpForFile(file.toString(), file.toString()); + if (wfpString != null && !wfpString.isEmpty()) + wfp.append(wfpString); + } catch (Exception e) { + log.warn("Exception while creating wfp for file: {}", file, e); } } return FileVisitResult.CONTINUE; @@ -122,9 +124,12 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) FileUtils.writeStringToFile(new File(TMP_SCAN_WFP), wfp.toString(), StandardCharsets.UTF_8); ScanDetails details = new ScanDetails(TMP_SCAN_WFP, scanType, sbomPath, format); InputStream inputStream = doScan(details); - OutputStream out = StringUtils.isEmpty(outfile) ? System.out : new FileOutputStream(new File(outfile)); - + OutputStream out = StringUtils.isEmpty(outfile) ? System.out : new FileOutputStream(outfile); IOUtils.copy(inputStream, out); + inputStream.close(); + if ( ! StringUtils.isEmpty(outfile) ) + out.close(); + } /** @@ -143,10 +148,13 @@ public void scanFileAndSave(String filename, ScanType scanType, String sbomPath, throws NoSuchAlgorithmException, IOException, InterruptedException { InputStream inputStream = scanFile(filename, scanType, sbomPath, format); - OutputStream out = StringUtils.isEmpty(outfile) ? System.out : new FileOutputStream(new File(outfile)); - - IOUtils.copy(inputStream, out); - + if ( inputStream != null ) { + OutputStream out = StringUtils.isEmpty(outfile) ? System.out : new FileOutputStream(outfile); + IOUtils.copy(inputStream, out); + inputStream.close(); + if (!StringUtils.isEmpty(outfile)) + out.close(); + } } public void scanFileAndSave(String filename, ScanDetails scanDetails, String outfile) @@ -166,13 +174,14 @@ public static void main(String[] args) throws ParseException, NoSuchAlgorithmExc new Option("f", "format", true, "Optional format for the scan result. One of: plain, spdx, cyclonedx")); options.addOption(new Option("h", false, "Shows usage")); Option input = new Option("i", "input", true, "The file to be scanned"); - input.setRequired(true); + input.setRequired(false); options.addOption(input); CommandLineParser parser = new DefaultParser(); CommandLine cmd = parser.parse(options, args); if (ArrayUtils.isEmpty(args) || cmd.hasOption("h")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("scanner", options); + System.exit(1); } String filename = cmd.getOptionValue("i"); diff --git a/src/main/java/com/scanoss/scanner/Winnowing.java b/src/main/java/com/scanoss/scanner/Winnowing.java index 35617fa..64c7b57 100644 --- a/src/main/java/com/scanoss/scanner/Winnowing.java +++ b/src/main/java/com/scanoss/scanner/Winnowing.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.nio.charset.Charset; import java.nio.file.Files; -import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -87,7 +86,6 @@ public class Winnowing { private static final int GRAM = 30; private static final int WINDOW = 64; - private static final long MAX_CRC32 = 4294967296L; private static byte[] toLittleEndian(long number) { @@ -147,24 +145,58 @@ private static long min(List l) { return sortedList.get(0); } + private static String byteToHex(Byte num) { + char[] hexDigits = new char[2]; + hexDigits[0] = Character.forDigit((num >> 4) & 0xF, 16); + hexDigits[1] = Character.forDigit((num & 0xF), 16); + return new String(hexDigits); + } + + private static String encodeHexString(Byte[] byteArray) { + StringBuffer hexStringBuffer = new StringBuffer(); + for (int i = 0; i < byteArray.length; i++) { + hexStringBuffer.append(byteToHex(byteArray[i])); + } + return hexStringBuffer.toString(); + } + + private static Byte CRC8(byte[] source, int offset, int length) { + int wCRCin = 0x00; + // Integer.reverse(0x31) >>> 24 + int wCPoly = 0x8C; + for (int i = offset, cnt = offset + length; i < cnt; i++) { + wCRCin ^= ((long) source[i] & 0xFF); + for (int j = 0; j < 8; j++) { + if ((wCRCin & 0x01) != 0) { + wCRCin >>= 1; + wCRCin ^= wCPoly; + } else { + wCRCin >>= 1; + } + } + } + return Byte.valueOf((byte) (wCRCin ^= 0x00)); + } + /** * Calculates the WFP * * @param filename * @param filepath * @return - * @throws NoSuchAlgorithmException * @throws IOException */ - public static String wfpForFile(String filename, String filepath) throws NoSuchAlgorithmException, IOException { + public static String wfpForFile(String filename, String filepath) throws IOException { File file = new File(filepath); String fileContents = FileUtils.readFileToString(file, Charset.defaultCharset()); String fileMD5 = DigestUtils.md5Hex(fileContents); StringBuilder wfpBuilder = new StringBuilder(); + StringBuilder hpsmBuilder = new StringBuilder(); + hpsmBuilder.append("hpsm="); wfpBuilder.append(String.format("file=%s,%d,%s\n", fileMD5, fileContents.length(), filename)); - // Skip snippet analysis for binaries or non source code files. - if (isBinaryFile(file) || BlacklistRules.isMarkupOrJSON(fileContents)) { + // Skip snippet analysis for binaries or non source code files, or empty files. + if (fileContents.length() == 0 || isBinaryFile(file) || BlacklistRules.isMarkupOrJSON(fileContents)) { return wfpBuilder.toString(); } String gram = ""; @@ -175,18 +207,23 @@ public static String wfpForFile(String filename, String filepath) throws NoSuchA int lastLine = 0; int line = 1; String output = ""; - + String normalized_line = ""; + ArrayList lines_crc = new ArrayList(); + //byte lines_crc[] = new byte[MAX_LINES]; for (char c : fileContents.toCharArray()) { if (c == '\n') { + lines_crc.add(CRC8(normalized_line.getBytes(), 0, normalized_line.length())); line++; normalized = 0; + normalized_line = ""; } else { normalized = normalize(c); + } if (normalized > 0) { gram += normalized; - + normalized_line += normalized; if (gram.length() >= GRAM) { Long gramCRC32 = crc32c(gram); window.add(gramCRC32); @@ -223,8 +260,12 @@ public static String wfpForFile(String filename, String filepath) throws NoSuchA { wfpBuilder.append(output + "\n"); } - - return wfpBuilder.toString(); + Byte aux [] = new Byte[lines_crc.size()]; + aux = lines_crc.toArray(aux); + hpsmBuilder.append(encodeHexString((aux))); + hpsmBuilder.append("\n"); + hpsmBuilder.append(wfpBuilder.toString()); + return hpsmBuilder.toString(); } diff --git a/src/test/java/com/scanoss/scanner/ScannerTest.java b/src/test/java/com/scanoss/scanner/ScannerTest.java index 66573ea..3bcb4ae 100644 --- a/src/test/java/com/scanoss/scanner/ScannerTest.java +++ b/src/test/java/com/scanoss/scanner/ScannerTest.java @@ -28,4 +28,10 @@ public void scanFile_try_c_plain() throws Exception{ scanner.scanFileAndSave("src/main/java/com/scanoss/scanner/Winnowing.java", null, "", null, ""); } + @Test + public void scanFile_try_empty() throws Exception{ + Scanner scanner = new Scanner(ScannerConf.defaultConf()); + scanner.scanFileAndSave("src/test/resources/empty.java", null, "", null, ""); + } + } diff --git a/src/test/java/com/scanoss/scanner/WinnowingTest.java b/src/test/java/com/scanoss/scanner/WinnowingTest.java index 1124257..b172bd6 100644 --- a/src/test/java/com/scanoss/scanner/WinnowingTest.java +++ b/src/test/java/com/scanoss/scanner/WinnowingTest.java @@ -23,7 +23,6 @@ import java.io.File; import java.io.IOException; import java.nio.charset.Charset; -import java.security.NoSuchAlgorithmException; import org.apache.commons.io.FileUtils; import org.junit.Test; @@ -31,14 +30,22 @@ public class WinnowingTest { @Test - public void wfpForFile_correct_for_try_c() throws IOException, NoSuchAlgorithmException { + public void wfpForFile_correct_for_try_c() throws IOException { String actualWFP = Winnowing.wfpForFile("Winnowing.java", "src/main/java/com/scanoss/scanner/Winnowing.java").trim(); String expectedWFP = FileUtils.readFileToString(new File("src/test/resources/Winnowing.java-scan.wfp"), Charset.defaultCharset()); assertEquals(expectedWFP, actualWFP); - + } + + @Test + public void wfpForEmptyfile() throws IOException { + String actualWFP1 = Winnowing.wfpForFile("empty.java", "src/test/resources/empty.java").trim(); + String actualWFP2 = Winnowing.wfpForFile("empty.java", "src/test/resources/empty2.java").trim(); + + System.out.printf("Empty output: %s\n", actualWFP1 ); + System.out.printf("Empty output: %s\n", actualWFP2 ); } } diff --git a/src/test/resources/Winnowing.java-scan.wfp b/src/test/resources/Winnowing.java-scan.wfp index 1f4e68e..e543547 100644 --- a/src/test/resources/Winnowing.java-scan.wfp +++ b/src/test/resources/Winnowing.java-scan.wfp @@ -1,4 +1,4 @@ -file=d82283e68c7e1e28ee1f46c937006a01,6248,Winnowing.java +file=38d4b50f664097ff623bfc9433eef0ab,6184,Winnowing.java 5=a4459cc2,01a1c17f,bd4cfaa3,fb0d20c4 6=028ead5a,cab55853,d7aead00 8=93951dcd @@ -9,67 +9,66 @@ file=d82283e68c7e1e28ee1f46c937006a01,6248,Winnowing.java 19=55ac5188 21=25f149a3,56097ed3 24=623ca392 -25=5aa8438f,04ef3f50,ec4c861a -28=8b39a228,b32da39f -32=5036b648,f95f1c75 -33=7fdcc4a5 -39=daf75ef5 -42=78bd20db,93281542 -43=512bf474,99432b1e,0bd87d29,2b07f103 -46=bea157c8 -49=e103d9c3,f6a3ddaa,79cf8773 -52=b2e2ac51,1d66e8b6,6332c139,5dd77eb6,aa98d03f,99f93ab7 -53=cb31ef98 -61=54a37629,cc530d26,06bb90a2 -63=4488464d -65=ff9833ab -70=1642121b -72=fa954895,944da194 -73=9fa5d906 -77=53ecefb1 -78=9b13ffe7,5c790893 -79=60bdaf1e -82=b43ed71c,1d4507dc -83=226c658b,97d9dab0,9f695265,76cbc885,328ccdac -89=772dd985,f1ff2584 -91=c3bf7e36,4bac267b,4cdc341c -93=84b95d5f -97=31f838ee -102=61a6718a,9c8b511e,fa2c10e9 -110=af089fc7 -114=4db0429a -115=e1119814 -117=7c168094 -124=6b459701 -125=088c6b9d -131=f429fc97 -132=62b7869f,79aec86d -137=4c41f918 -139=dbcbf0cd,025e3f25 -140=3e3a28f9 -144=91e27ec4,7995d0e7,e9fadd54,651bfb7c -146=8cda25dc -147=d80610d2 -151=0abb6be7 -156=a5296e0a -159=05abf206,c420ccb1 -161=cecbee5c,3b5bde97 -162=220e81d6,2ba1375c -164=dec5f2d3 -165=199bc335,a0501b0f -166=c0864ba6 -167=84e9e8be,fcc3049d -171=8774ed34 -174=820edebb -179=0ce8df87 -182=eb93cf74 -187=dae61b62 -190=730fc485 -194=fd6bbf08 -197=77e7a292 -199=75750d1d -200=c5bb8fda,9fccba00 -204=99afdefd -212=6d7e9570 -224=13e6ea16,9fccba00 -227=732b841d,c3f09b41 \ No newline at end of file +25=5aa8438f,55b31ec9 +28=b32da39f +31=5036b648,f95f1c75 +32=7fdcc4a5 +38=daf75ef5 +41=78bd20db,93281542 +42=512bf474,99432b1e,0bd87d29,2b07f103 +45=bea157c8 +48=e103d9c3,f6a3ddaa,79cf8773 +51=b2e2ac51,1d66e8b6,6332c139,5dd77eb6,aa98d03f,99f93ab7 +52=cb31ef98 +60=54a37629,cc530d26,06bb90a2 +62=4488464d +64=ff9833ab +69=1642121b +71=fa954895,944da194 +72=9fa5d906 +76=53ecefb1 +77=9b13ffe7,5c790893 +78=60bdaf1e +81=b43ed71c,1d4507dc +82=226c658b,97d9dab0,9f695265,76cbc885,328ccdac +88=772dd985,f1ff2584 +90=c3bf7e36,4bac267b,4cdc341c +92=84b95d5f +96=31f838ee +101=61a6718a,9c8b511e,fa2c10e9 +109=af089fc7 +113=4db0429a +114=e1119814 +116=7c168094 +123=6b459701 +124=088c6b9d +130=f429fc97 +131=62b7869f,79aec86d +136=4c41f918 +138=dbcbf0cd,025e3f25 +139=3e3a28f9 +143=91e27ec4,7995d0e7,e9fadd54,651bfb7c +145=8cda25dc +146=d80610d2 +150=0abb6be7 +157=9d3d0d53,3cd157d6,c420ccb1 +159=3b5bde97 +160=220e81d6,2ba1375c +162=dec5f2d3 +163=199bc335,a0501b0f +164=c0864ba6 +165=2be4566b,fcc3049d +169=8774ed34 +172=820edebb +177=0ce8df87 +180=eb93cf74 +185=dae61b62 +188=730fc485 +192=fd6bbf08 +195=77e7a292 +197=75750d1d +198=c5bb8fda,9fccba00 +202=99afdefd +210=6d7e9570 +222=13e6ea16,9fccba00 +225=732b841d,c3f09b41 \ No newline at end of file diff --git a/src/test/resources/empty.java b/src/test/resources/empty.java new file mode 100644 index 0000000..e69de29 diff --git a/src/test/resources/empty2.java b/src/test/resources/empty2.java new file mode 100644 index 0000000..e69de29