From 13245e003bc18836be57c429f5f672de662dfd0a Mon Sep 17 00:00:00 2001 From: guangwu Date: Fri, 12 May 2023 21:34:57 +0800 Subject: [PATCH 01/11] chore: more idiomatic code (#212) Signed-off-by: guoguangwu --- reporter/reporter.go | 6 +++--- spdx/v2/v2_1/tagvalue/writer/save_package.go | 8 ++++---- spdx/v2/v2_2/rdf/reader/license_utils.go | 2 +- spdx/v2/v2_2/rdf/reader/parse_snippet_info.go | 3 +++ spdx/v2/v2_2/tagvalue/writer/save_package.go | 8 ++++---- spdx/v2/v2_3/rdf/reader/license_utils.go | 2 +- spdx/v2/v2_3/tagvalue/writer/save_package.go | 6 +++--- 7 files changed, 19 insertions(+), 16 deletions(-) diff --git a/reporter/reporter.go b/reporter/reporter.go index af883437..7ef32d0e 100644 --- a/reporter/reporter.go +++ b/reporter/reporter.go @@ -17,7 +17,7 @@ import ( // io.Writer, and outputs to the io.Writer a tabulated count of // the number of Files for each unique LicenseConcluded in the set. func Generate(pkg *spdx.Package, w io.Writer) error { - if pkg.FilesAnalyzed == false { + if !pkg.FilesAnalyzed { return fmt.Errorf("Package FilesAnalyzed is false") } totalFound, totalNotFound, foundCounts := countLicenses(pkg) @@ -29,10 +29,10 @@ func Generate(pkg *spdx.Package, w io.Writer) error { fmt.Fprintf(wr, "%d\t TOTAL\n", totalFound+totalNotFound) fmt.Fprintf(wr, "\n") - counts := []struct { + var counts []struct { lic string count int - }{} + } for k, v := range foundCounts { var entry struct { lic string diff --git a/spdx/v2/v2_1/tagvalue/writer/save_package.go b/spdx/v2/v2_1/tagvalue/writer/save_package.go index 56f94418..19d47cd2 100644 --- a/spdx/v2/v2_1/tagvalue/writer/save_package.go +++ b/spdx/v2/v2_1/tagvalue/writer/save_package.go @@ -42,14 +42,14 @@ func renderPackage(pkg *spdx.Package, w io.Writer) error { if pkg.PackageDownloadLocation != "" { fmt.Fprintf(w, "PackageDownloadLocation: %s\n", pkg.PackageDownloadLocation) } - if pkg.FilesAnalyzed == true { - if pkg.IsFilesAnalyzedTagPresent == true { + if pkg.FilesAnalyzed { + if pkg.IsFilesAnalyzedTagPresent { fmt.Fprintf(w, "FilesAnalyzed: true\n") } } else { fmt.Fprintf(w, "FilesAnalyzed: false\n") } - if pkg.PackageVerificationCode.Value != "" && pkg.FilesAnalyzed == true { + if pkg.PackageVerificationCode.Value != "" && pkg.FilesAnalyzed { if len(pkg.PackageVerificationCode.ExcludedFiles) == 0 { fmt.Fprintf(w, "PackageVerificationCode: %s\n", pkg.PackageVerificationCode.Value) } else { @@ -70,7 +70,7 @@ func renderPackage(pkg *spdx.Package, w io.Writer) error { if pkg.PackageLicenseConcluded != "" { fmt.Fprintf(w, "PackageLicenseConcluded: %s\n", pkg.PackageLicenseConcluded) } - if pkg.FilesAnalyzed == true { + if pkg.FilesAnalyzed { for _, s := range pkg.PackageLicenseInfoFromFiles { fmt.Fprintf(w, "PackageLicenseInfoFromFiles: %s\n", s) } diff --git a/spdx/v2/v2_2/rdf/reader/license_utils.go b/spdx/v2/v2_2/rdf/reader/license_utils.go index bdf2ba93..87633a03 100644 --- a/spdx/v2/v2_2/rdf/reader/license_utils.go +++ b/spdx/v2/v2_2/rdf/reader/license_utils.go @@ -71,7 +71,7 @@ func getAlgorithmFromURI(algorithmURI string) (checksumAlgorithm string, err err // from a list of licenses, it returns a // list of string representation of those licenses. func mapLicensesToStrings(licences []AnyLicenseInfo) []string { - res := make([]string, len(licences), len(licences)) + res := make([]string, len(licences)) for i, lic := range licences { res[i] = lic.ToLicenseString() } diff --git a/spdx/v2/v2_2/rdf/reader/parse_snippet_info.go b/spdx/v2/v2_2/rdf/reader/parse_snippet_info.go index 6b920e7d..a74238be 100644 --- a/spdx/v2/v2_2/rdf/reader/parse_snippet_info.go +++ b/spdx/v2/v2_2/rdf/reader/parse_snippet_info.go @@ -34,6 +34,9 @@ func (parser *rdfParser2_2) getSnippetInformationFromNode2_2(node *gordfParser.N return nil, err } docElemID, err := ExtractDocElementID(getLastPartOfURI(siTriple.Object.ID)) + if err != nil { + return nil, err + } si.SnippetFromFileSPDXIdentifier = docElemID.ElementRefID case SPDX_RANGE: // cardinality: min 1 diff --git a/spdx/v2/v2_2/tagvalue/writer/save_package.go b/spdx/v2/v2_2/tagvalue/writer/save_package.go index 8e04d6a8..56e96f7f 100644 --- a/spdx/v2/v2_2/tagvalue/writer/save_package.go +++ b/spdx/v2/v2_2/tagvalue/writer/save_package.go @@ -42,14 +42,14 @@ func renderPackage(pkg *spdx.Package, w io.Writer) error { if pkg.PackageDownloadLocation != "" { fmt.Fprintf(w, "PackageDownloadLocation: %s\n", pkg.PackageDownloadLocation) } - if pkg.FilesAnalyzed == true { - if pkg.IsFilesAnalyzedTagPresent == true { + if pkg.FilesAnalyzed { + if pkg.IsFilesAnalyzedTagPresent { fmt.Fprintf(w, "FilesAnalyzed: true\n") } } else { fmt.Fprintf(w, "FilesAnalyzed: false\n") } - if pkg.PackageVerificationCode.Value != "" && pkg.FilesAnalyzed == true { + if pkg.PackageVerificationCode.Value != "" && pkg.FilesAnalyzed { if len(pkg.PackageVerificationCode.ExcludedFiles) == 0 { fmt.Fprintf(w, "PackageVerificationCode: %s\n", pkg.PackageVerificationCode.Value) } else { @@ -70,7 +70,7 @@ func renderPackage(pkg *spdx.Package, w io.Writer) error { if pkg.PackageLicenseConcluded != "" { fmt.Fprintf(w, "PackageLicenseConcluded: %s\n", pkg.PackageLicenseConcluded) } - if pkg.FilesAnalyzed == true { + if pkg.FilesAnalyzed { for _, s := range pkg.PackageLicenseInfoFromFiles { fmt.Fprintf(w, "PackageLicenseInfoFromFiles: %s\n", s) } diff --git a/spdx/v2/v2_3/rdf/reader/license_utils.go b/spdx/v2/v2_3/rdf/reader/license_utils.go index 74ade497..0faa7401 100644 --- a/spdx/v2/v2_3/rdf/reader/license_utils.go +++ b/spdx/v2/v2_3/rdf/reader/license_utils.go @@ -71,7 +71,7 @@ func getAlgorithmFromURI(algorithmURI string) (checksumAlgorithm string, err err // from a list of licenses, it returns a // list of string representation of those licenses. func mapLicensesToStrings(licences []AnyLicenseInfo) []string { - res := make([]string, len(licences), len(licences)) + res := make([]string, len(licences)) for i, lic := range licences { res[i] = lic.ToLicenseString() } diff --git a/spdx/v2/v2_3/tagvalue/writer/save_package.go b/spdx/v2/v2_3/tagvalue/writer/save_package.go index 01574ddc..22ae2ccd 100644 --- a/spdx/v2/v2_3/tagvalue/writer/save_package.go +++ b/spdx/v2/v2_3/tagvalue/writer/save_package.go @@ -54,8 +54,8 @@ func renderPackage(pkg *spdx.Package, w io.Writer) error { if pkg.ValidUntilDate != "" { fmt.Fprintf(w, "ValidUntilDate: %s\n", pkg.ValidUntilDate) } - if pkg.FilesAnalyzed == true { - if pkg.IsFilesAnalyzedTagPresent == true { + if pkg.FilesAnalyzed { + if pkg.IsFilesAnalyzedTagPresent { fmt.Fprintf(w, "FilesAnalyzed: true\n") } } else { @@ -82,7 +82,7 @@ func renderPackage(pkg *spdx.Package, w io.Writer) error { if pkg.PackageLicenseConcluded != "" { fmt.Fprintf(w, "PackageLicenseConcluded: %s\n", pkg.PackageLicenseConcluded) } - if pkg.FilesAnalyzed == true { + if pkg.FilesAnalyzed { for _, s := range pkg.PackageLicenseInfoFromFiles { fmt.Fprintf(w, "PackageLicenseInfoFromFiles: %s\n", s) } From 564d598c05824d66f071057614699dee2a427c1e Mon Sep 17 00:00:00 2001 From: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com> Date: Wed, 17 May 2023 19:19:07 +0600 Subject: [PATCH 02/11] feat(json): add option to write json in multiline mode (#213) Signed-off-by: DmitriyLewen --- examples/9-tvtojson/exampletvtojson.go | 10 +++- json/writer.go | 26 +++++--- json/writer_test.go | 82 ++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 10 deletions(-) create mode 100644 json/writer_test.go diff --git a/examples/9-tvtojson/exampletvtojson.go b/examples/9-tvtojson/exampletvtojson.go index 5e625619..76ee8084 100644 --- a/examples/9-tvtojson/exampletvtojson.go +++ b/examples/9-tvtojson/exampletvtojson.go @@ -56,8 +56,16 @@ func main() { } defer w.Close() + var opt []json.WriteOption + // you can use WriteOption to change JSON format + // uncomment the following code to test it + /* + opt = append(opt, json.Indent(" ")) // to create multiline json + opt = append(opt, json.EscapeHTML(true)) // to escape HTML characters + */ + // try to save the document to disk as JSON file - err = json.Write(doc, w) + err = json.Write(doc, w, opt...) if err != nil { fmt.Printf("Error while saving %v: %v", fileOut, err) return diff --git a/json/writer.go b/json/writer.go index 88563a5e..a944dccb 100644 --- a/json/writer.go +++ b/json/writer.go @@ -9,17 +9,25 @@ import ( "github.com/spdx/tools-golang/spdx/common" ) -// Write takes an SPDX Document and an io.Writer, and writes the document to the writer in JSON format. -func Write(doc common.AnyDocument, w io.Writer) error { - buf, err := json.Marshal(doc) - if err != nil { - return err +type WriteOption func(*json.Encoder) + +func Indent(indent string) WriteOption { + return func(e *json.Encoder) { + e.SetIndent("", indent) } +} - _, err = w.Write(buf) - if err != nil { - return err +func EscapeHTML(escape bool) WriteOption { + return func(e *json.Encoder) { + e.SetEscapeHTML(escape) } +} - return nil +// Write takes an SPDX Document and an io.Writer, and writes the document to the writer in JSON format. +func Write(doc common.AnyDocument, w io.Writer, opts ...WriteOption) error { + e := json.NewEncoder(w) + for _, opt := range opts { + opt(e) + } + return e.Encode(doc) } diff --git a/json/writer_test.go b/json/writer_test.go new file mode 100644 index 00000000..f00f156c --- /dev/null +++ b/json/writer_test.go @@ -0,0 +1,82 @@ +package json_test + +import ( + "bytes" + "github.com/spdx/tools-golang/json" + "github.com/spdx/tools-golang/spdx/common" + spdx "github.com/spdx/tools-golang/spdx/v2/v2_3" + "github.com/stretchr/testify/assert" + "testing" +) + +func Test_Write(t *testing.T) { + tests := []struct { + name string + doc common.AnyDocument + option []json.WriteOption + want string + }{ + { + name: "happy path", + doc: spdx.Document{ + SPDXVersion: "2.3", + DocumentName: "test_doc", + }, + want: `{"spdxVersion":"2.3","dataLicense":"","SPDXID":"SPDXRef-","name":"test_doc","documentNamespace":"","creationInfo":null} +`, + }, + { + name: "happy path with Indent option", + doc: spdx.Document{ + SPDXVersion: "2.3", + DocumentName: "test_doc", + }, + option: []json.WriteOption{json.Indent(" ")}, + want: `{ + "spdxVersion": "2.3", + "dataLicense": "", + "SPDXID": "SPDXRef-", + "name": "test_doc", + "documentNamespace": "", + "creationInfo": null +} +`, + }, + { + name: "happy path with EscapeHTML==true option", + doc: spdx.Document{ + SPDXVersion: "2.3", + DocumentName: "test_doc_>", + }, + option: []json.WriteOption{json.EscapeHTML(true)}, + want: "{\"spdxVersion\":\"2.3\",\"dataLicense\":\"\",\"SPDXID\":\"SPDXRef-\",\"name\":\"test_doc_\\u003e\",\"documentNamespace\":\"\",\"creationInfo\":null}\n", + }, + { + name: "happy path with EscapeHTML==false option", + doc: spdx.Document{ + SPDXVersion: "2.3", + DocumentName: "test_doc_>", + }, + option: []json.WriteOption{json.EscapeHTML(false)}, + want: "{\"spdxVersion\":\"2.3\",\"dataLicense\":\"\",\"SPDXID\":\"SPDXRef-\",\"name\":\"test_doc_>\",\"documentNamespace\":\"\",\"creationInfo\":null}\n", + }, + { + name: "happy path with EscapeHTML==false option", + doc: spdx.Document{ + SPDXVersion: "2.3", + DocumentName: "test_doc_>", + }, + option: []json.WriteOption{json.EscapeHTML(false)}, + want: "{\"spdxVersion\":\"2.3\",\"dataLicense\":\"\",\"SPDXID\":\"SPDXRef-\",\"name\":\"test_doc_>\",\"documentNamespace\":\"\",\"creationInfo\":null}\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := new(bytes.Buffer) + err := json.Write(tt.doc, buf, tt.option...) + assert.NoError(t, err) + assert.Equal(t, tt.want, buf.String()) + }) + } +} From e56da91beaf4b40d725bf2406cb0e64caf1236f5 Mon Sep 17 00:00:00 2001 From: Brandon Lum Date: Wed, 17 May 2023 17:13:25 -0400 Subject: [PATCH 03/11] licenseListVersion optional Signed-off-by: Brandon Lum --- spdx/v2/v2_1/creation_info.go | 2 +- spdx/v2/v2_2/creation_info.go | 2 +- spdx/v2/v2_3/creation_info.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spdx/v2/v2_1/creation_info.go b/spdx/v2/v2_1/creation_info.go index 8b6ca182..c75e8ea8 100644 --- a/spdx/v2/v2_1/creation_info.go +++ b/spdx/v2/v2_1/creation_info.go @@ -11,7 +11,7 @@ import ( type CreationInfo struct { // 2.7: License List Version // Cardinality: optional, one - LicenseListVersion string `json:"licenseListVersion"` + LicenseListVersion string `json:"licenseListVersion,omitempty"` // 2.8: Creators: may have multiple keys for Person, Organization // and/or Tool diff --git a/spdx/v2/v2_2/creation_info.go b/spdx/v2/v2_2/creation_info.go index 8496462e..39082e7a 100644 --- a/spdx/v2/v2_2/creation_info.go +++ b/spdx/v2/v2_2/creation_info.go @@ -11,7 +11,7 @@ import ( type CreationInfo struct { // 6.7: License List Version // Cardinality: optional, one - LicenseListVersion string `json:"licenseListVersion"` + LicenseListVersion string `json:"licenseListVersion,omitempty"` // 6.8: Creators: may have multiple keys for Person, Organization // and/or Tool diff --git a/spdx/v2/v2_3/creation_info.go b/spdx/v2/v2_3/creation_info.go index 90ee4030..84d5bf08 100644 --- a/spdx/v2/v2_3/creation_info.go +++ b/spdx/v2/v2_3/creation_info.go @@ -10,7 +10,7 @@ import ( type CreationInfo struct { // 6.7: License List Version // Cardinality: optional, one - LicenseListVersion string `json:"licenseListVersion"` + LicenseListVersion string `json:"licenseListVersion,omitempty"` // 6.8: Creators: may have multiple keys for Person, Organization // and/or Tool From 7029eaf29c1740444de3375ea9285580a51aed55 Mon Sep 17 00:00:00 2001 From: Brandon Lum Date: Wed, 17 May 2023 17:13:39 -0400 Subject: [PATCH 04/11] package copyrighttext 2.3 optional Signed-off-by: Brandon Lum --- spdx/v2/v2_3/package.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spdx/v2/v2_3/package.go b/spdx/v2/v2_3/package.go index 88edf0d3..b191d7b2 100644 --- a/spdx/v2/v2_3/package.go +++ b/spdx/v2/v2_3/package.go @@ -86,8 +86,8 @@ type Package struct { PackageLicenseComments string `json:"licenseComments,omitempty"` // 7.17: Copyright Text: copyright notice(s) text, "NONE" or "NOASSERTION" - // Cardinality: mandatory, one - PackageCopyrightText string `json:"copyrightText"` + // Cardinality: optional, zero or one + PackageCopyrightText string `json:"copyrightText,omitempty"` // 7.18: Package Summary Description // Cardinality: optional, one From a857218af5e2c059af935cd303c30b5576e25f5c Mon Sep 17 00:00:00 2001 From: Brandon Lum Date: Wed, 17 May 2023 17:21:55 -0400 Subject: [PATCH 05/11] change packageVerification code for 2.1,2.2 to omitempty Signed-off-by: Brandon Lum --- spdx/v2/v2_1/package.go | 2 +- spdx/v2/v2_2/package.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spdx/v2/v2_1/package.go b/spdx/v2/v2_1/package.go index 85a4c4bc..9800c2c2 100644 --- a/spdx/v2/v2_1/package.go +++ b/spdx/v2/v2_1/package.go @@ -45,7 +45,7 @@ type Package struct { IsFilesAnalyzedTagPresent bool `json:"-"` // 3.9: Package Verification Code - PackageVerificationCode common.PackageVerificationCode `json:"packageVerificationCode"` + PackageVerificationCode common.PackageVerificationCode `json:"packageVerificationCode,omitempty"` // 3.10: Package Checksum: may have keys for SHA1, SHA256 and/or MD5 // Cardinality: optional, one or many diff --git a/spdx/v2/v2_2/package.go b/spdx/v2/v2_2/package.go index aaaeabd2..d9f6bad4 100644 --- a/spdx/v2/v2_2/package.go +++ b/spdx/v2/v2_2/package.go @@ -53,7 +53,7 @@ type Package struct { IsFilesAnalyzedTagPresent bool `json:"-"` // 7.9: Package Verification Code - PackageVerificationCode common.PackageVerificationCode `json:"packageVerificationCode"` + PackageVerificationCode common.PackageVerificationCode `json:"packageVerificationCode,omitempty"` // 7.10: Package Checksum: may have keys for SHA1, SHA256, SHA512 and/or MD5 // Cardinality: optional, one or many From 93754d38a02f7126b9d45083024abe4de5762b4c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 15:29:02 -0400 Subject: [PATCH 06/11] build(deps): Bump github.com/stretchr/testify from 1.8.2 to 1.8.3 (#217) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4c21f032..b1190713 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,6 @@ require ( github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 github.com/google/go-cmp v0.5.9 github.com/spdx/gordf v0.0.0-20201111095634-7098f93598fb - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.3 sigs.k8s.io/yaml v1.3.0 ) diff --git a/go.sum b/go.sum index c1426986..7d292893 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= From 00864c01536765d8491e81d284a615a4023902e0 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Tue, 23 May 2023 15:52:28 -0400 Subject: [PATCH 07/11] fix: properly output and read the filesAnalyzed field in JSON/YAML (#210) Signed-off-by: Keith Zantow --- .../json/SPDXJSONExample-v2.2.spdx.json | 1 - .../json/SPDXJSONExample-v2.3.spdx.json | 2 + .../sample-docs/tv/SPDXTagExample-v2.3.spdx | 4 +- .../yaml/SPDXYAMLExample-2.2.spdx.yaml | 1 - .../yaml/SPDXYAMLExample-2.3.spdx.yaml | 2 + spdx/v2/v2_2/example/example.go | 56 +++++++++--------- spdx/v2/v2_2/json/json_test.go | 13 +++++ spdx/v2/v2_2/package.go | 11 +++- spdx/v2/v2_2/yaml/yaml_test.go | 5 ++ spdx/v2/v2_3/example/example.go | 58 ++++++++++--------- spdx/v2/v2_3/json/json_test.go | 13 +++++ spdx/v2/v2_3/package.go | 12 +++- spdx/v2/v2_3/yaml/yaml_test.go | 5 ++ 13 files changed, 122 insertions(+), 61 deletions(-) diff --git a/examples/sample-docs/json/SPDXJSONExample-v2.2.spdx.json b/examples/sample-docs/json/SPDXJSONExample-v2.2.spdx.json index 89171a14..af8545fb 100644 --- a/examples/sample-docs/json/SPDXJSONExample-v2.2.spdx.json +++ b/examples/sample-docs/json/SPDXJSONExample-v2.2.spdx.json @@ -126,7 +126,6 @@ "referenceLocator" : "pkg:maven/org.apache.jena/apache-jena@3.12.0", "referenceType" : "purl" } ], - "filesAnalyzed" : false, "homepage" : "http://www.openjena.org/", "licenseConcluded" : "NOASSERTION", "licenseDeclared" : "NOASSERTION", diff --git a/examples/sample-docs/json/SPDXJSONExample-v2.3.spdx.json b/examples/sample-docs/json/SPDXJSONExample-v2.3.spdx.json index 07d1374d..def7ad8e 100644 --- a/examples/sample-docs/json/SPDXJSONExample-v2.3.spdx.json +++ b/examples/sample-docs/json/SPDXJSONExample-v2.3.spdx.json @@ -97,6 +97,7 @@ "name": "Apache Commons Lang", "SPDXID": "SPDXRef-fromDoap-1", "downloadLocation": "NOASSERTION", + "filesAnalyzed": false, "homepage": "http://commons.apache.org/proper/commons-lang/", "licenseConcluded": "NOASSERTION", "licenseDeclared": "NOASSERTION", @@ -125,6 +126,7 @@ "versionInfo": "8.8", "packageFileName": "saxonB-8.8.zip", "downloadLocation": "https://sourceforge.net/projects/saxon/files/Saxon-B/8.8.0.7/saxonb8-8-0-7j.zip/download", + "filesAnalyzed": false, "checksums": [ { "algorithm": "SHA1", diff --git a/examples/sample-docs/tv/SPDXTagExample-v2.3.spdx b/examples/sample-docs/tv/SPDXTagExample-v2.3.spdx index e7483b4a..8d4bb61a 100644 --- a/examples/sample-docs/tv/SPDXTagExample-v2.3.spdx +++ b/examples/sample-docs/tv/SPDXTagExample-v2.3.spdx @@ -92,7 +92,7 @@ PrimaryPackagePurpose: CONTAINER ReleaseDate: 2021-10-15T02:38:00Z BuiltDate: 2021-09-15T02:38:00Z ValidUntilDate: 2022-10-15T02:38:00Z -FilesAnalyzed: false +FilesAnalyzed: true PackageHomePage: https://www.centos.org/ PackageCopyrightText: NOASSERTION PackageDescription: The CentOS container used to run the application. @@ -148,7 +148,7 @@ PackageName: Jena SPDXID: SPDXRef-fromDoap-0 PackageVersion: 3.12.0 PackageDownloadLocation: https://search.maven.org/remotecontent?filepath=org/apache/jena/apache-jena/3.12.0/apache-jena-3.12.0.tar.gz -FilesAnalyzed: false +FilesAnalyzed: true PackageHomePage: http://www.openjena.org/ PackageLicenseConcluded: NOASSERTION PackageLicenseDeclared: NOASSERTION diff --git a/examples/sample-docs/yaml/SPDXYAMLExample-2.2.spdx.yaml b/examples/sample-docs/yaml/SPDXYAMLExample-2.2.spdx.yaml index d58cf229..de14ae5c 100644 --- a/examples/sample-docs/yaml/SPDXYAMLExample-2.2.spdx.yaml +++ b/examples/sample-docs/yaml/SPDXYAMLExample-2.2.spdx.yaml @@ -217,7 +217,6 @@ packages: - referenceCategory: "PACKAGE_MANAGER" referenceLocator: "pkg:maven/org.apache.jena/apache-jena@3.12.0" referenceType: "purl" - filesAnalyzed: false homepage: "http://www.openjena.org/" licenseConcluded: "NOASSERTION" licenseDeclared: "NOASSERTION" diff --git a/examples/sample-docs/yaml/SPDXYAMLExample-2.3.spdx.yaml b/examples/sample-docs/yaml/SPDXYAMLExample-2.3.spdx.yaml index 52d333f9..ccc6eb90 100644 --- a/examples/sample-docs/yaml/SPDXYAMLExample-2.3.spdx.yaml +++ b/examples/sample-docs/yaml/SPDXYAMLExample-2.3.spdx.yaml @@ -309,6 +309,7 @@ packages: - SPDXID: SPDXRef-fromDoap-1 copyrightText: NOASSERTION downloadLocation: NOASSERTION + filesAnalyzed: false homepage: http://commons.apache.org/proper/commons-lang/ licenseConcluded: NOASSERTION licenseDeclared: NOASSERTION @@ -331,6 +332,7 @@ packages: checksumValue: 85ed0817af83a24ad8da68c2b5094de69833983c copyrightText: Copyright Saxonica Ltd description: The Saxon package is a collection of tools for processing XML documents. + filesAnalyzed: false downloadLocation: https://sourceforge.net/projects/saxon/files/Saxon-B/8.8.0.7/saxonb8-8-0-7j.zip/download homepage: http://saxon.sourceforge.net/ licenseComments: Other versions available for a commercial license diff --git a/spdx/v2/v2_2/example/example.go b/spdx/v2/v2_2/example/example.go index 61b446a4..010ed73f 100644 --- a/spdx/v2/v2_2/example/example.go +++ b/spdx/v2/v2_2/example/example.go @@ -123,8 +123,9 @@ var example = spdx.Document{ Originator: "ExampleCodeInspect (contact@example.com)", OriginatorType: "Organization", }, - PackageDownloadLocation: "http://ftp.gnu.org/gnu/glibc/glibc-ports-2.15.tar.gz", - FilesAnalyzed: true, + PackageDownloadLocation: "http://ftp.gnu.org/gnu/glibc/glibc-ports-2.15.tar.gz", + FilesAnalyzed: true, + IsFilesAnalyzedTagPresent: true, PackageVerificationCode: common.PackageVerificationCode{ Value: "d6a770ba38583ed4bb4525bd96e50461655d2758", ExcludedFiles: []string{"./package.spdx"}, @@ -187,14 +188,15 @@ var example = spdx.Document{ }, }, { - PackageSPDXIdentifier: "fromDoap-1", - PackageCopyrightText: "NOASSERTION", - PackageDownloadLocation: "NOASSERTION", - FilesAnalyzed: false, - PackageHomePage: "http://commons.apache.org/proper/commons-lang/", - PackageLicenseConcluded: "NOASSERTION", - PackageLicenseDeclared: "NOASSERTION", - PackageName: "Apache Commons Lang", + PackageSPDXIdentifier: "fromDoap-1", + PackageCopyrightText: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + FilesAnalyzed: false, + IsFilesAnalyzedTagPresent: true, + PackageHomePage: "http://commons.apache.org/proper/commons-lang/", + PackageLicenseConcluded: "NOASSERTION", + PackageLicenseDeclared: "NOASSERTION", + PackageName: "Apache Commons Lang", }, { PackageName: "Jena", @@ -208,11 +210,12 @@ var example = spdx.Document{ Locator: "pkg:maven/org.apache.jena/apache-jena@3.12.0", }, }, - FilesAnalyzed: false, - PackageHomePage: "http://www.openjena.org/", - PackageLicenseConcluded: "NOASSERTION", - PackageLicenseDeclared: "NOASSERTION", - PackageVersion: "3.12.0", + FilesAnalyzed: true, + IsFilesAnalyzedTagPresent: false, + PackageHomePage: "http://www.openjena.org/", + PackageLicenseConcluded: "NOASSERTION", + PackageLicenseDeclared: "NOASSERTION", + PackageVersion: "3.12.0", }, { PackageSPDXIdentifier: "Saxon", @@ -222,17 +225,18 @@ var example = spdx.Document{ Value: "85ed0817af83a24ad8da68c2b5094de69833983c", }, }, - PackageCopyrightText: "Copyright Saxonica Ltd", - PackageDescription: "The Saxon package is a collection of tools for processing XML documents.", - PackageDownloadLocation: "https://sourceforge.net/projects/saxon/files/Saxon-B/8.8.0.7/saxonb8-8-0-7j.zip/download", - FilesAnalyzed: false, - PackageHomePage: "http://saxon.sourceforge.net/", - PackageLicenseComments: "Other versions available for a commercial license", - PackageLicenseConcluded: "MPL-1.0", - PackageLicenseDeclared: "MPL-1.0", - PackageName: "Saxon", - PackageFileName: "saxonB-8.8.zip", - PackageVersion: "8.8", + PackageCopyrightText: "Copyright Saxonica Ltd", + PackageDescription: "The Saxon package is a collection of tools for processing XML documents.", + PackageDownloadLocation: "https://sourceforge.net/projects/saxon/files/Saxon-B/8.8.0.7/saxonb8-8-0-7j.zip/download", + FilesAnalyzed: false, + IsFilesAnalyzedTagPresent: true, + PackageHomePage: "http://saxon.sourceforge.net/", + PackageLicenseComments: "Other versions available for a commercial license", + PackageLicenseConcluded: "MPL-1.0", + PackageLicenseDeclared: "MPL-1.0", + PackageName: "Saxon", + PackageFileName: "saxonB-8.8.zip", + PackageVersion: "8.8", }, }, Files: []*spdx.File{ diff --git a/spdx/v2/v2_2/json/json_test.go b/spdx/v2/v2_2/json/json_test.go index b2411a8e..d6246277 100644 --- a/spdx/v2/v2_2/json/json_test.go +++ b/spdx/v2/v2_2/json/json_test.go @@ -71,6 +71,11 @@ func TestLoad(t *testing.T) { func Test_Write(t *testing.T) { want := example.Copy() + // we always output FilesAnalyzed, even though we handle reading files where it is omitted + for _, p := range want.Packages { + p.IsFilesAnalyzedTagPresent = true + } + w := &bytes.Buffer{} if err := json.Write(&want, w); err != nil { @@ -153,16 +158,19 @@ func Test_ShorthandFields(t *testing.T) { { PackageName: "Container", PackageSPDXIdentifier: "Container", + FilesAnalyzed: true, }, { PackageName: "Package-1", PackageSPDXIdentifier: "Package-1", PackageVersion: "1.1.1", + FilesAnalyzed: true, }, { PackageName: "Package-2", PackageSPDXIdentifier: "Package-2", PackageVersion: "2.2.2", + FilesAnalyzed: true, }, }, Files: []*spdx.File{ @@ -272,6 +280,7 @@ func Test_JsonEnums(t *testing.T) { { PackageName: "Container", PackageSPDXIdentifier: "Container", + FilesAnalyzed: true, }, { PackageName: "Package-1", @@ -284,6 +293,7 @@ func Test_JsonEnums(t *testing.T) { Locator: "pkg:somepkg/ns/name1", }, }, + FilesAnalyzed: true, }, { PackageName: "Package-2", @@ -296,6 +306,7 @@ func Test_JsonEnums(t *testing.T) { Locator: "pkg:somepkg/ns/name2", }, }, + FilesAnalyzed: true, }, { PackageName: "Package-3", @@ -308,6 +319,7 @@ func Test_JsonEnums(t *testing.T) { Locator: "gitoid:blob:sha1:261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64", }, }, + FilesAnalyzed: true, }, { PackageName: "Package-4", @@ -320,6 +332,7 @@ func Test_JsonEnums(t *testing.T) { Locator: "gitoid:blob:sha1:261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64", }, }, + FilesAnalyzed: true, }, }, Relationships: []*spdx.Relationship{ diff --git a/spdx/v2/v2_2/package.go b/spdx/v2/v2_2/package.go index d9f6bad4..54de537c 100644 --- a/spdx/v2/v2_2/package.go +++ b/spdx/v2/v2_2/package.go @@ -48,7 +48,7 @@ type Package struct { // 7.8: FilesAnalyzed // Cardinality: optional, one; default value is "true" if omitted - FilesAnalyzed bool `json:"filesAnalyzed,omitempty"` + FilesAnalyzed bool `json:"filesAnalyzed"` // NOT PART OF SPEC: did FilesAnalyzed tag appear? IsFilesAnalyzedTagPresent bool `json:"-"` @@ -125,7 +125,8 @@ type Package struct { func (p *Package) UnmarshalJSON(b []byte) error { type pkg Package type extras struct { - HasFiles []common.DocElementID `json:"hasFiles"` + HasFiles []common.DocElementID `json:"hasFiles"` + FilesAnalyzed *bool `json:"filesAnalyzed"` } var p2 pkg @@ -141,6 +142,12 @@ func (p *Package) UnmarshalJSON(b []byte) error { *p = Package(p2) p.hasFiles = e.HasFiles + // FilesAnalyzed defaults to true if omitted + if e.FilesAnalyzed == nil { + p.FilesAnalyzed = true + } else { + p.IsFilesAnalyzedTagPresent = true + } return nil } diff --git a/spdx/v2/v2_2/yaml/yaml_test.go b/spdx/v2/v2_2/yaml/yaml_test.go index 62743b12..bacfc805 100644 --- a/spdx/v2/v2_2/yaml/yaml_test.go +++ b/spdx/v2/v2_2/yaml/yaml_test.go @@ -69,6 +69,11 @@ func Test_Read(t *testing.T) { func Test_Write(t *testing.T) { want := example.Copy() + // we always output FilesAnalyzed, even though we handle reading files where it is omitted + for _, p := range want.Packages { + p.IsFilesAnalyzedTagPresent = true + } + w := &bytes.Buffer{} if err := yaml.Write(want, w); err != nil { t.Errorf("Save() error = %v", err.Error()) diff --git a/spdx/v2/v2_3/example/example.go b/spdx/v2/v2_3/example/example.go index ee327534..e426d876 100644 --- a/spdx/v2/v2_3/example/example.go +++ b/spdx/v2/v2_3/example/example.go @@ -123,8 +123,9 @@ var example = spdx.Document{ Originator: "ExampleCodeInspect (contact@example.com)", OriginatorType: "Organization", }, - PackageDownloadLocation: "http://ftp.gnu.org/gnu/glibc/glibc-ports-2.15.tar.gz", - FilesAnalyzed: true, + PackageDownloadLocation: "http://ftp.gnu.org/gnu/glibc/glibc-ports-2.15.tar.gz", + FilesAnalyzed: true, + IsFilesAnalyzedTagPresent: true, PackageVerificationCode: &common.PackageVerificationCode{ Value: "d6a770ba38583ed4bb4525bd96e50461655d2758", ExcludedFiles: []string{"./package.spdx"}, @@ -187,14 +188,15 @@ var example = spdx.Document{ }, }, { - PackageSPDXIdentifier: "fromDoap-1", - PackageCopyrightText: "NOASSERTION", - PackageDownloadLocation: "NOASSERTION", - FilesAnalyzed: false, - PackageHomePage: "http://commons.apache.org/proper/commons-lang/", - PackageLicenseConcluded: "NOASSERTION", - PackageLicenseDeclared: "NOASSERTION", - PackageName: "Apache Commons Lang", + PackageSPDXIdentifier: "fromDoap-1", + PackageCopyrightText: "NOASSERTION", + PackageDownloadLocation: "NOASSERTION", + FilesAnalyzed: false, + IsFilesAnalyzedTagPresent: true, + PackageHomePage: "http://commons.apache.org/proper/commons-lang/", + PackageLicenseConcluded: "NOASSERTION", + PackageLicenseDeclared: "NOASSERTION", + PackageName: "Apache Commons Lang", }, { PackageName: "Jena", @@ -208,11 +210,12 @@ var example = spdx.Document{ Locator: "pkg:maven/org.apache.jena/apache-jena@3.12.0", }, }, - FilesAnalyzed: false, - PackageHomePage: "http://www.openjena.org/", - PackageLicenseConcluded: "NOASSERTION", - PackageLicenseDeclared: "NOASSERTION", - PackageVersion: "3.12.0", + FilesAnalyzed: true, + IsFilesAnalyzedTagPresent: false, + PackageHomePage: "http://www.openjena.org/", + PackageLicenseConcluded: "NOASSERTION", + PackageLicenseDeclared: "NOASSERTION", + PackageVersion: "3.12.0", }, { PackageSPDXIdentifier: "Saxon", @@ -222,17 +225,18 @@ var example = spdx.Document{ Value: "85ed0817af83a24ad8da68c2b5094de69833983c", }, }, - PackageCopyrightText: "Copyright Saxonica Ltd", - PackageDescription: "The Saxon package is a collection of tools for processing XML documents.", - PackageDownloadLocation: "https://sourceforge.net/projects/saxon/files/Saxon-B/8.8.0.7/saxonb8-8-0-7j.zip/download", - FilesAnalyzed: false, - PackageHomePage: "http://saxon.sourceforge.net/", - PackageLicenseComments: "Other versions available for a commercial license", - PackageLicenseConcluded: "MPL-1.0", - PackageLicenseDeclared: "MPL-1.0", - PackageName: "Saxon", - PackageFileName: "saxonB-8.8.zip", - PackageVersion: "8.8", + PackageCopyrightText: "Copyright Saxonica Ltd", + PackageDescription: "The Saxon package is a collection of tools for processing XML documents.", + PackageDownloadLocation: "https://sourceforge.net/projects/saxon/files/Saxon-B/8.8.0.7/saxonb8-8-0-7j.zip/download", + FilesAnalyzed: false, + IsFilesAnalyzedTagPresent: true, + PackageHomePage: "http://saxon.sourceforge.net/", + PackageLicenseComments: "Other versions available for a commercial license", + PackageLicenseConcluded: "MPL-1.0", + PackageLicenseDeclared: "MPL-1.0", + PackageName: "Saxon", + PackageFileName: "saxonB-8.8.zip", + PackageVersion: "8.8", }, { PrimaryPackagePurpose: "CONTAINER", @@ -240,7 +244,7 @@ var example = spdx.Document{ PackageCopyrightText: "NOASSERTION", PackageDescription: "The CentOS container used to run the application.", PackageDownloadLocation: "NOASSERTION", - FilesAnalyzed: false, + FilesAnalyzed: true, PackageHomePage: "https://www.centos.org/", PackageName: "centos", PackageFileName: "saxonB-8.8.zip", diff --git a/spdx/v2/v2_3/json/json_test.go b/spdx/v2/v2_3/json/json_test.go index 6bed4ea9..69801438 100644 --- a/spdx/v2/v2_3/json/json_test.go +++ b/spdx/v2/v2_3/json/json_test.go @@ -61,6 +61,11 @@ func Test_Read(t *testing.T) { func Test_Write(t *testing.T) { want := example.Copy() + // we always output FilesAnalyzed, even though we handle reading files where it is omitted + for _, p := range want.Packages { + p.IsFilesAnalyzedTagPresent = true + } + w := &bytes.Buffer{} if err := json.Write(&want, w); err != nil { @@ -143,16 +148,19 @@ func Test_ShorthandFields(t *testing.T) { { PackageName: "Container", PackageSPDXIdentifier: "Container", + FilesAnalyzed: true, }, { PackageName: "Package-1", PackageSPDXIdentifier: "Package-1", PackageVersion: "1.1.1", + FilesAnalyzed: true, }, { PackageName: "Package-2", PackageSPDXIdentifier: "Package-2", PackageVersion: "2.2.2", + FilesAnalyzed: true, }, }, Files: []*spdx.File{ @@ -262,6 +270,7 @@ func Test_JsonEnums(t *testing.T) { { PackageName: "Container", PackageSPDXIdentifier: "Container", + FilesAnalyzed: true, }, { PackageName: "Package-1", @@ -274,6 +283,7 @@ func Test_JsonEnums(t *testing.T) { Locator: "pkg:somepkg/ns/name1", }, }, + FilesAnalyzed: true, }, { PackageName: "Package-2", @@ -286,6 +296,7 @@ func Test_JsonEnums(t *testing.T) { Locator: "pkg:somepkg/ns/name2", }, }, + FilesAnalyzed: true, }, { PackageName: "Package-3", @@ -298,6 +309,7 @@ func Test_JsonEnums(t *testing.T) { Locator: "gitoid:blob:sha1:261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64", }, }, + FilesAnalyzed: true, }, { PackageName: "Package-4", @@ -310,6 +322,7 @@ func Test_JsonEnums(t *testing.T) { Locator: "gitoid:blob:sha1:261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64", }, }, + FilesAnalyzed: true, }, }, Relationships: []*spdx.Relationship{ diff --git a/spdx/v2/v2_3/package.go b/spdx/v2/v2_3/package.go index b191d7b2..0acadc27 100644 --- a/spdx/v2/v2_3/package.go +++ b/spdx/v2/v2_3/package.go @@ -48,7 +48,7 @@ type Package struct { // 7.8: FilesAnalyzed // Cardinality: optional, one; default value is "true" if omitted - FilesAnalyzed bool `json:"filesAnalyzed,omitempty"` + FilesAnalyzed bool `json:"filesAnalyzed"` // NOT PART OF SPEC: did FilesAnalyzed tag appear? IsFilesAnalyzedTagPresent bool `json:"-" yaml:"-"` @@ -143,7 +143,8 @@ type Package struct { func (p *Package) UnmarshalJSON(b []byte) error { type pkg Package type extras struct { - HasFiles []common.DocElementID `json:"hasFiles"` + HasFiles []common.DocElementID `json:"hasFiles"` + FilesAnalyzed *bool `json:"filesAnalyzed"` } var p2 pkg @@ -160,6 +161,13 @@ func (p *Package) UnmarshalJSON(b []byte) error { p.hasFiles = e.HasFiles + // FilesAnalyzed defaults to true if omitted + if e.FilesAnalyzed == nil { + p.FilesAnalyzed = true + } else { + p.IsFilesAnalyzedTagPresent = true + } + return nil } diff --git a/spdx/v2/v2_3/yaml/yaml_test.go b/spdx/v2/v2_3/yaml/yaml_test.go index adaa4846..766a7c33 100644 --- a/spdx/v2/v2_3/yaml/yaml_test.go +++ b/spdx/v2/v2_3/yaml/yaml_test.go @@ -56,6 +56,11 @@ func Test_Read(t *testing.T) { func Test_Write(t *testing.T) { want := example.Copy() + // we always output FilesAnalyzed, even though we handle reading files where it is omitted + for _, p := range want.Packages { + p.IsFilesAnalyzedTagPresent = true + } + w := &bytes.Buffer{} if err := yaml.Write(&want, w); err != nil { From 8b16c5548e7400c23f32660f5f5cd795be3c8197 Mon Sep 17 00:00:00 2001 From: Brandon Lum Date: Tue, 23 May 2023 17:46:19 -0400 Subject: [PATCH 08/11] fix duplicate shorthand rels for 2.2 Signed-off-by: Brandon Lum --- spdx/v2/v2_2/document.go | 30 +++++- spdx/v2/v2_2/example/example.go | 10 ++ spdx/v2/v2_2/json/json_test.go | 177 +++++++++++++++++++++++++------- 3 files changed, 177 insertions(+), 40 deletions(-) diff --git a/spdx/v2/v2_2/document.go b/spdx/v2/v2_2/document.go index 3bd605ff..d94f5b06 100644 --- a/spdx/v2/v2_2/document.go +++ b/spdx/v2/v2_2/document.go @@ -5,8 +5,9 @@ package v2_2 import ( "encoding/json" + "fmt" - "github.com/anchore/go-struct-converter" + converter "github.com/anchore/go-struct-converter" "github.com/spdx/tools-golang/spdx/v2/common" ) @@ -98,27 +99,46 @@ func (d *Document) UnmarshalJSON(b []byte) error { *d = Document(d2) + relationshipExists := map[string]bool{} + serializeRel := func(r *Relationship) string { + return fmt.Sprintf("%v-%v->%v", common.RenderDocElementID(r.RefA), r.Relationship, common.RenderDocElementID(r.RefB)) + } + + // index current list of relationships to ensure no duplication + for _, r := range d.Relationships { + relationshipExists[serializeRel(r)] = true + } + // build relationships for documentDescribes field for _, id := range e.DocumentDescribes { - d.Relationships = append(d.Relationships, &Relationship{ + r := &Relationship{ RefA: common.DocElementID{ ElementRefID: d.SPDXIdentifier, }, RefB: id, Relationship: common.TypeRelationshipDescribe, - }) + } + + if !relationshipExists[serializeRel(r)] { + d.Relationships = append(d.Relationships, r) + relationshipExists[serializeRel(r)] = true + } } // build relationships for package hasFiles field for _, p := range d.Packages { for _, f := range p.hasFiles { - d.Relationships = append(d.Relationships, &Relationship{ + r := &Relationship{ RefA: common.DocElementID{ ElementRefID: p.PackageSPDXIdentifier, }, RefB: f, Relationship: common.TypeRelationshipContains, - }) + } + if !relationshipExists[serializeRel(r)] { + d.Relationships = append(d.Relationships, r) + relationshipExists[serializeRel(r)] = true + } } p.hasFiles = nil diff --git a/spdx/v2/v2_2/example/example.go b/spdx/v2/v2_2/example/example.go index 010ed73f..501eb773 100644 --- a/spdx/v2/v2_2/example/example.go +++ b/spdx/v2/v2_2/example/example.go @@ -373,6 +373,16 @@ var example = spdx.Document{ RefB: common.MakeDocElementID("", "Package"), Relationship: "CONTAINS", }, + { + RefA: common.MakeDocElementID("", "Package"), + RefB: common.MakeDocElementID("", "CommonsLangSrc"), + Relationship: "CONTAINS", + }, + { + RefA: common.MakeDocElementID("", "Package"), + RefB: common.MakeDocElementID("", "DoapSource"), + Relationship: "CONTAINS", + }, { RefA: common.MakeDocElementID("", "DOCUMENT"), RefB: common.MakeDocElementID("spdx-tool-1.2", "ToolsElement"), diff --git a/spdx/v2/v2_2/json/json_test.go b/spdx/v2/v2_2/json/json_test.go index d6246277..3b87b4b6 100644 --- a/spdx/v2/v2_2/json/json_test.go +++ b/spdx/v2/v2_2/json/json_test.go @@ -4,6 +4,7 @@ package json import ( "bytes" + jsonenc "encoding/json" "fmt" "os" "strings" @@ -21,35 +22,6 @@ import ( func TestLoad(t *testing.T) { want := example.Copy() - - want.Relationships = append(want.Relationships, []*spdx.Relationship{ - { - RefA: common.DocElementID{ElementRefID: "DOCUMENT"}, - RefB: common.DocElementID{ElementRefID: "File"}, - Relationship: "DESCRIBES", - }, - { - RefA: common.DocElementID{ElementRefID: "DOCUMENT"}, - RefB: common.DocElementID{ElementRefID: "Package"}, - Relationship: "DESCRIBES", - }, - { - RefA: common.DocElementID{ElementRefID: "Package"}, - RefB: common.DocElementID{ElementRefID: "CommonsLangSrc"}, - Relationship: "CONTAINS", - }, - { - RefA: common.DocElementID{ElementRefID: "Package"}, - RefB: common.DocElementID{ElementRefID: "JenaLib"}, - Relationship: "CONTAINS", - }, - { - RefA: common.DocElementID{ElementRefID: "Package"}, - RefB: common.DocElementID{ElementRefID: "DoapSource"}, - Relationship: "CONTAINS", - }, - }...) - file, err := os.Open("../../../../examples/sample-docs/json/SPDXJSONExample-v2.2.spdx.json") if err != nil { panic(fmt.Errorf("error opening File: %s", err)) @@ -62,8 +34,8 @@ func TestLoad(t *testing.T) { return } - if !cmp.Equal(want, got, cmpopts.IgnoreUnexported(spdx.Package{})) { - t.Errorf("got incorrect struct after parsing YAML example: %s", cmp.Diff(want, got, cmpopts.IgnoreUnexported(spdx.Package{}))) + if diff := cmp.Diff(want, got, cmpopts.IgnoreUnexported(spdx.Package{}), cmpopts.SortSlices(relationshipLess)); len(diff) > 0 { + t.Errorf("got incorrect struct after parsing JSON example: %s", diff) return } } @@ -91,8 +63,8 @@ func Test_Write(t *testing.T) { return } - if !cmp.Equal(want, got, cmpopts.IgnoreUnexported(spdx.Package{})) { - t.Errorf("got incorrect struct after writing and re-parsing JSON example: %s", cmp.Diff(want, got, cmpopts.IgnoreUnexported(spdx.Package{}))) + if diff := cmp.Diff(want, got, cmpopts.IgnoreUnexported(spdx.Package{}), cmpopts.SortSlices(relationshipLess)); len(diff) > 0 { + t.Errorf("got incorrect struct after writing and re-parsing JSON example: %s", diff) return } } @@ -149,7 +121,7 @@ func Test_ShorthandFields(t *testing.T) { } } - require.Equal(t, spdx.Document{ + want := spdx.Document{ SPDXVersion: spdx.Version, DataLicense: spdx.DataLicense, SPDXIdentifier: "DOCUMENT", @@ -200,7 +172,136 @@ func Test_ShorthandFields(t *testing.T) { Relationship: common.TypeRelationshipContains, }, }, - }, doc) + } + + if diff := cmp.Diff(want, doc, cmpopts.IgnoreUnexported(spdx.Package{}), cmpopts.SortSlices(relationshipLess)); len(diff) > 0 { + t.Errorf("got incorrect struct after parsing JSON example: %s", cmp.Diff(want, doc, cmpopts.IgnoreUnexported(spdx.Package{}))) + return + } + +} + +func Test_ShorthandFieldsNoDuplicates(t *testing.T) { + contents := `{ + "spdxVersion": "SPDX-2.2", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "SPDX-Tools-v2.0", + "documentDescribes": [ + "SPDXRef-Container" + ], + "packages": [ + { + "name": "Container", + "SPDXID": "SPDXRef-Container" + }, + { + "name": "Package-1", + "SPDXID": "SPDXRef-Package-1", + "versionInfo": "1.1.1", + "hasFiles": [ + "SPDXRef-File-1", + "SPDXRef-File-2" + ] + }, + { + "name": "Package-2", + "SPDXID": "SPDXRef-Package-2", + "versionInfo": "2.2.2" + } + ], + "files": [ + { + "fileName": "./f1", + "SPDXID": "SPDXRef-File-1" + }, + { + "fileName": "./f2", + "SPDXID": "SPDXRef-File-2" + } + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-Package-1", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-File-1" + }, + { + "spdxElementId": "SPDXRef-Package-1", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-File-2" + } + ] + }` + + doc := spdx.Document{} + err := json.ReadInto(strings.NewReader(contents), &doc) + + require.NoError(t, err) + + id := func(s string) common.DocElementID { + return common.DocElementID{ + ElementRefID: common.ElementID(s), + } + } + + want := spdx.Document{ + SPDXVersion: spdx.Version, + DataLicense: spdx.DataLicense, + SPDXIdentifier: "DOCUMENT", + DocumentName: "SPDX-Tools-v2.0", + Packages: []*spdx.Package{ + { + PackageName: "Container", + PackageSPDXIdentifier: "Container", + FilesAnalyzed: true, + }, + { + PackageName: "Package-1", + PackageSPDXIdentifier: "Package-1", + PackageVersion: "1.1.1", + FilesAnalyzed: true, + }, + { + PackageName: "Package-2", + PackageSPDXIdentifier: "Package-2", + PackageVersion: "2.2.2", + FilesAnalyzed: true, + }, + }, + Files: []*spdx.File{ + { + FileName: "./f1", + FileSPDXIdentifier: "File-1", + }, + { + FileName: "./f2", + FileSPDXIdentifier: "File-2", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: id("DOCUMENT"), + RefB: id("Container"), + Relationship: common.TypeRelationshipDescribe, + }, + { + RefA: id("Package-1"), + RefB: id("File-1"), + Relationship: common.TypeRelationshipContains, + }, + { + RefA: id("Package-1"), + RefB: id("File-2"), + Relationship: common.TypeRelationshipContains, + }, + }, + } + + if diff := cmp.Diff(want, doc, cmpopts.IgnoreUnexported(spdx.Package{}), cmpopts.SortSlices(relationshipLess)); len(diff) > 0 { + t.Errorf("got incorrect struct after parsing JSON example: %s", diff) + return + } } func Test_JsonEnums(t *testing.T) { @@ -344,3 +445,9 @@ func Test_JsonEnums(t *testing.T) { }, }, doc) } + +func relationshipLess(a, b *spdx.Relationship) bool { + aStr, _ := jsonenc.Marshal(a) + bStr, _ := jsonenc.Marshal(b) + return string(aStr) < string(bStr) +} From 3f743e1a0e0a4b1d8e52f5ed440abe14ec5bcaa6 Mon Sep 17 00:00:00 2001 From: Brandon Lum Date: Tue, 23 May 2023 17:57:47 -0400 Subject: [PATCH 09/11] fix duplicate shorthand rels for 2.3 Signed-off-by: Brandon Lum --- spdx/v2/v2_3/document.go | 29 ++++++- spdx/v2/v2_3/json/json_test.go | 147 +++++++++++++++++++++++++++++++-- 2 files changed, 166 insertions(+), 10 deletions(-) diff --git a/spdx/v2/v2_3/document.go b/spdx/v2/v2_3/document.go index 9dbb8099..279e976c 100644 --- a/spdx/v2/v2_3/document.go +++ b/spdx/v2/v2_3/document.go @@ -5,6 +5,7 @@ package v2_3 import ( "encoding/json" + "fmt" converter "github.com/anchore/go-struct-converter" @@ -97,27 +98,47 @@ func (d *Document) UnmarshalJSON(b []byte) error { *d = Document(d2) + relationshipExists := map[string]bool{} + serializeRel := func(r *Relationship) string { + return fmt.Sprintf("%v-%v->%v", common.RenderDocElementID(r.RefA), r.Relationship, common.RenderDocElementID(r.RefB)) + } + + // index current list of relationships to ensure no duplication + for _, r := range d.Relationships { + relationshipExists[serializeRel(r)] = true + } + // build relationships for documentDescribes field for _, id := range e.DocumentDescribes { - d.Relationships = append(d.Relationships, &Relationship{ + r := &Relationship{ RefA: common.DocElementID{ ElementRefID: d.SPDXIdentifier, }, RefB: id, Relationship: common.TypeRelationshipDescribe, - }) + } + + if !relationshipExists[serializeRel(r)] { + d.Relationships = append(d.Relationships, r) + relationshipExists[serializeRel(r)] = true + } } + // build relationships for package hasFiles field // build relationships for package hasFiles field for _, p := range d.Packages { for _, f := range p.hasFiles { - d.Relationships = append(d.Relationships, &Relationship{ + r := &Relationship{ RefA: common.DocElementID{ ElementRefID: p.PackageSPDXIdentifier, }, RefB: f, Relationship: common.TypeRelationshipContains, - }) + } + if !relationshipExists[serializeRel(r)] { + d.Relationships = append(d.Relationships, r) + relationshipExists[serializeRel(r)] = true + } } p.hasFiles = nil diff --git a/spdx/v2/v2_3/json/json_test.go b/spdx/v2/v2_3/json/json_test.go index 69801438..432aad2a 100644 --- a/spdx/v2/v2_3/json/json_test.go +++ b/spdx/v2/v2_3/json/json_test.go @@ -4,6 +4,7 @@ package json import ( "bytes" + jsonenc "encoding/json" "flag" "fmt" "os" @@ -52,8 +53,8 @@ func Test_Read(t *testing.T) { return } - if !cmp.Equal(want, got, cmpopts.IgnoreUnexported(spdx.Package{})) { - t.Errorf("got incorrect struct after parsing YAML example: %s", cmp.Diff(want, got, cmpopts.IgnoreUnexported(spdx.Package{}))) + if diff := cmp.Diff(want, got, cmpopts.IgnoreUnexported(spdx.Package{}), cmpopts.SortSlices(relationshipLess)); len(diff) > 0 { + t.Errorf("got incorrect struct after parsing JSON example: %s", diff) return } } @@ -81,8 +82,8 @@ func Test_Write(t *testing.T) { return } - if !cmp.Equal(want, got, cmpopts.IgnoreUnexported(spdx.Package{})) { - t.Errorf("got incorrect struct after writing and re-parsing JSON example: %s", cmp.Diff(want, got, cmpopts.IgnoreUnexported(spdx.Package{}))) + if diff := cmp.Diff(want, got, cmpopts.IgnoreUnexported(spdx.Package{}), cmpopts.SortSlices(relationshipLess)); len(diff) > 0 { + t.Errorf("got incorrect struct after parsing JSON example: %s", diff) return } } @@ -139,7 +140,7 @@ func Test_ShorthandFields(t *testing.T) { } } - require.Equal(t, spdx.Document{ + want := spdx.Document{ SPDXVersion: spdx.Version, DataLicense: spdx.DataLicense, SPDXIdentifier: "DOCUMENT", @@ -190,7 +191,135 @@ func Test_ShorthandFields(t *testing.T) { Relationship: common.TypeRelationshipContains, }, }, - }, doc) + } + + if diff := cmp.Diff(want, doc, cmpopts.IgnoreUnexported(spdx.Package{}), cmpopts.SortSlices(relationshipLess)); len(diff) > 0 { + t.Errorf("got incorrect struct after parsing JSON example: %s", cmp.Diff(want, doc, cmpopts.IgnoreUnexported(spdx.Package{}))) + return + } +} + +func Test_ShorthandFieldsNoDuplicates(t *testing.T) { + contents := `{ + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "SPDX-Tools-v2.0", + "documentDescribes": [ + "SPDXRef-Container" + ], + "packages": [ + { + "name": "Container", + "SPDXID": "SPDXRef-Container" + }, + { + "name": "Package-1", + "SPDXID": "SPDXRef-Package-1", + "versionInfo": "1.1.1", + "hasFiles": [ + "SPDXRef-File-1", + "SPDXRef-File-2" + ] + }, + { + "name": "Package-2", + "SPDXID": "SPDXRef-Package-2", + "versionInfo": "2.2.2" + } + ], + "files": [ + { + "fileName": "./f1", + "SPDXID": "SPDXRef-File-1" + }, + { + "fileName": "./f2", + "SPDXID": "SPDXRef-File-2" + } + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-Package-1", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-File-1" + }, + { + "spdxElementId": "SPDXRef-Package-1", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-File-2" + } + ] + }` + + doc := spdx.Document{} + err := json.ReadInto(strings.NewReader(contents), &doc) + + require.NoError(t, err) + + id := func(s string) common.DocElementID { + return common.DocElementID{ + ElementRefID: common.ElementID(s), + } + } + + want := spdx.Document{ + SPDXVersion: spdx.Version, + DataLicense: spdx.DataLicense, + SPDXIdentifier: "DOCUMENT", + DocumentName: "SPDX-Tools-v2.0", + Packages: []*spdx.Package{ + { + PackageName: "Container", + PackageSPDXIdentifier: "Container", + FilesAnalyzed: true, + }, + { + PackageName: "Package-1", + PackageSPDXIdentifier: "Package-1", + PackageVersion: "1.1.1", + FilesAnalyzed: true, + }, + { + PackageName: "Package-2", + PackageSPDXIdentifier: "Package-2", + PackageVersion: "2.2.2", + FilesAnalyzed: true, + }, + }, + Files: []*spdx.File{ + { + FileName: "./f1", + FileSPDXIdentifier: "File-1", + }, + { + FileName: "./f2", + FileSPDXIdentifier: "File-2", + }, + }, + Relationships: []*spdx.Relationship{ + { + RefA: id("DOCUMENT"), + RefB: id("Container"), + Relationship: common.TypeRelationshipDescribe, + }, + { + RefA: id("Package-1"), + RefB: id("File-1"), + Relationship: common.TypeRelationshipContains, + }, + { + RefA: id("Package-1"), + RefB: id("File-2"), + Relationship: common.TypeRelationshipContains, + }, + }, + } + + if diff := cmp.Diff(want, doc, cmpopts.IgnoreUnexported(spdx.Package{}), cmpopts.SortSlices(relationshipLess)); len(diff) > 0 { + t.Errorf("got incorrect struct after parsing JSON example: %s", diff) + return + } } func Test_JsonEnums(t *testing.T) { @@ -334,3 +463,9 @@ func Test_JsonEnums(t *testing.T) { }, }, doc) } + +func relationshipLess(a, b *spdx.Relationship) bool { + aStr, _ := jsonenc.Marshal(a) + bStr, _ := jsonenc.Marshal(b) + return string(aStr) < string(bStr) +} From 6b2ab7af0e62c49db66b14c50a0e302cb12a416f Mon Sep 17 00:00:00 2001 From: Brandon Lum Date: Wed, 24 May 2023 09:57:45 -0400 Subject: [PATCH 10/11] update yaml tests to ignore order of relationships Signed-off-by: Brandon Lum --- spdx/v2/v2_2/yaml/yaml_test.go | 44 +++++++++------------------------- spdx/v2/v2_3/yaml/yaml_test.go | 15 ++++++++---- 2 files changed, 22 insertions(+), 37 deletions(-) diff --git a/spdx/v2/v2_2/yaml/yaml_test.go b/spdx/v2/v2_2/yaml/yaml_test.go index bacfc805..404a2613 100644 --- a/spdx/v2/v2_2/yaml/yaml_test.go +++ b/spdx/v2/v2_2/yaml/yaml_test.go @@ -4,6 +4,7 @@ package yaml import ( "bytes" + jsonenc "encoding/json" "fmt" "os" "testing" @@ -11,7 +12,6 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "github.com/spdx/tools-golang/spdx/v2/common" spdx "github.com/spdx/tools-golang/spdx/v2/v2_2" "github.com/spdx/tools-golang/spdx/v2/v2_2/example" "github.com/spdx/tools-golang/yaml" @@ -20,34 +20,6 @@ import ( func Test_Read(t *testing.T) { want := example.Copy() - want.Relationships = append(want.Relationships, []*spdx.Relationship{ - { - RefA: common.DocElementID{ElementRefID: "DOCUMENT"}, - RefB: common.DocElementID{ElementRefID: "File"}, - Relationship: "DESCRIBES", - }, - { - RefA: common.DocElementID{ElementRefID: "DOCUMENT"}, - RefB: common.DocElementID{ElementRefID: "Package"}, - Relationship: "DESCRIBES", - }, - { - RefA: common.DocElementID{ElementRefID: "Package"}, - RefB: common.DocElementID{ElementRefID: "CommonsLangSrc"}, - Relationship: "CONTAINS", - }, - { - RefA: common.DocElementID{ElementRefID: "Package"}, - RefB: common.DocElementID{ElementRefID: "JenaLib"}, - Relationship: "CONTAINS", - }, - { - RefA: common.DocElementID{ElementRefID: "Package"}, - RefB: common.DocElementID{ElementRefID: "DoapSource"}, - Relationship: "CONTAINS", - }, - }...) - file, err := os.Open("../../../../examples/sample-docs/yaml/SPDXYAMLExample-2.2.spdx.yaml") if err != nil { panic(fmt.Errorf("error opening File: %s", err)) @@ -60,8 +32,8 @@ func Test_Read(t *testing.T) { return } - if !cmp.Equal(want, got, cmpopts.IgnoreUnexported(spdx.Package{})) { - t.Errorf("got incorrect struct after parsing YAML example: %s", cmp.Diff(want, got, cmpopts.IgnoreUnexported(spdx.Package{}))) + if diff := cmp.Diff(want, got, cmpopts.IgnoreUnexported(spdx.Package{}), cmpopts.SortSlices(relationshipLess)); len(diff) > 0 { + t.Errorf("got incorrect struct after parsing YAML example: %s", diff) return } } @@ -88,8 +60,14 @@ func Test_Write(t *testing.T) { return } - if !cmp.Equal(want, got, cmpopts.IgnoreUnexported(spdx.Package{})) { - t.Errorf("got incorrect struct after writing and re-parsing YAML example: %s", cmp.Diff(want, got, cmpopts.IgnoreUnexported(spdx.Package{}))) + if diff := cmp.Diff(want, got, cmpopts.IgnoreUnexported(spdx.Package{}), cmpopts.SortSlices(relationshipLess)); len(diff) > 0 { + t.Errorf("got incorrect struct after writing and re-parsing YAML example: %s", diff) return } } + +func relationshipLess(a, b *spdx.Relationship) bool { + aStr, _ := jsonenc.Marshal(a) + bStr, _ := jsonenc.Marshal(b) + return string(aStr) < string(bStr) +} diff --git a/spdx/v2/v2_3/yaml/yaml_test.go b/spdx/v2/v2_3/yaml/yaml_test.go index 766a7c33..c0f787d0 100644 --- a/spdx/v2/v2_3/yaml/yaml_test.go +++ b/spdx/v2/v2_3/yaml/yaml_test.go @@ -4,6 +4,7 @@ package yaml import ( "bytes" + jsonenc "encoding/json" "flag" "fmt" "os" @@ -47,8 +48,8 @@ func Test_Read(t *testing.T) { return } - if !cmp.Equal(want, got, cmpopts.IgnoreUnexported(spdx.Package{})) { - t.Errorf("got incorrect struct after parsing YAML example: %s", cmp.Diff(want, got, cmpopts.IgnoreUnexported(spdx.Package{}))) + if diff := cmp.Diff(want, got, cmpopts.IgnoreUnexported(spdx.Package{}), cmpopts.SortSlices(relationshipLess)); len(diff) > 0 { + t.Errorf("got incorrect struct after parsing YAML example: %s", diff) return } } @@ -76,8 +77,14 @@ func Test_Write(t *testing.T) { return } - if !cmp.Equal(want, got, cmpopts.IgnoreUnexported(spdx.Package{})) { - t.Errorf("got incorrect struct after writing and re-parsing YAML example: %s", cmp.Diff(want, got, cmpopts.IgnoreUnexported(spdx.Package{}))) + if diff := cmp.Diff(want, got, cmpopts.IgnoreUnexported(spdx.Package{}), cmpopts.SortSlices(relationshipLess)); len(diff) > 0 { + t.Errorf("got incorrect struct after parsing YAML example: %s", diff) return } } + +func relationshipLess(a, b *spdx.Relationship) bool { + aStr, _ := jsonenc.Marshal(a) + bStr, _ := jsonenc.Marshal(b) + return string(aStr) < string(bStr) +} From fb0f8d8b593f51a2a60014d48a02c1b0d20d3b34 Mon Sep 17 00:00:00 2001 From: Brandon Lum Date: Fri, 26 May 2023 11:52:20 -0400 Subject: [PATCH 11/11] address pr comments Signed-off-by: Brandon Lum --- spdx/v2/v2_2/json/json_test.go | 2 +- spdx/v2/v2_3/json/json_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spdx/v2/v2_2/json/json_test.go b/spdx/v2/v2_2/json/json_test.go index 3b87b4b6..a4a02755 100644 --- a/spdx/v2/v2_2/json/json_test.go +++ b/spdx/v2/v2_2/json/json_test.go @@ -175,7 +175,7 @@ func Test_ShorthandFields(t *testing.T) { } if diff := cmp.Diff(want, doc, cmpopts.IgnoreUnexported(spdx.Package{}), cmpopts.SortSlices(relationshipLess)); len(diff) > 0 { - t.Errorf("got incorrect struct after parsing JSON example: %s", cmp.Diff(want, doc, cmpopts.IgnoreUnexported(spdx.Package{}))) + t.Errorf("got incorrect struct after parsing JSON example: %s", diff) return } diff --git a/spdx/v2/v2_3/json/json_test.go b/spdx/v2/v2_3/json/json_test.go index 432aad2a..ea648df2 100644 --- a/spdx/v2/v2_3/json/json_test.go +++ b/spdx/v2/v2_3/json/json_test.go @@ -194,7 +194,7 @@ func Test_ShorthandFields(t *testing.T) { } if diff := cmp.Diff(want, doc, cmpopts.IgnoreUnexported(spdx.Package{}), cmpopts.SortSlices(relationshipLess)); len(diff) > 0 { - t.Errorf("got incorrect struct after parsing JSON example: %s", cmp.Diff(want, doc, cmpopts.IgnoreUnexported(spdx.Package{}))) + t.Errorf("got incorrect struct after parsing JSON example: %s", diff) return } }