Merge pull request #98 from specter25/jsonparser1

[GSoC'21] Pr:8 . JSON saver bug fixes and Documentation enhancements

docs/jsonloader.md

@@ -0,0 +1,24 @@
+SPDX-License-Identifier: CC-BY-4.0
+
+## Working
+
+A UnmarshallJSON function on the spdx.Document2_2 struct is defined so that when the JSON is unmarshalled in it the function is called and we can implement the process in a custom way . Then  a new map[string]interface{} is deifined which temporarily holds the unmarshalled json . The map is then parsed into the spdx.Document2_2 using functions defined for it’s different sections .
+
+JSON  →  map[string]interface{}  → spdx.Document2_2
+
+## Some Key Points 
+
+- The packages have a property "hasFiles" defined in the schema which is an array of the SPDX Identifiers of the files of that pacakge . The parses first parses all the files into the Unpackaged files map of the document and then when it parses the packages , it removes the respective files from the unpackaged files map and places it inside the files map of that package .
+
+- The snippets have a property "snippetFromFile" which has the SPDX identiifer of the file to which the snippet is related . Thus the snippets require the files to be parsed before them . Then the snippets are parsed one by one and inserted into the respective files using this property .
+
+
+The json file loader in `package jsonloader` makes the following assumptions:
+
+
+### Order of appearance of the properties 
+* The parser does not make any pre-assumptions based on the order in which the properties appear . 
+
+
+### Annotations
+* The json spdx schema does not define the SPDX Identifier property for the annotation object . The parser assumes the spdx Identifier of the parent property of the currently being parsed annotation array to be the SPDX Identifer for all the annotation objects of that array.

docs/jsonsaver.md

@@ -0,0 +1,28 @@
+SPDX-License-Identifier: CC-BY-4.0
+
+## Working
+
+The spdx document is converted to map[string]interface{} and then the entire map is converted to json using a single json Marshall function call . The saver uses a tempoarary storage to store all the files (Paackaged and Unpackaged) together in a single data structure in order to comply with the json schema defined by spdx .
+
+spdx.Document2_2  →  map[string]interface{}  → JSON
+
+## Some Key Points
+
+- The packages have a property "hasFiles" defined in the schema which is an array of the SPDX Identifiers of the files of that pacakge . The saver iterates through the files of a package and inserted all the SPDX Identifiers of the files in the "hasFiles" array . In addition it adds the file to a temporary storage map to store all the files of the entire document at a single place .
+
+- The files require the packages to be saved before them in order to ensure that the packaged files are added to the temporary storage before the files are saved .
+
+- The snippets are saved after the files and a property "snippetFromFile" identifies the file of the snippets.
+
+The json file loader in `package jsonsaver` makes the following assumptions:
+
+
+### Order of appearance of the properties
+* The saver does not make any pre-assumptions based on the order in which the properties are saved . 
+
+
+### Annotations
+* The json spdx schema does not define the SPDX Identifier property for the annotation object . The saver inserts the annotation inside the element who spdx identifier mathches the annotation SPDX identifier .
+
+### Indentation
+* The jsonsaver uses the marshall indent function with "" as he prefix and "\t" as the indent character  , passed as funtion parameters .

jsonsaver/jsonsaver.go

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+
 package jsonsaver
 
 import (

jsonsaver/jsonsaver_test.go

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+
 package jsonsaver
 
 import (

jsonsaver/saver2v2/save_document.go

@@ -57,22 +57,26 @@ func RenderDocument2_2(doc *spdx.Document2_2, buf *bytes.Buffer) error {
 		}
 		jsondocument["documentDescribes"] = describesID
 	}
-
+	allfiles := make(map[spdx.ElementID]*spdx.File2_2)
 	// save packages from spdx to json
 	if doc.Packages != nil {
-		_, err = renderPackage2_2(doc, jsondocument)
+		_, err = renderPackage2_2(doc, jsondocument, allfiles)
 		if err != nil {
 			return err
 		}
 	}
 
+	for k, v := range doc.UnpackagedFiles {
+		allfiles[k] = v
+	}
+
 	// save files and snippets from spdx to json
-	if doc.UnpackagedFiles != nil {
-		_, err = renderFiles2_2(doc, jsondocument)
+	if allfiles != nil {
+		_, err = renderFiles2_2(doc, jsondocument, allfiles)
 		if err != nil {
 			return err
 		}
-		_, err = renderSnippets2_2(doc, jsondocument)
+		_, err = renderSnippets2_2(jsondocument, allfiles)
 		if err != nil {
 			return err
 		}

jsonsaver/saver2v2/save_files.go

@@ -8,18 +8,17 @@ import (
 	"github.com/spdx/tools-golang/spdx"
 )
 
-func renderFiles2_2(doc *spdx.Document2_2, jsondocument map[string]interface{}) ([]interface{}, error) {
+func renderFiles2_2(doc *spdx.Document2_2, jsondocument map[string]interface{}, allfiles map[spdx.ElementID]*spdx.File2_2) ([]interface{}, error) {
 
 	var keys []string
-	for ke := range doc.UnpackagedFiles {
+	for ke := range allfiles {
 		keys = append(keys, string(ke))
 	}
 	sort.Strings(keys)
 
 	var files []interface{}
-	// for k, v := range doc.UnpackagedFiles {
 	for _, k := range keys {
-		v := doc.UnpackagedFiles[spdx.ElementID(k)]
+		v := allfiles[spdx.ElementID(k)]
 		file := make(map[string]interface{})
 		file["SPDXID"] = spdx.RenderElementID(spdx.ElementID(k))
 		ann, _ := renderAnnotations2_2(doc.Annotations, spdx.MakeDocElementID("", string(v.FileSPDXIdentifier)))

jsonsaver/saver2v2/save_files_test.go

@@ -132,7 +132,7 @@ func Test_renderFiles2_2(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := renderFiles2_2(tt.args.doc, tt.args.jsondocument)
+			got, err := renderFiles2_2(tt.args.doc, tt.args.jsondocument, tt.args.doc.UnpackagedFiles)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("renderFiles2_2() error = %v, wantErr %v", err, tt.wantErr)
 			}

jsonsaver/saver2v2/save_package.go

@@ -9,7 +9,7 @@ import (
 	"github.com/spdx/tools-golang/spdx"
 )
 
-func renderPackage2_2(doc *spdx.Document2_2, jsondocument map[string]interface{}) ([]interface{}, error) {
+func renderPackage2_2(doc *spdx.Document2_2, jsondocument map[string]interface{}, allfiles map[spdx.ElementID]*spdx.File2_2) ([]interface{}, error) {
 
 	var packages []interface{}
 
@@ -82,7 +82,7 @@ func renderPackage2_2(doc *spdx.Document2_2, jsondocument map[string]interface{}
 		if v.Files != nil {
 			var fileIds []string
 			for k, v := range v.Files {
-				doc.UnpackagedFiles[k] = v
+				allfiles[k] = v
 				fileIds = append(fileIds, spdx.RenderElementID(k))
 			}
 			pkg["hasFiles"] = fileIds

jsonsaver/saver2v2/save_package_test.go

@@ -206,7 +206,7 @@ func Test_renderPackage2_2(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := renderPackage2_2(tt.args.doc, tt.args.jsondocument)
+			got, err := renderPackage2_2(tt.args.doc, tt.args.jsondocument, make(map[spdx.ElementID]*spdx.File2_2))
 			if (err != nil) != tt.wantErr {
 				t.Errorf("renderPackage2_2() error = %v, wantErr %v", err, tt.wantErr)
 				return

jsonsaver/saver2v2/save_snippets.go

@@ -3,15 +3,24 @@
 package saver2v2
 
 import (
+	"sort"
+
 	"github.com/spdx/tools-golang/spdx"
 )
 
-func renderSnippets2_2(doc *spdx.Document2_2, jsondocument map[string]interface{}) ([]interface{}, error) {
+func renderSnippets2_2(jsondocument map[string]interface{}, allfiles map[spdx.ElementID]*spdx.File2_2) ([]interface{}, error) {
 
 	var snippets []interface{}
-	for _, value := range doc.UnpackagedFiles {
+	for _, value := range allfiles {
 		snippet := make(map[string]interface{})
-		for _, v := range value.Snippets {
+
+		var keys []string
+		for ke := range value.Snippets {
+			keys = append(keys, string(ke))
+		}
+		sort.Strings(keys)
+		for _, k := range keys {
+			v := value.Snippets[spdx.ElementID(k)]
 			snippet["SPDXID"] = spdx.RenderElementID(v.SnippetSPDXIdentifier)
 			if v.SnippetComment != "" {
 				snippet["comment"] = v.SnippetComment

jsonsaver/saver2v2/save_snippets_test.go

@@ -102,7 +102,7 @@ func Test_renderSnippets2_2(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := renderSnippets2_2(tt.args.doc, tt.args.jsondocument)
+			got, err := renderSnippets2_2(tt.args.jsondocument, tt.args.doc.UnpackagedFiles)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("renderSnippets2_2() error = %v, wantErr %v", err, tt.wantErr)
 				return