Merge pull request #1680 from kubescape/fix/repo-scanning

Fix scanning repo

core/cautils/remotegitutils.go

@@ -1,11 +1,11 @@
 package cautils
 
 import (
+	"crypto/sha256"
 	"errors"
 	"fmt"
 	nethttp "net/http"
 	"os"
-	"path/filepath"
 
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
@@ -17,6 +17,40 @@ import (
 	"github.com/kubescape/go-logger/helpers"
 )
 
+var tmpDirPaths map[string]string
+
+func hashRepoURL(repoURL string) string {
+	h := sha256.New()
+	h.Write([]byte(repoURL))
+	return string(h.Sum(nil))
+}
+
+func getDirPath(repoURL string) string {
+	if tmpDirPaths == nil {
+		return ""
+	}
+	return tmpDirPaths[hashRepoURL(repoURL)]
+}
+
+// Create a temporary directory this function is called once
+func createTempDir(repoURL string) (string, error) {
+	tmpDirPath := getDirPath(repoURL)
+	if tmpDirPath != "" {
+		return tmpDirPath, nil
+	}
+	// create temp directory
+	tmpDir, err := os.MkdirTemp("", "")
+	if err != nil {
+		return "", fmt.Errorf("failed to create temporary directory: %w", err)
+	}
+	if tmpDirPaths == nil {
+		tmpDirPaths = make(map[string]string)
+	}
+	tmpDirPaths[hashRepoURL(repoURL)] = tmpDir
+
+	return tmpDir, nil
+}
+
 // To Check if the given repository is Public(No Authentication needed), send a HTTP GET request to the URL
 // If response code is 200, the repository is Public.
 func isGitRepoPublic(u string) bool {
@@ -58,16 +92,21 @@ func getProviderError(gitURL giturl.IGitAPI) error {
 
 // cloneRepo clones a repository to a local temporary directory and returns the directory
 func cloneRepo(gitURL giturl.IGitAPI) (string, error) {
+	cloneURL := gitURL.GetHttpCloneURL()
+
+	// Check if directory exists
+	if p := getDirPath(cloneURL); p != "" {
+		// directory exists, meaning this repo was cloned
+		return p, nil
+	}
+	// Get the URL to clone
 
 	// Create temp directory
-	tmpDir, err := os.MkdirTemp("", "")
+	tmpDir, err := createTempDir(cloneURL)
 	if err != nil {
-		return "", fmt.Errorf("failed to create temporary directory: %w", err)
+		return "", err
 	}
 
-	// Get the URL to clone
-	cloneURL := gitURL.GetHttpCloneURL()
-
 	isGitTokenPresent := isGitTokenPresent(gitURL)
 
 	// Declare the authentication variable required for cloneOptions
@@ -104,6 +143,8 @@ func cloneRepo(gitURL giturl.IGitAPI) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("failed to clone %s. %w", gitURL.GetRepoName(), err)
 	}
+	// tmpDir = filepath.Join(tmpDir, gitURL.GetRepoName())
+	tmpDirPaths[hashRepoURL(cloneURL)] = tmpDir
 
 	return tmpDir, nil
 }
@@ -125,9 +166,19 @@ func CloneGitRepo(path *string) (string, error) {
 		logger.L().StopError("failed to clone git repo", helpers.String("url", gitURL.GetURL().String()), helpers.Error(err))
 		return "", fmt.Errorf("failed to clone git repo '%s',  %w", gitURL.GetURL().String(), err)
 	}
+	*path = clonedDir
 
-	*path = filepath.Join(clonedDir, gitURL.GetPath())
-	logger.L().StopSuccess("Done accessing local objects")
+	logger.L().StopSuccess("Done accessing remote repo")
 
 	return clonedDir, nil
 }
+
+func GetClonedPath(path string) string {
+
+	gitURL, err := giturl.NewGitAPI(path)
+	if err != nil {
+		return ""
+	}
+
+	return getDirPath(gitURL.GetHttpCloneURL())
+}

core/cautils/remotegitutils_test.go

@@ -93,3 +93,63 @@ func TestCloneRepo(t *testing.T) {
 		})
 	}
 }
+func TestGetClonedPath(t *testing.T) {
+	testCases := []struct {
+		name     string
+		path     string
+		expected string
+	}{
+		{
+			name:     "Valid Git URL",
+			path:     "https://github.com/kubescape/kubescape.git",
+			expected: "/path/to/cloned/repo", // replace with the expected path
+		},
+		{
+			name:     "Invalid Git URL",
+			path:     "invalid",
+			expected: "",
+		},
+	}
+	tmpDirPaths = make(map[string]string)
+	tmpDirPaths[hashRepoURL("https://github.com/kubescape/kubescape.git")] = "/path/to/cloned/repo" // replace with the actual path
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := GetClonedPath(tc.path)
+			if result != tc.expected {
+				t.Errorf("Expected %q, got %q", tc.expected, result)
+			}
+		})
+	}
+}
+func TestGetDirPath(t *testing.T) {
+	testCases := []struct {
+		name     string
+		repoURL  string
+		expected string
+	}{
+		{
+			name:     "Existing Repo URL",
+			repoURL:  "https://github.com/user/repo.git",
+			expected: "/path/to/cloned/repo", // replace with the expected path
+		},
+		{
+			name:     "Non-Existing Repo URL",
+			repoURL:  "https://github.com/user/nonexistentrepo.git",
+			expected: "",
+		},
+	}
+
+	// Initialize tmpDirPaths
+	tmpDirPaths = make(map[string]string)
+	tmpDirPaths[hashRepoURL("https://github.com/user/repo.git")] = "/path/to/cloned/repo" // replace with the actual path
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := getDirPath(tc.repoURL)
+			if result != tc.expected {
+				t.Errorf("Expected %q, got %q", tc.expected, result)
+			}
+		})
+	}
+}

core/cautils/scaninfo.go

@@ -25,10 +25,11 @@ import (
 type ScanningContext string
 
 const (
-	ContextCluster  ScanningContext = "cluster"
-	ContextFile     ScanningContext = "single-file"
-	ContextDir      ScanningContext = "local-dir"
-	ContextGitLocal ScanningContext = "git-local"
+	ContextCluster   ScanningContext = "cluster"
+	ContextFile      ScanningContext = "single-file"
+	ContextDir       ScanningContext = "local-dir"
+	ContextGitLocal  ScanningContext = "git-local"
+	ContextGitRemote ScanningContext = "git-remote"
 )
 
 const ( // deprecated
@@ -281,6 +282,9 @@ func scanInfoToScanMetadata(ctx context.Context, scanInfo *ScanInfo) *reporthand
 	case ContextGitLocal:
 		// local-git
 		metadata.ScanMetadata.ScanningTarget = reporthandlingv2.GitLocal
+	case ContextGitRemote:
+		// remote
+		metadata.ScanMetadata.ScanningTarget = reporthandlingv2.Repo
 	case ContextDir:
 		// directory
 		metadata.ScanMetadata.ScanningTarget = reporthandlingv2.Directory
@@ -308,6 +312,7 @@ func (scanInfo *ScanInfo) GetScanningContext() ScanningContext {
 }
 
 // getScanningContext get scanning context from the input param
+// this function should be called only once. Call GetScanningContext() to get the scanning context
 func (scanInfo *ScanInfo) getScanningContext(input string) ScanningContext {
 	//  cluster
 	if input == "" {
@@ -321,7 +326,7 @@ func (scanInfo *ScanInfo) getScanningContext(input string) ScanningContext {
 				scanInfo.cleanups = append(scanInfo.cleanups, func() {
 					_ = os.RemoveAll(repo)
 				})
-				return ContextGitLocal
+				return ContextGitRemote
 			}
 		}
 	}
@@ -389,6 +394,13 @@ func (scanInfo *ScanInfo) setContextMetadata(ctx context.Context, contextMetadat
 			logger.L().Ctx(ctx).Warning("in setContextMetadata", helpers.Interface("case", ContextGitLocal), helpers.Error(err))
 		}
 		contextMetadata.RepoContextMetadata = repoContext
+	case ContextGitRemote:
+		// remote
+		repoContext, err := metadataGitLocal(GetClonedPath(input))
+		if err != nil {
+			logger.L().Ctx(ctx).Warning("in setContextMetadata", helpers.Interface("case", ContextGitRemote), helpers.Error(err))
+		}
+		contextMetadata.RepoContextMetadata = repoContext
 	}
 }
 

core/cautils/scaninfo_test.go

@@ -71,7 +71,7 @@ func TestGetScanningContext(t *testing.T) {
 		{
 			name:  "git URL input",
 			input: "https://github.com/kubescape/http-request",
-			want:  ContextGitLocal,
+			want:  ContextGitRemote,
 		},
 		{
 			name:  "local git input",

core/pkg/resourcehandler/filesloader.go

@@ -3,7 +3,6 @@ package resourcehandler
 import (
 	"context"
 	"fmt"
-	"os"
 	"path/filepath"
 	"strings"
 
@@ -45,7 +44,7 @@ func (fileHandler *FileResourceHandler) GetResources(ctx context.Context, sessio
 		var err error
 
 		if scanInfo.ChartPath != "" && scanInfo.FilePath != "" {
-			workloadIDToSource, workloads, workloadIDToMappingNodes, err = getWorkloadFromHelmChart(ctx, scanInfo.ChartPath, scanInfo.FilePath)
+			workloadIDToSource, workloads, workloadIDToMappingNodes, err = getWorkloadFromHelmChart(ctx, scanInfo.InputPatterns[path], scanInfo.ChartPath, scanInfo.FilePath)
 			if err != nil {
 				// We should probably ignore the error so we can continue scanning other charts
 			}
@@ -107,26 +106,22 @@ func (fileHandler *FileResourceHandler) GetResources(ctx context.Context, sessio
 func (fileHandler *FileResourceHandler) GetCloudProvider() string {
 	return ""
 }
-func getWorkloadFromHelmChart(ctx context.Context, helmPath, workloadPath string) (map[string]reporthandling.Source, []workloadinterface.IMetadata, map[string]cautils.MappingNodes, error) {
-	clonedRepo, err := cautils.CloneGitRepo(&helmPath)
-	if err != nil {
-		return nil, nil, nil, err
-	}
+func getWorkloadFromHelmChart(ctx context.Context, path, helmPath, workloadPath string) (map[string]reporthandling.Source, []workloadinterface.IMetadata, map[string]cautils.MappingNodes, error) {
+	clonedRepo := cautils.GetClonedPath(path)
+
 	if clonedRepo != "" {
-		defer func(path string) {
-			_ = os.RemoveAll(path)
-		}(clonedRepo)
+		// if the repo was cloned, add the workload path to the cloned repo
+		workloadPath = filepath.Join(clonedRepo, workloadPath)
+	} else {
+		// if the repo was not cloned
+		clonedRepo = path
 	}
 
 	// Get repo root
-	repoRoot, gitRepo := extractGitRepo(helmPath)
+	repoRoot, gitRepo := extractGitRepo(clonedRepo)
 
 	helmSourceToWorkloads, helmSourceToChart, helmSourceToNodes := cautils.LoadResourcesFromHelmCharts(ctx, helmPath)
 
-	if clonedRepo != "" {
-		workloadPath = clonedRepo + workloadPath
-	}
-
 	wlSource, ok := helmSourceToWorkloads[workloadPath]
 	if !ok {
 		return nil, nil, nil, fmt.Errorf("workload %s not found in chart %s", workloadPath, helmPath)
@@ -195,14 +190,10 @@ func getResourcesFromPath(ctx context.Context, path string) (map[string]reportha
 	workloadIDToNodes := make(map[string]cautils.MappingNodes)
 	var workloads []workloadinterface.IMetadata
 
-	clonedRepo, err := cautils.CloneGitRepo(&path)
-	if err != nil {
-		return nil, nil, nil, err
-	}
+	clonedRepo := cautils.GetClonedPath(path)
 	if clonedRepo != "" {
-		defer func(path string) {
-			_ = os.RemoveAll(path)
-		}(clonedRepo)
+		// if the repo was cloned, add the workload path to the cloned repo
+		path = clonedRepo
 	}
 
 	// Get repo root

core/pkg/resultshandling/printer/v2/prettyprinter/reposcan.go

@@ -3,6 +3,7 @@ package prettyprinter
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 
 	"github.com/kubescape/kubescape/v3/core/cautils"
 	"github.com/kubescape/kubescape/v3/core/pkg/resultshandling/printer/v2/prettyprinter/tableprinter/configurationprinter"
@@ -77,9 +78,8 @@ func (rp *RepoPrinter) getWorkloadScanCommand(ns, kind, name string, source repo
 	}
 
 	if source.FileType == reporthandling.SourceTypeHelmChart {
-		return fmt.Sprintf("%s --chart-path=%s --file-path=%s", cmd, source.HelmPath, fmt.Sprintf("%s/%s", source.Path, source.RelativePath))
-
+		return fmt.Sprintf("%s --chart-path=%s --file-path=%s", cmd, source.HelmPath, filepath.Join(source.Path, source.RelativePath))
 	} else {
-		return fmt.Sprintf("%s --file-path=%s", cmd, fmt.Sprintf("%s/%s", source.Path, source.RelativePath))
+		return fmt.Sprintf("%s --file-path=%s", cmd, filepath.Join(source.Path, source.RelativePath))
 	}
 }

core/pkg/resultshandling/printer/v2/prettyprinter/reposcan_test.go

@@ -1,6 +1,7 @@
 package prettyprinter
 
 import (
+	"path/filepath"
 	"testing"
 
 	"github.com/kubescape/opa-utils/reporthandling"
@@ -50,7 +51,18 @@ func TestRepoScan_getWorkloadScanCommand(t *testing.T) {
 				Path:         "path",
 				RelativePath: "relativePath",
 			},
-			want: "$ kubescape scan workload kind/name --namespace ns --file-path=path/relativePath",
+			want: "$ kubescape scan workload kind/name --namespace ns --file-path=" + filepath.Join("path", "relativePath"),
+		},
+		{
+			testName: "relative file path",
+			ns:       "ns",
+			kind:     "kind",
+			name:     "name",
+			source: reporthandling.Source{
+				Path:         "",
+				RelativePath: "relativePath",
+			},
+			want: "$ kubescape scan workload kind/name --namespace ns --file-path=relativePath",
 		},
 		{
 			testName: "helm path",
@@ -63,7 +75,7 @@ func TestRepoScan_getWorkloadScanCommand(t *testing.T) {
 				HelmPath:     "helmPath",
 				FileType:     "Helm Chart",
 			},
-			want: "$ kubescape scan workload kind/name --namespace ns --chart-path=helmPath --file-path=path/relativePath",
+			want: "$ kubescape scan workload kind/name --namespace ns --chart-path=helmPath --file-path=" + filepath.Join("path", "relativePath"),
 		},
 		{
 			testName: "file path - no namespace",
@@ -73,7 +85,7 @@ func TestRepoScan_getWorkloadScanCommand(t *testing.T) {
 				Path:         "path",
 				RelativePath: "relativePath",
 			},
-			want: "$ kubescape scan workload kind/name --file-path=path/relativePath",
+			want: "$ kubescape scan workload kind/name --file-path=" + filepath.Join("path", "relativePath"),
 		},
 		{
 			testName: "helm path - no namespace",
@@ -85,7 +97,7 @@ func TestRepoScan_getWorkloadScanCommand(t *testing.T) {
 				HelmPath:     "helmPath",
 				FileType:     "Helm Chart",
 			},
-			want: "$ kubescape scan workload kind/name --chart-path=helmPath --file-path=path/relativePath",
+			want: "$ kubescape scan workload kind/name --chart-path=helmPath --file-path=" + filepath.Join("path", "relativePath"),
 		},
 	}
 

core/pkg/resultshandling/results.go

@@ -146,7 +146,7 @@ func ValidatePrinter(scanType cautils.ScanTypes, scanContext cautils.ScanningCon
 	if printFormat == printer.SARIFFormat {
 		// supported types for SARIF
 		switch scanContext {
-		case cautils.ContextDir, cautils.ContextFile, cautils.ContextGitLocal:
+		case cautils.ContextDir, cautils.ContextFile, cautils.ContextGitLocal, cautils.ContextGitRemote:
 			return nil
 		default:
 			return fmt.Errorf("format \"%s\" is only supported when scanning local files", printFormat)