Version 0.4.39 - Multi-Instance Support (checkmarx-ltd#103)

* Allowing to specify source directory for a SCA scan (checkmarx-ltd#100)

* Updated Common Client version.

* Allowing to specify SCA sources as a directory (in addition to a zip file).

Added to support "manifest and fingerprints" SCA flow.

Merged AstClient.scanRemoteRepo and AstClient.scanLocalSource methods, because they have the same implementation.

* Using Optional to reduce code duplication.

* Updated version.

* Feature/shard support (checkmarx-ltd#102)

Adding Shard Support (Multi-Instance)

Co-authored-by: Alexey K <alex-ko-dev@users.noreply.github.com>
Co-authored-by: Jeff Armstrong <jarmstrong@nezasoft.com>

pom.xml

@@ -10,7 +10,7 @@
 	</parent>
 	<groupId>com.github.checkmarx-ts</groupId>
 	<artifactId>cx-spring-boot-sdk</artifactId>
-	<version>0.4.38</version>
+	<version>0.4.39</version>
 	<name>cx-spring-boot-sdk</name>
 	<description>Checkmarx Java Spring Boot SDK</description>
 

src/main/java/com/checkmarx/sdk/ShardManager/ShardConfig.java

@@ -0,0 +1,103 @@
+package com.checkmarx.sdk.ShardManager;
+
+public class ShardConfig {
+    private String name = "Unknown";
+    private String url = "";
+    private int projectLimit = 1;
+    private int teamLimit = 1;
+    private int isDisabled = 0;
+    private int isCredentialOverride = 0;
+    private String username = "";
+    private String password = "";
+    private int isTeamOverride = 0;
+    private String team = "";
+    private int forceSettingReload = 0;
+
+    public String getName() {
+        return this.name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getUrl() {
+        return this.url;
+    }
+
+    public void setUrl(String url) {
+        this.url = url;
+    }
+
+    public int getProjectLimit() {
+        return this.projectLimit;
+    }
+
+    public void setProjectLimit(int projectLimit) {
+        this.projectLimit = projectLimit;
+    }
+
+    public int getTeamLimit() {
+        return this.teamLimit;
+    }
+
+    public void setTeamLimit(int teamLimit) {
+        this.teamLimit = teamLimit;
+    }
+
+    public int getIsDisabled() {
+        return this.isDisabled;
+    }
+
+    public void setIsDisabled(int isDisabled) {
+        this.isDisabled = isDisabled;
+    }
+
+    public int getIsCredentialOverride() {
+        return this.isCredentialOverride;
+    }
+
+    public void setIsCredentialOverride(int isCredentialOverride) {
+        this.isCredentialOverride = isCredentialOverride;
+    }
+
+    public String getUsername() {
+        return this.username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public String getPassword() {
+        return this.password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public int getIsTeamOverride() {
+        return this.isTeamOverride;
+    }
+
+    public void setIsTeamOverride(int isTeamOverride) {
+        this.isTeamOverride = isTeamOverride;
+    }
+
+    public String getTeam() {
+        return this.team;
+    }
+
+    public void setTeam(String team) {
+        this.team = team;
+    }
+
+    public int getForceSettingReload() {
+        return this.forceSettingReload;
+    }
+
+    public void setForceSettingReload(int forceSettingReload) {
+        this.forceSettingReload = forceSettingReload;
+    }
+}

src/main/java/com/checkmarx/sdk/ShardManager/ShardManagerHTTPInterceptor.java

@@ -0,0 +1,160 @@
+package com.checkmarx.sdk.ShardManager;
+import com.checkmarx.sdk.config.ShardProperties;
+import com.checkmarx.sdk.service.ScanSettingsClient;
+import groovy.util.GroovyScriptEngine;
+import groovy.util.ResourceException;
+import groovy.util.ScriptException;
+import groovy.lang.Binding;
+import groovy.lang.GroovyRuntimeException;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpRequest;
+import org.springframework.http.client.ClientHttpRequestExecution;
+import org.springframework.http.client.ClientHttpRequestInterceptor;
+import org.springframework.http.client.ClientHttpResponse;
+import org.springframework.http.client.support.HttpRequestWrapper;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ShardManagerHTTPInterceptor implements ClientHttpRequestInterceptor {
+    private static final Logger log = org.slf4j.LoggerFactory.getLogger(ShardManagerHTTPInterceptor.class);
+    private ShardProperties shardProperties;
+    private ShardSessionTracker sessionTracker;
+    //
+    /// predefined URI path strings
+    //
+    private static String authReq = "/cxrestapi/auth/identity/connect/token";
+
+    public ShardManagerHTTPInterceptor(ShardProperties shardProperties,
+                                       ShardSessionTracker sessionTracker) {
+        this.shardProperties = shardProperties;
+        this.sessionTracker = sessionTracker;
+    }
+
+    @Override
+    public ClientHttpResponse intercept(
+            HttpRequest request,
+            byte[] body,
+            ClientHttpRequestExecution execution) throws IOException {
+        ShardSession shard = sessionTracker.getShardSession();
+        if (!shard.getShardFound() && request.getURI().getPath().equals(authReq)) {
+            runShardManager(sessionTracker.getScanRequestID(), shard);
+            body = overrideCredentials(request, body, shard);
+            ClientHttpResponse clientResp = execution.execute(new ShardRequestWrapper(request, shard.getUrl()), body);
+            return clientResp;
+        }
+        else {
+            return execution.execute(new ShardRequestWrapper(request, shard.getUrl()), body);
+        }
+    }
+
+    //
+    /// This will only change the payload if the request path indicates an auth request
+    /// ADD isIncrementalOverride is turned on for the current shard.
+    //
+    public byte[] overrideCredentials(HttpRequest request, byte []body, ShardSession session) {
+        if (request.getURI().getPath().equals(authReq) && session.getIsCredentialOverride()) {
+            try {
+                // Breakdown the payload and then recompose it with the new credentials
+                String bodyStr = new String(body, "UTF-8");
+                String[] tokens = bodyStr.split("&");
+                String newBody = "";
+                for (int x=0; x < tokens.length; x++) {
+                    if (x == 0) {
+                        String usernameEnc = URLEncoder.encode(session.getUsername(), "UTF-8");
+                        newBody += ("username=" + usernameEnc);
+                    }
+                    if (x == 1) {
+                        String passwordEnc = URLEncoder.encode(session.getPassword(), "UTF-8");
+                        newBody += ("&password=" + passwordEnc);
+                    }
+                    if (x > 1) {
+                        newBody += ("&" + tokens[x]);
+                    }
+                }
+                return newBody.getBytes();
+            } catch(UnsupportedEncodingException e) {
+                log.error("Error decoding request body, returning original content - {}", ExceptionUtils.getMessage(e), e);
+                return body;
+            }
+        }
+        else {
+            return body;
+        }
+    }
+
+    public void runShardManager(String scanID, ShardSession shard) {
+        Binding bindings = new Binding();
+        bindings.setProperty("shardProperties", shardProperties);
+        bindings.setVariable("teamName", shard.getTeam());
+        bindings.setVariable("projectName", shard.getProject());
+        try {
+            String scriptName = shardProperties.getScriptName();
+            String scriptDir = shardProperties.getScriptPath();
+            String[] roots = new String[] { scriptDir };
+            GroovyScriptEngine gse = new GroovyScriptEngine(roots);
+            gse.run(scriptName, bindings);
+            shard.setShardFound(true);
+            shard.setUrl((String)bindings.getVariable("url"));
+            shard.setName((String)bindings.getVariable("shardName"));
+            shard.setIsCredentialOverride((boolean)bindings.getVariable("isCredentialOverride"));
+            // Examin the local shard list for username/password values
+            List<ShardConfig> shardProps = shardProperties.getShardConfig();
+            for(int i=0; i < shardProps.size(); i++) {
+                ShardConfig shardProp = shardProps.get(i);
+                if (shardProp.getName().equals(shard.getName()) ) {
+                    shard.setUsername(shardProp.getUsername());
+                    shard.setPassword(shardProp.getPassword());
+                }
+            }
+        } catch (GroovyRuntimeException | IOException | ResourceException | ScriptException e) {
+            log.error("Error occurred while executing Shard Manager, returning null - {}", ExceptionUtils.getMessage(e), e);
+        }
+    }
+
+    //
+    /// You cannot directly alter the URI associated with an HttpRequest and the URI needs to altered based on the
+    /// picked shard. This simply handles the rewrite process.
+    //
+    private class ShardRequestWrapper extends HttpRequestWrapper {
+        HttpRequest request;
+        String shardAddress;
+
+        public ShardRequestWrapper(HttpRequest request, String shardAddress) {
+            super(request);
+            this.request = request;
+            this.shardAddress = shardAddress;
+        }
+
+        @Override
+        public String getMethodValue() { return super.getMethodValue(); }
+
+        @Override
+        public HttpHeaders getHeaders() {
+            return this.request.getHeaders();
+        }
+
+        @Override
+        public URI getURI() {
+            try {
+                String rewrittenURI = shardAddress + super.getURI().getPath();
+                if (super.getURI().getQuery() != null) {
+                    rewrittenURI = rewrittenURI + "?" + super.getURI().getQuery();
+                }
+                return new URI(rewrittenURI);
+            } catch (URISyntaxException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+}

src/main/java/com/checkmarx/sdk/ShardManager/ShardManagerInterceptorConfig.java

@@ -0,0 +1,66 @@
+package com.checkmarx.sdk.ShardManager;
+import com.checkmarx.sdk.config.CxProperties;
+import com.checkmarx.sdk.config.ShardProperties;
+import groovy.lang.Binding;
+import groovy.lang.GroovyRuntimeException;
+import groovy.util.GroovyScriptEngine;
+import groovy.util.ResourceException;
+import groovy.util.ScriptException;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.client.ClientHttpRequestInterceptor;
+import org.springframework.web.client.RestTemplate;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+@Configuration
+public class ShardManagerInterceptorConfig {
+    private static final Logger log = org.slf4j.LoggerFactory.getLogger(ShardManagerHTTPInterceptor.class);
+    private CxProperties cxProperties;
+    private ShardProperties shardProperties;
+    private ShardSessionTracker sessionTracker;
+
+    public ShardManagerInterceptorConfig(ShardProperties shardProperties,
+                                         CxProperties cxProperties,
+                                         ShardSessionTracker sessionTracker) {
+        this.shardProperties = shardProperties;
+        this.cxProperties = cxProperties;
+        this.sessionTracker = sessionTracker;
+    }
+
+    @Bean
+    public RestTemplate restTemplate(@Qualifier("cxRestTemplate") RestTemplate restTemplate,
+                                     ShardProperties shardProperties) {
+        if(cxProperties.getEnableShardManager()) {
+            loadShardSettings();
+            List<ClientHttpRequestInterceptor> interceptors = restTemplate.getInterceptors();
+            if (CollectionUtils.isEmpty(interceptors)) {
+                interceptors = new ArrayList<>();
+            }
+            interceptors.add(new ShardManagerHTTPInterceptor(shardProperties, sessionTracker));
+            restTemplate.setInterceptors(interceptors);
+        }
+        return restTemplate;
+    }
+
+    public String loadShardSettings() {
+        Binding binding = new Binding();
+        binding.setProperty("shardProperties", shardProperties);
+        try {
+            String scriptName = shardProperties.getScriptSetup();
+            String scriptDir = shardProperties.getScriptPath();
+            String[] roots = new String[] { scriptDir };
+            GroovyScriptEngine gse = new GroovyScriptEngine(roots);
+            gse.run(scriptName, binding);
+            return (String)binding.getVariable("output");
+        } catch (GroovyRuntimeException | IOException | ResourceException | ScriptException e) {
+            log.error("Error occurred while executing Shard Manager Setup, returning null - {}", ExceptionUtils.getMessage(e), e);
+            return null;
+        }
+    }
+}