Merge pull request #33 from IBM/saikumar1607-202309291030

new format configurations for bootstrap file

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$|go.sum|vendor|.cra/.cveignore",
     "lines": null
   },
-  "generated_at": "2023-08-21T06:52:04Z",
+  "generated_at": "2023-09-29T10:32:04Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"

README.md

@@ -1,4 +1,4 @@
-# IBM Cloud App Configuration Go server SDK 0.4.2
+# IBM Cloud App Configuration Go server SDK 0.5.0
 
 IBM Cloud App Configuration SDK is used to perform feature flag and property evaluation based on the configuration on
 IBM Cloud App Configuration service.
@@ -103,10 +103,6 @@ service across your application restarts, you can configure the SDK to work usin
 persistent cache to store the App Configuration data that will be available across your application restarts.
 
 ```go
-// 1. default (without persistent cache)
-appConfigClient.SetContext(collectionId, environmentId)
-
-// 2. optional (with persistent cache)
 appConfigClient.SetContext(collectionId, environmentId, AppConfiguration.ContextOptions{
     PersistentCacheDirectory: "/var/lib/docker/volumes/",
 })
@@ -135,9 +131,9 @@ appConfigClient.SetContext(collectionId, environmentId, AppConfiguration.Context
 ```
 
 * BootstrapFile: Absolute path of the JSON file, which contains configuration details. Make sure to provide a proper
-  JSON file. You can generate this file using `ibmcloud ac config` command of the IBM Cloud App Configuration CLI.
+  JSON file. You can generate this file using `ibmcloud ac export` command of the IBM Cloud App Configuration CLI.
 * LiveConfigUpdateEnabled: Live configuration update from the server. Set this value to `false` if the new configuration
-  values shouldn't be fetched from the server. By default, this value is enabled.
+  values shouldn't be fetched from the server. By default, this value is set to `true`.
 
 ## Get single feature
 

examples/go.mod

@@ -3,7 +3,7 @@ module examples
 go 1.16
 
 require (
-	github.com/IBM/appconfiguration-go-sdk v0.4.2
-	github.com/gorilla/mux v1.7.2
-	github.com/joho/godotenv v1.4.0
+	github.com/IBM/appconfiguration-go-sdk v0.5.0
+	github.com/gorilla/mux v1.8.0
+	github.com/joho/godotenv v1.5.1
 )

go.mod

@@ -3,8 +3,8 @@ module github.com/IBM/appconfiguration-go-sdk
 go 1.16
 
 require (
-	github.com/IBM/go-sdk-core/v5 v5.13.4
-	github.com/IBM/secrets-manager-go-sdk/v2 v2.0.0
+	github.com/IBM/go-sdk-core/v5 v5.14.1
+	github.com/IBM/secrets-manager-go-sdk/v2 v2.0.1
 	github.com/go-openapi/strfmt v0.21.7 // indirect
 	github.com/gorilla/websocket v1.5.0
 	github.com/robfig/cron v1.2.0

go.sum

@@ -1,10 +1,10 @@
 github.com/IBM/go-sdk-core/v5 v5.10.2/go.mod h1:WZPFasUzsKab/2mzt29xPcfruSk5js2ywAPwW4VJjdI=
-github.com/IBM/go-sdk-core/v5 v5.13.4 h1:kJvBNQOwhFRkXCPapjNvKVC7n7n2vd1Nr6uUtDZGcfo=
-github.com/IBM/go-sdk-core/v5 v5.13.4/go.mod h1:gKRSB+YyKsGlRQW7v5frlLbue5afulSvrRa4O26o4MM=
+github.com/IBM/go-sdk-core/v5 v5.14.1 h1:WR1r0zz+gDW++xzZjF41r9ueY4JyjS2vgZjiYs8lO3c=
+github.com/IBM/go-sdk-core/v5 v5.14.1/go.mod h1:MUvIr/1mgGh198ZXL+ByKz9Qs1JoEh80v/96x8jPXNY=
 github.com/IBM/secrets-manager-go-sdk v1.2.0 h1:bgFfBF+LjHLtUfV3hTLkfgE8EjFsJaeU2icA2Hg+M50=
 github.com/IBM/secrets-manager-go-sdk v1.2.0/go.mod h1:qv+tQg8Z3Vb11DQYxDjEGeROHDtTLQxUWuOIrIdWg6E=
-github.com/IBM/secrets-manager-go-sdk/v2 v2.0.0 h1:Lx4Bvim/MfoHEYR+n312bty5DirAJypBGGS9YZo3zCw=
-github.com/IBM/secrets-manager-go-sdk/v2 v2.0.0/go.mod h1:jagqWmjZ0zUEqh5jdGB42ApSQS40fu2LWw6pdg8JJko=
+github.com/IBM/secrets-manager-go-sdk/v2 v2.0.1 h1:0Ouu31RsuOLdH26oNsnPErEjctWTplLEIXxwExnTZT0=
+github.com/IBM/secrets-manager-go-sdk/v2 v2.0.1/go.mod h1:jagqWmjZ0zUEqh5jdGB42ApSQS40fu2LWw6pdg8JJko=
 github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=

lib/AppConfiguration.go

@@ -18,12 +18,11 @@ package lib
 
 import (
 	"errors"
-	"os"
-
 	"github.com/IBM/appconfiguration-go-sdk/lib/internal/messages"
 	"github.com/IBM/appconfiguration-go-sdk/lib/internal/models"
 	"github.com/IBM/appconfiguration-go-sdk/lib/internal/utils/log"
 	sm "github.com/IBM/secrets-manager-go-sdk/v2/secretsmanagerv2"
+	"path/filepath"
 )
 
 // AppConfiguration : Struct having init and configInstance.
@@ -37,7 +36,6 @@ type AppConfiguration struct {
 type ContextOptions struct {
 	PersistentCacheDirectory string
 	BootstrapFile            string
-	ConfigurationFile        string
 	LiveConfigUpdateEnabled  bool
 }
 
@@ -61,10 +59,6 @@ const REGION_AU_SYD = "au-syd"
 // REGION_US_EAST : Washington DC Region
 const REGION_US_EAST = "us-east"
 
-func init() {
-	log.SetLogLevel("info")
-}
-
 // GetInstance : Get App Configuration Instance
 func GetInstance() *AppConfiguration {
 	log.Debug(messages.RetrieveingAppConfig)
@@ -134,9 +128,9 @@ func (ac *AppConfiguration) SetContext(collectionID string, environmentID string
 		})
 	case 1:
 		var temp = options[0]
-		if len(temp.ConfigurationFile) > 0 && len(temp.BootstrapFile) == 0 {
-			temp.BootstrapFile = temp.ConfigurationFile
-			log.Info(messages.ContextOptionsParameterDeprecation)
+		if len(temp.BootstrapFile) > 0 && filepath.Ext(temp.BootstrapFile) != ".json" {
+			log.Error(messages.InvalidBootstrapFile, " - ", temp.BootstrapFile)
+			return
 		}
 		if !temp.LiveConfigUpdateEnabled && len(temp.BootstrapFile) == 0 {
 			log.Error(messages.BootstrapFileNotFoundError)
@@ -228,10 +222,8 @@ func (ac *AppConfiguration) GetSecret(propertyID string, secretsManagerService *
 // EnableDebug : Enable Debug
 func (ac *AppConfiguration) EnableDebug(enabled bool) {
 	if enabled {
-		os.Setenv("ENABLE_DEBUG", "true")
 		log.SetLogLevel("debug")
 	} else {
-		os.Setenv("ENABLE_DEBUG", "false")
 		log.SetLogLevel("info")
 	}
 }

lib/ConfigurationHandler.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net/http"
 	"path/filepath"
 	"sync"
@@ -55,6 +56,7 @@ type ConfigurationHandler struct {
 	liveConfigUpdateEnabled     bool
 	persistentData              []byte
 	retryInterval               int64
+	scheduledRetry              *time.Timer
 	socketConnection            *websocket.Conn
 	socketConnectionResponse    *http.Response
 	mu                          sync.Mutex
@@ -89,35 +91,41 @@ func (ch *ConfigurationHandler) SetContext(collectionID, environmentID string, o
 	ch.bootstrapFile = options.BootstrapFile
 	ch.liveConfigUpdateEnabled = options.LiveConfigUpdateEnabled
 	ch.isInitialized = true
-	ch.retryInterval = 600
+	ch.retryInterval = 2 // two minutes
 }
 func (ch *ConfigurationHandler) loadData() {
-	if !ch.isInitialized {
-		log.Error(messages.ConfigurationHandlerInitError)
-	}
+	persistentCacheRead := false
+
 	if len(ch.persistentCacheDirectory) > 0 {
-		ch.persistentData = utils.ReadFiles(filepath.Join(utils.SanitizePath(ch.persistentCacheDirectory), constants.ConfigurationFile))
+		path := filepath.Join(utils.SanitizePath(ch.persistentCacheDirectory), constants.ConfigurationFile)
+		log.Info(messages.ReadPersistentCache, path)
+		ch.persistentData = utils.ReadFiles(path)
 		if !bytes.Equal(ch.persistentData, []byte(`{}`)) {
-			// no updating the listener here. Only updating cache is enough
-			ch.saveInCache(ch.persistentData)
+			configurations := models.ExtractConfigurationsFromPersistentCache(ch.persistentData)
+			if configurations != nil {
+				ch.saveInCache(configurations)
+				persistentCacheRead = true
+			}
 		}
 	}
 	if len(ch.bootstrapFile) > 0 {
-		log.Info(messages.BootstrapFileProvided, "file path is:", ch.bootstrapFile)
+		path := utils.SanitizePath(ch.bootstrapFile)
 		if len(ch.persistentCacheDirectory) > 0 {
-			if bytes.Equal(ch.persistentData, []byte(`{}`)) {
-				bootstrapFileData := utils.ReadFiles(utils.SanitizePath(ch.bootstrapFile))
-				go utils.StoreFiles(string(bootstrapFileData), ch.persistentCacheDirectory)
-				ch.updateCacheAndListener(bootstrapFileData)
-			} else {
-				// update the only listener here. Because, cache is already updated above (line 100)
-				if ch.configurationUpdateListener != nil {
-					ch.configurationUpdateListener()
+			if !persistentCacheRead {
+				bootstrapFileData := utils.ReadFiles(path)
+				bootstrapConfigurations := models.ExtractConfigurationsFromBootstrapJson(bootstrapFileData, ch.collectionID, ch.environmentID)
+				if bootstrapConfigurations != nil {
+					ch.saveInCache(bootstrapConfigurations)
+					go utils.StoreFiles(string(models.FormatConfig(bootstrapConfigurations, ch.environmentID)), ch.persistentCacheDirectory)
 				}
 			}
 		} else {
-			bootstrapFileData := utils.ReadFiles(utils.SanitizePath(ch.bootstrapFile))
-			ch.updateCacheAndListener(bootstrapFileData)
+			log.Info(messages.ReadBootstrapConfigurations, path)
+			bootstrapFileData := utils.ReadFiles(path)
+			bootstrapConfigurations := models.ExtractConfigurationsFromBootstrapJson(bootstrapFileData, ch.collectionID, ch.environmentID)
+			if bootstrapConfigurations != nil {
+				ch.saveInCache(bootstrapConfigurations)
+			}
 		}
 	}
 	if ch.liveConfigUpdateEnabled {
@@ -136,25 +144,25 @@ func (ch *ConfigurationHandler) FetchConfigurationData() {
 func (ch *ConfigurationHandler) saveInCache(data []byte) {
 	ch.mu.Lock()
 	defer ch.mu.Unlock()
-	configResponse := models.ConfigResponse{}
-	err := json.Unmarshal(data, &configResponse)
+	configurations := models.Configurations{}
+	err := json.Unmarshal(data, &configurations)
 	if err != nil {
 		log.Error(messages.UnmarshalJSONErr, err)
 		return
 	}
-	log.Debug(configResponse)
+	log.Debug(configurations)
 	featureMap := make(map[string]models.Feature)
-	for _, feature := range configResponse.Features {
+	for _, feature := range configurations.Features {
 		featureMap[feature.GetFeatureID()] = feature
 	}
 
 	propertyMap := make(map[string]models.Property)
-	for _, property := range configResponse.Properties {
+	for _, property := range configurations.Properties {
 		propertyMap[property.GetPropertyID()] = property
 	}
 
 	segmentMap := make(map[string]models.Segment)
-	for _, segment := range configResponse.Segments {
+	for _, segment := range configurations.Segments {
 		segmentMap[segment.GetSegmentID()] = segment
 	}
 	log.Debug(messages.SetInMemoryCache)
@@ -170,14 +178,15 @@ func (ch *ConfigurationHandler) updateCacheAndListener(data []byte) {
 func (ch *ConfigurationHandler) fetchFromAPI() {
 	if ch.isInitialized {
 		builder := core.NewRequestBuilder(core.GET)
+		builder.AddQuery("action", "sdkConfig")
+		builder.AddQuery("collection_id", ch.collectionID)
 		builder.AddQuery("environment_id", ch.environmentID)
 		pathParamsMap := map[string]string{
-			"guid":          ch.guid,
-			"collection_id": ch.collectionID,
+			"guid": ch.guid,
 		}
-		_, err := builder.ResolveRequestURL(ch.urlBuilder.GetBaseServiceURL(), `/apprapp/feature/v1/instances/{guid}/collections/{collection_id}/config`, pathParamsMap)
+		_, err := builder.ResolveRequestURL(ch.urlBuilder.GetBaseServiceURL(), `/apprapp/feature/v1/instances/{guid}/config`, pathParamsMap)
 		if err != nil {
-			log.Error(err)
+			log.Error(err.Error())
 			return
 		}
 		builder.AddHeader("Accept", "application/json")
@@ -197,33 +206,43 @@ func (ch *ConfigurationHandler) fetchFromAPI() {
 		// For 429 error code - The Request() will retry the request 3 times in an interval of time mentioned in ["Retry-after"] header.
 		// If all the 3 retries exhausts the call is returned and execution is given back to us to take the response object ahead.
 		//
-		// Both the cases [429 & 5xx] we schedule a retry after 10 minutes.
+		// Both the cases [429, 499 & 5xx] we schedule a retry after 2 minutes.
 
 		response, err := utils.GetAPIManagerInstance().Request(builder)
-		if response != nil && response.StatusCode == constants.StatusCodeOK {
+		if response != nil && response.StatusCode == 200 {
 			log.Debug(messages.FetchAPISuccessful)
 			jsonData, _ := json.Marshal(response.Result)
+			configurations := models.ExtractConfigurationsFromAPIResponse(jsonData)
+			if configurations == nil {
+				return
+			}
 			// asynchronously write the response to persistent volume, if enabled
 			if len(ch.persistentCacheDirectory) > 0 {
 				go utils.StoreFiles(string(jsonData), ch.persistentCacheDirectory)
 			}
 			// load the configurations in the response to cache maps
-			ch.updateCacheAndListener(jsonData)
+			ch.updateCacheAndListener(configurations)
 		} else {
 			if response != nil {
 				if response.Result != nil {
-					log.Error(response.Result, err)
+					log.Error(response.Result, err.Error())
 				} else {
-					log.Error(string(response.RawResult), err)
+					log.Error(string(response.RawResult), err.Error())
+				}
+				statusCode := response.StatusCode
+				if statusCode >= 400 && statusCode < 499 && statusCode != 429 {
+					// Do Nothing! GET "/config" failed due to client-side error.
+					return
 				}
-				if response.StatusCode == constants.StatusCodeTooManyRequests || (response.StatusCode >= constants.StatusCodeServerErrorBegin && response.StatusCode <= constants.StatusCodeServerErrorEnd) {
-					time.AfterFunc(time.Second*time.Duration(ch.retryInterval), func() {
-						ch.fetchFromAPI()
-					})
-					log.Info(messages.RetryScheduledMessage)
+				if ch.scheduledRetry != nil {
+					ch.scheduledRetry.Stop()
 				}
+				ch.scheduledRetry = time.AfterFunc(time.Minute*time.Duration(ch.retryInterval), func() {
+					ch.fetchFromAPI()
+				})
+				log.Info(fmt.Sprintf(messages.RetryScheduledMessage, ch.retryInterval))
 			} else {
-				log.Error(messages.ConfigAPIError, err)
+				log.Error(messages.ConfigAPIError, err.Error())
 			}
 		}
 	} else {
@@ -239,23 +258,26 @@ func (ch *ConfigurationHandler) startWebSocket() {
 		log.Error(messages.WebSocketConnectFailed, messages.AuthTokenError)
 		return
 	}
-	h := http.Header{"Authorization": []string{authToken}}
+	h := make(http.Header)
+	h.Add("Authorization", authToken)
+	h.Add("User-Agent", constants.UserAgent)
 	var err error
 	if ch.socketConnection != nil {
 		ch.socketConnection.Close()
 	}
 	ch.socketConnection, ch.socketConnectionResponse, err = websocket.DefaultDialer.Dial(ch.urlBuilder.GetWebSocketURL(), h)
 	if err != nil {
 		if ch.socketConnectionResponse != nil {
-			log.Error(messages.WebSocketConnectErr, err, ch.socketConnectionResponse.StatusCode)
-			// websocket dial that fails with response status code in between 400-499, except 429, are not retried as failure is due to client side error
-			socketConnectRespStatusCode := ch.socketConnectionResponse.StatusCode
-			if socketConnectRespStatusCode >= constants.StatusCodeClientErrorBegin &&
-				socketConnectRespStatusCode <= constants.StatusCodeClientErrorEnd &&
-				socketConnectRespStatusCode != constants.StatusCodeTooManyRequests {
+			statusCode := ch.socketConnectionResponse.StatusCode
+			if statusCode >= 400 && statusCode < 499 && statusCode != 429 {
+				// websocket dial that fails with response status code in between 400-499, except 429 & 499, are not retried.
+				// Do Nothing! Since websocket connect failed due to client-side error.
+				log.Error(messages.WebSocketConnectErr+err.Error(), " ", statusCode)
 				return
 			}
 		}
+		log.Error(messages.WebSocketConnectErr, err.Error(), " Retrying websocket connect in 15 seconds...")
+		time.Sleep(15 * time.Second)
 		go ch.startWebSocket()
 		return
 	}
@@ -314,8 +336,6 @@ func (ch *ConfigurationHandler) getProperty(propertyID string) (models.Property,
 	log.Error(messages.InvalidPropertyID, propertyID)
 	return models.Property{}, errors.New(messages.ErrorInvalidPropertyID + propertyID)
 }
-
-// GetSecret : Get Secret
 func (ch *ConfigurationHandler) getSecret(propertyID string, secretsManagerService *sm.SecretsManagerV2) (models.SecretProperty, error) {
 	property, err := ch.getProperty(propertyID)
 	if err != nil {

lib/appconfiguration_test.go

@@ -119,6 +119,19 @@ func TestSetContext(t *testing.T) {
 		t.Errorf("Test failed: Incorrect error message")
 	}
 	reset(ac)
+
+	// test bootstrap initialisation when file path in invalid. (it should be string ending with .json only)
+	ac.Init("a", "b", "c")
+	ac.isInitialized = true
+	assert.Equal(t, false, ac.isInitializedConfig)
+	ac.SetContext("c1", "dev", ContextOptions{
+		BootstrapFile:           "my-bootstrap-file",
+		LiveConfigUpdateEnabled: false,
+	})
+	if hook.LastEntry().Message != "AppConfiguration - Invalid value provided for BootstrapFile parameter - my-bootstrap-file" {
+		t.Errorf("Test failed: Incorrect error message")
+	}
+	reset(ac)
 }
 func TestGetFeature(t *testing.T) {
 	// test get feature when not initialised properly