From 6061b4718e49c94668a19b295f870141ba9f16a7 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Tue, 6 Dec 2022 16:19:23 -0500 Subject: [PATCH 1/6] Reduce memory use for PULUMI_OPTIMIZED_CHECKPOINT_PATCH --- pkg/backend/httpstate/client/client_test.go | 48 ++++++--- pkg/backend/httpstate/client/marshal.go | 101 ++++++++++++++++-- pkg/backend/httpstate/diffs.go | 27 ++--- pkg/backend/httpstate/marshal.go | 35 ------ pkg/backend/httpstate/snapshot.go | 20 +--- pkg/backend/httpstate/snapshot_test.go | 69 +++++++----- .../httpstate/testdata/snapshot_test.json | 7 ++ pkg/go.mod | 5 +- pkg/go.sum | 7 +- tests/go.mod | 5 +- tests/go.sum | 10 +- 11 files changed, 198 insertions(+), 136 deletions(-) delete mode 100644 pkg/backend/httpstate/marshal.go create mode 100755 pkg/backend/httpstate/testdata/snapshot_test.json diff --git a/pkg/backend/httpstate/client/client_test.go b/pkg/backend/httpstate/client/client_test.go index 30eaef55b35d..6e227827f569 100644 --- a/pkg/backend/httpstate/client/client_test.go +++ b/pkg/backend/httpstate/client/client_test.go @@ -14,11 +14,13 @@ package client import ( + "bytes" "compress/gzip" "context" "encoding/json" "net/http" "net/http/httptest" + "strings" "testing" "github.com/pulumi/pulumi/sdk/v3/go/common/apitype" @@ -135,25 +137,25 @@ func TestGzip(t *testing.T) { } -func TestPatchUpdateCheckpointVerbatimPreservesIndent(t *testing.T) { +func TestPatchUpdateCheckpointVerbatimIndents(t *testing.T) { t.Parallel() deployment := apitype.DeploymentV3{ - Resources: []apitype.ResourceV3{{URN: resource.URN("urn1")}}, + Resources: []apitype.ResourceV3{ + {URN: resource.URN("urn1")}, + {URN: resource.URN("urn2")}, + }, } - var indented json.RawMessage - { - indented1, err := json.MarshalIndent(deployment, "", "") - require.NoError(t, err) - untyped := apitype.UntypedDeployment{ - Version: 3, - Deployment: indented1, - } - indented2, err := json.MarshalIndent(untyped, "", "") - require.NoError(t, err) - indented = indented2 - } + var serializedDeployment json.RawMessage + serializedDeployment, err := json.Marshal(deployment) + assert.NoError(t, err) + + untypedDeployment, err := json.Marshal(apitype.UntypedDeployment{ + Version: 3, + Deployment: serializedDeployment, + }) + assert.NoError(t, err) var request apitype.PatchUpdateVerbatimCheckpointRequest @@ -173,11 +175,25 @@ func TestPatchUpdateCheckpointVerbatimPreservesIndent(t *testing.T) { sequenceNumber := 1 - err := client.PatchUpdateCheckpointVerbatim(context.Background(), + indented, err := MarshalUntypedDeployment(&deployment) + assert.NoError(t, err) + + err = client.PatchUpdateCheckpointVerbatim(context.Background(), UpdateIdentifier{}, sequenceNumber, indented, "token") assert.NoError(t, err) - assert.Equal(t, string(indented), string(request.UntypedDeployment)) + compacted := func(raw json.RawMessage) string { + var buf bytes.Buffer + err := json.Compact(&buf, []byte(raw)) + assert.NoError(t, err) + return buf.String() + } + + // It should have more than one line as json.Marshal would produce. + assert.Equal(t, 4, len(strings.Split(string(request.UntypedDeployment), "\n"))) + + // Compacting should recover the same form as json.Marshal would produce. + assert.Equal(t, string(untypedDeployment), compacted(request.UntypedDeployment)) } func TestGetCapabilities(t *testing.T) { diff --git a/pkg/backend/httpstate/client/marshal.go b/pkg/backend/httpstate/client/marshal.go index 5fc10bd70e12..39a9a240287a 100644 --- a/pkg/backend/httpstate/client/marshal.go +++ b/pkg/backend/httpstate/client/marshal.go @@ -17,19 +17,102 @@ package client import ( "bytes" "encoding/json" + "io" + "math" + + "github.com/json-iterator/go" "github.com/pulumi/pulumi/sdk/v3/go/common/apitype" ) -// Unlike json.Marshal preserves possible indentation in req.Deployment. -func marshalVerbatimCheckpointRequest(req apitype.PatchUpdateVerbatimCheckpointRequest) (json.RawMessage, error) { - sentinel := []byte(`"*"`) - cp := req - cp.UntypedDeployment = sentinel - pattern, err := json.Marshal(cp) - if err != nil { +var jsonIterConfig = jsoniter.Config{SortMapKeys: true}.Froze() + +// Marshals to canonical JSON in the apitype.UntypedDeployment format. +// +// Optimized for large checkpoints. +// +// Injects newlines to allow efficient textual diffs over the JSON. Textual diffs currently use O(N^2) memory in the +// number of newlines, so the injection needs to be conservative. Currently it limits to up to 1024 newlines which would +// result in max 8MB memory use by the algorithm. +func MarshalUntypedDeployment(deployment *apitype.DeploymentV3) (json.RawMessage, error) { + var buf bytes.Buffer + md := &marshalUntypedDeployment{deployment} + if err := md.Write(&buf); err != nil { return nil, err } - f := bytes.ReplaceAll(pattern, sentinel, req.UntypedDeployment) - return f, nil + return buf.Bytes(), nil +} + +func marshalVerbatimCheckpointRequest(req apitype.PatchUpdateVerbatimCheckpointRequest) (json.RawMessage, error) { + // Unlike encoding/json, using jsonIter here will not reindent req.UntypedDeployment, which is what is needed + // for the Verbatim protocol. + return jsonIterConfig.Marshal(req) +} + +type marshalUntypedDeployment struct { + deployment *apitype.DeploymentV3 +} + +func (c *marshalUntypedDeployment) Write(w io.Writer) error { + cfg := jsonIterConfig + stream := cfg.BorrowStream(w) + defer cfg.ReturnStream(stream) + err := c.writeToStream(stream) + return err +} + +func (c *marshalUntypedDeployment) writeToStream(stream *jsoniter.Stream) error { + stream.WriteObjectStart() + stream.WriteObjectField("version") + stream.WriteInt(3) + stream.WriteMore() + stream.WriteObjectField("deployment") + c.writeDeploymentV3(stream) + stream.WriteObjectEnd() + return stream.Flush() +} + +func (c *marshalUntypedDeployment) writeDeploymentV3(stream *jsoniter.Stream) (err error) { + deployment := c.deployment + stream.WriteObjectStart() + stream.WriteObjectField("manifest") + stream.WriteVal(deployment.Manifest) + if deployment.SecretsProviders != nil { + stream.WriteMore() + stream.WriteObjectField("secrets_providers") + stream.WriteVal(deployment.SecretsProviders) + } + if err = stream.Flush(); err != nil { + return err + } + nResources := len(deployment.Resources) + + maxNewlines := 1024 - 2 + newlinePeriod := int(math.Ceil(float64(nResources) / float64(maxNewlines))) + + if nResources > 0 { + stream.WriteMore() + stream.WriteObjectField("resources") + stream.WriteRaw("[\n") + for i, r := range deployment.Resources { + if i > 0 { + stream.WriteRaw(",") + if (nResources <= maxNewlines) || (i%newlinePeriod == 0) { + stream.WriteRaw("\n") + } + } + stream.WriteVal(r) + if err = stream.Flush(); err != nil { + return err + } + } + stream.WriteRaw("\n]") + } + if len(deployment.PendingOperations) > 0 { + stream.WriteMore() + stream.WriteObjectField("pendingOperations") + stream.WriteVal(deployment.PendingOperations) + } + stream.WriteObjectEnd() + return stream.Flush() } diff --git a/pkg/backend/httpstate/diffs.go b/pkg/backend/httpstate/diffs.go index 19a40ca6af8f..a7f91ad4c0a3 100644 --- a/pkg/backend/httpstate/diffs.go +++ b/pkg/backend/httpstate/diffs.go @@ -25,8 +25,6 @@ import ( "github.com/hexops/gotextdiff/myers" "github.com/hexops/gotextdiff/span" - "github.com/pulumi/pulumi/sdk/v3/go/common/apitype" - opentracing "github.com/opentracing/opentracing-go" ) @@ -59,39 +57,30 @@ func (dds *deploymentDiffState) CanDiff() bool { // Size-based heuristics trying to estimate if the diff method will be // worth it and take less time than sending the entire deployment. -func (dds *deploymentDiffState) ShouldDiff(new *apitype.UntypedDeployment) bool { +func (dds *deploymentDiffState) ShouldDiff(new json.RawMessage) bool { if !dds.CanDiff() { return false } if len(dds.lastSavedDeployment) < dds.minimalDiffSize { return false } - if len(new.Deployment) < dds.minimalDiffSize { + if len(new) < dds.minimalDiffSize { return false } return true } -func (dds *deploymentDiffState) Diff(ctx context.Context, - deployment *apitype.UntypedDeployment) (deploymentDiff, error) { +func (dds *deploymentDiffState) Diff(ctx context.Context, deployment json.RawMessage) (deploymentDiff, error) { if !dds.CanDiff() { return deploymentDiff{}, fmt.Errorf("Diff() cannot be called before Saved()") } - if deployment.Version == 0 { - return deploymentDiff{}, fmt.Errorf("deployment.Version should be set") - } - tracingSpan, childCtx := opentracing.StartSpanFromContext(ctx, "Diff") defer tracingSpan.Finish() before := dds.lastSavedDeployment - - after, err := marshalUntypedDeployment(deployment) - if err != nil { - return deploymentDiff{}, fmt.Errorf("marshalUntypedDeployment failed: %v", err) - } + after := deployment var checkpointHash string checkpointHashReady := &sync.WaitGroup{} @@ -125,12 +114,8 @@ func (dds *deploymentDiffState) Diff(ctx context.Context, } // Indicates that a deployment was just saved to the service. -func (dds *deploymentDiffState) Saved(ctx context.Context, deployment *apitype.UntypedDeployment) error { - d, err := marshalUntypedDeployment(deployment) - if err != nil { - return err - } - dds.lastSavedDeployment = d +func (dds *deploymentDiffState) Saved(ctx context.Context, deployment json.RawMessage) error { + dds.lastSavedDeployment = deployment dds.sequenceNumber++ return nil diff --git a/pkg/backend/httpstate/marshal.go b/pkg/backend/httpstate/marshal.go deleted file mode 100644 index 5ffd12097794..000000000000 --- a/pkg/backend/httpstate/marshal.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2016-2022, Pulumi Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package httpstate - -import ( - "bytes" - "encoding/json" - - "github.com/pulumi/pulumi/sdk/v3/go/common/apitype" -) - -// Unlike json.Marshal preserves possible indentation in the Deployment field. -func marshalUntypedDeployment(deployment *apitype.UntypedDeployment) (json.RawMessage, error) { - sentinel := []byte(`"*"`) - cp := *deployment - cp.Deployment = sentinel - pattern, err := json.Marshal(cp) - if err != nil { - return nil, err - } - fixed := bytes.ReplaceAll(pattern, sentinel, deployment.Deployment) - return fixed, nil -} diff --git a/pkg/backend/httpstate/snapshot.go b/pkg/backend/httpstate/snapshot.go index a2801a4216b9..5964d0899e42 100644 --- a/pkg/backend/httpstate/snapshot.go +++ b/pkg/backend/httpstate/snapshot.go @@ -24,7 +24,6 @@ import ( "github.com/pulumi/pulumi/pkg/v3/resource/deploy" "github.com/pulumi/pulumi/pkg/v3/resource/stack" "github.com/pulumi/pulumi/pkg/v3/secrets" - "github.com/pulumi/pulumi/sdk/v3/go/common/apitype" "github.com/pulumi/pulumi/sdk/v3/go/common/util/logging" ) @@ -62,7 +61,7 @@ func (persister *cloudSnapshotPersister) Save(snapshot *deploy.Snapshot) error { persister.context, persister.update, deploymentV3, token) } - deployment, err := persister.marshalDeployment(deploymentV3) + deployment, err := client.MarshalUntypedDeployment(deploymentV3) if err != nil { return err } @@ -102,23 +101,10 @@ func (persister *cloudSnapshotPersister) saveDiff(ctx context.Context, } func (persister *cloudSnapshotPersister) saveFullVerbatim(ctx context.Context, - differ *deploymentDiffState, deployment *apitype.UntypedDeployment, token string) error { - untypedDeploymentBytes, err := marshalUntypedDeployment(deployment) - if err != nil { - return err - } + differ *deploymentDiffState, deployment json.RawMessage, token string) error { return persister.backend.client.PatchUpdateCheckpointVerbatim( persister.context, persister.update, differ.SequenceNumber(), - untypedDeploymentBytes, token) -} - -func (persister *cloudSnapshotPersister) marshalDeployment( - deployment *apitype.DeploymentV3) (*apitype.UntypedDeployment, error) { - raw, err := json.MarshalIndent(deployment, "", "") - if err != nil { - return nil, fmt.Errorf("serializing deployment to json: %w", err) - } - return &apitype.UntypedDeployment{Deployment: raw, Version: 3}, nil + deployment, token) } var _ backend.SnapshotPersister = (*cloudSnapshotPersister)(nil) diff --git a/pkg/backend/httpstate/snapshot_test.go b/pkg/backend/httpstate/snapshot_test.go index 1d49b6b13afa..fd36b8ec47cf 100644 --- a/pkg/backend/httpstate/snapshot_test.go +++ b/pkg/backend/httpstate/snapshot_test.go @@ -23,6 +23,7 @@ import ( "fmt" "io" "io/ioutil" + "os" "testing" "net/http" @@ -30,12 +31,13 @@ import ( "github.com/hexops/gotextdiff" "github.com/stretchr/testify/assert" - - "github.com/pulumi/pulumi/sdk/v3/go/common/apitype" - "github.com/pulumi/pulumi/sdk/v3/go/common/resource" + "github.com/stretchr/testify/require" "github.com/pulumi/pulumi/pkg/v3/backend/httpstate/client" "github.com/pulumi/pulumi/pkg/v3/resource/deploy" + "github.com/pulumi/pulumi/sdk/v3/go/common/apitype" + "github.com/pulumi/pulumi/sdk/v3/go/common/resource" + "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" ) // Check that cloudSnapshotPersister can talk the diff-based @@ -45,6 +47,37 @@ func TestCloudSnapshotPersisterUseOfDiffProtocol(t *testing.T) { t.Parallel() ctx := context.Background() + expectationsFile := "testdata/snapshot_test.json" + expectations := map[string]string{} + accept := cmdutil.IsTruthy(os.Getenv("PULUMI_ACCEPT")) + if accept { + t.Cleanup(func() { + bytes, err := json.MarshalIndent(expectations, "", " ") + require.NoError(t, err) + err = os.WriteFile(expectationsFile, bytes, 0700) + require.NoError(t, err) + }) + } else { + data, err := os.ReadFile(expectationsFile) + require.NoError(t, err) + err = json.Unmarshal(data, &expectations) + require.NoError(t, err) + } + + assertEquals := func(expectedKey string, actual string) { + if accept { + expectations[expectedKey] = actual + return + } + expected, ok := expectations[expectedKey] + assert.True(t, ok) + assert.Equal(t, expected, actual, expectedKey) + } + + assertEqual := func(expectedKey string, actual json.RawMessage) { + assertEquals(expectedKey, string(actual)) + } + stackID := client.StackIdentifier{ Owner: "owner", Project: "project", @@ -156,9 +189,7 @@ func TestCloudSnapshotPersisterUseOfDiffProtocol(t *testing.T) { req1 := lastRequestAsVerbatim() assert.Equal(t, 1, req1.SequenceNumber) assert.Equal(t, 3, req1.Version) - assert.Equal(t, "{\"version\":3,\"deployment\":{\n\"manifest\": {\n\"time\": \"0001-01-01T00:00:00Z\""+ - ",\n\"magic\": \"\",\n\"version\": \"\"\n},\n\"resources\": [\n{\n\"urn\": \"urn-1\",\n\"custom\":"+ - " false,\n\"type\": \"\"\n}\n]\n}}", string(req1.UntypedDeployment)) + assertEqual("req1", req1.UntypedDeployment) handleVerbatim(req1) assert.Equal(t, []apitype.ResourceV3{ @@ -178,17 +209,8 @@ func TestCloudSnapshotPersisterUseOfDiffProtocol(t *testing.T) { req2 := lastRequestAsDelta() assert.Equal(t, 2, req2.SequenceNumber) - assert.Equal(t, "[{\"Span\":{\"uri\":\"\",\"start\":{\"line\":12,\"column\":1,\"offset\":-1},\"end\""+ - ":{\"line\":12,\"column\":1,\"offset\":-1}},\"NewText\":\"},\\n\"},{\"Span\":{\"uri\":\"\","+ - "\"start\":{\"line\":12,\"column\":1,\"offset\":-1},\"end\":{\"line\":12,"+ - "\"column\":1,\"offset\":-1}},\"NewText\":\"{\\n\"},{\"Span\":{\"uri\":\"\",\"start\":"+ - "{\"line\":12,\"column\":1,\"offset\":-1},\"end\":{\"line\":12,\"column\":1,\"offset\":-1}}"+ - ",\"NewText\":\"\\\"urn\\\": \\\"urn-2\\\",\\n\"},{\"Span\":{\"uri\":\"\",\"start\":"+ - "{\"line\":12,\"column\":1,\"offset\":-1},\"end\":{\"line\":12,\"column\":1,\"offset\":-1}}"+ - ",\"NewText\":\"\\\"custom\\\": false,\\n\"},{\"Span\":{\"uri\":\"\",\"start\":{\"line\":12,"+ - "\"column\":1,\"offset\":-1},\"end\":{\"line\":12,\"column\":1,\"offset\":-1}},\"NewText\":\""+ - "\\\"type\\\": \\\"\\\"\\n\"}]", string(req2.DeploymentDelta)) - assert.Equal(t, "75e2f82ca2735650366fba27b53ec97f310abd457aca27266fb29c4377ee00e7", req2.CheckpointHash) + assertEqual("req2", req2.DeploymentDelta) + assertEquals("req2.hash", req2.CheckpointHash) handleDelta(req2) assert.Equal(t, []apitype.ResourceV3{ @@ -207,17 +229,8 @@ func TestCloudSnapshotPersisterUseOfDiffProtocol(t *testing.T) { req3 := lastRequestAsDelta() assert.Equal(t, 3, req3.SequenceNumber) - assert.Equal(t, "[{\"Span\":{\"uri\":\"\",\"start\":{\"line\":12,\"column\":1,\"offset\":-1"+ - "},\"end\":{\"line\":13,\"column\":1,\"offset\":-1}},\"NewText\":\"\"},{\"Span\":{"+ - "\"uri\":\"\",\"start\":{\"line\":13,\"column\":1,\"offset\":-1},\"end\":{\"line\":14,"+ - "\"column\":1,\"offset\":-1}},\"NewText\":\"\"},{\"Span\":{\"uri\":\"\",\"start\":"+ - "{\"line\":14,\"column\":1,\"offset\":-1},\"end\":{\"line\":15,\"column\":1,"+ - "\"offset\":-1}},\"NewText\":\"\"},{\"Span\":{\"uri\":\"\",\"start\":{\"line\":15,"+ - "\"column\":1,\"offset\":-1},\"end\":{\"line\":16,\"column\":1,\"offset\":-1}},"+ - "\"NewText\":\"\"},{\"Span\":{\"uri\":\"\",\"start\":{\"line\":16,\"column\":1,"+ - "\"offset\":-1},\"end\":{\"line\":17,\"column\":1,\"offset\":-1}},\"NewText\":\"\"}]", - string(req3.DeploymentDelta)) - assert.Equal(t, "5f2fd84a225e7c1895528b4b9394607d6b39aefa4ee74f57e018dadcb16bf2e2", req3.CheckpointHash) + assertEqual("req3", req3.DeploymentDelta) + assertEquals("req3.hash", req3.CheckpointHash) handleDelta(req3) assert.Equal(t, []apitype.ResourceV3{ diff --git a/pkg/backend/httpstate/testdata/snapshot_test.json b/pkg/backend/httpstate/testdata/snapshot_test.json new file mode 100755 index 000000000000..65293295f700 --- /dev/null +++ b/pkg/backend/httpstate/testdata/snapshot_test.json @@ -0,0 +1,7 @@ +{ + "req1": "{\"version\":3,\"deployment\":{\"manifest\":{\"time\":\"0001-01-01T00:00:00Z\",\"magic\":\"\",\"version\":\"\"},\"resources\":[\n{\"urn\":\"urn-1\",\"custom\":false,\"type\":\"\"}\n]}}", + "req2": "[{\"Span\":{\"uri\":\"\",\"start\":{\"line\":2,\"column\":1,\"offset\":-1},\"end\":{\"line\":3,\"column\":1,\"offset\":-1}},\"NewText\":\"\"},{\"Span\":{\"uri\":\"\",\"start\":{\"line\":3,\"column\":1,\"offset\":-1},\"end\":{\"line\":3,\"column\":1,\"offset\":-1}},\"NewText\":\"{\\\"urn\\\":\\\"urn-1\\\",\\\"custom\\\":false,\\\"type\\\":\\\"\\\"},\\n\"},{\"Span\":{\"uri\":\"\",\"start\":{\"line\":3,\"column\":1,\"offset\":-1},\"end\":{\"line\":3,\"column\":1,\"offset\":-1}},\"NewText\":\"{\\\"urn\\\":\\\"urn-2\\\",\\\"custom\\\":false,\\\"type\\\":\\\"\\\"}\\n\"}]", + "req2.hash": "cc4369e46842399a9c997b84ada5cbc23b05ec247c6966d1ed1b4e4b5e4a4075", + "req3": "[{\"Span\":{\"uri\":\"\",\"start\":{\"line\":2,\"column\":1,\"offset\":-1},\"end\":{\"line\":3,\"column\":1,\"offset\":-1}},\"NewText\":\"\"},{\"Span\":{\"uri\":\"\",\"start\":{\"line\":3,\"column\":1,\"offset\":-1},\"end\":{\"line\":4,\"column\":1,\"offset\":-1}},\"NewText\":\"\"},{\"Span\":{\"uri\":\"\",\"start\":{\"line\":4,\"column\":1,\"offset\":-1},\"end\":{\"line\":4,\"column\":1,\"offset\":-1}},\"NewText\":\"{\\\"urn\\\":\\\"urn-1\\\",\\\"custom\\\":false,\\\"type\\\":\\\"\\\"}\\n\"}]", + "req3.hash": "8ac31c0d00401a0a5b8051220c2e593de7e0db7f89c7927be676558838aacd0a" +} \ No newline at end of file diff --git a/pkg/go.mod b/pkg/go.mod index ebf11ae030bc..c1437c9f1bad 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -69,6 +69,7 @@ require ( github.com/edsrzf/mmap-go v1.1.0 github.com/go-git/go-git/v5 v5.4.2 github.com/hexops/gotextdiff v1.0.3 + github.com/json-iterator/go v1.1.12 github.com/muesli/cancelreader v0.2.2 github.com/natefinch/atomic v1.0.1 github.com/pulumi/pulumi-java/pkg v0.7.0 @@ -184,10 +185,10 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kevinburke/ssh_config v1.1.0 // indirect github.com/klauspost/compress v1.15.1 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.12 // indirect @@ -213,7 +214,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/posener/complete v1.2.3 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/rogpeppe/go-internal v1.8.1 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect diff --git a/pkg/go.sum b/pkg/go.sum index 49ceed0ccd32..c7577a24bf6d 100644 --- a/pkg/go.sum +++ b/pkg/go.sum @@ -1153,8 +1153,9 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= @@ -1500,8 +1501,8 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= -github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= diff --git a/tests/go.mod b/tests/go.mod index 488c404f460f..a18cdaf84887 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -127,6 +127,7 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kevinburke/ssh_config v1.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect @@ -141,6 +142,8 @@ require ( github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/natefinch/atomic v1.0.1 // indirect github.com/nightlyone/lockfile v1.0.0 // indirect @@ -154,7 +157,7 @@ require ( github.com/pkg/term v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect - github.com/rogpeppe/go-internal v1.8.1 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect diff --git a/tests/go.sum b/tests/go.sum index ec4359918566..e0bd204b7c62 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -1079,6 +1079,7 @@ github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -1111,8 +1112,8 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= @@ -1230,9 +1231,11 @@ github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/f github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= @@ -1351,7 +1354,6 @@ github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1433,8 +1435,8 @@ github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= -github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= From 757d815b8823e4507f34e4c4dfe278a1bff9eb27 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Thu, 15 Dec 2022 11:30:31 -0500 Subject: [PATCH 2/6] CHANGELOG --- ...using-pulumi_optimized_checkpoint_patch-protocol-1165.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog/pending/20221215--backend-service--fixes-out-of-memory-issues-when-using-pulumi_optimized_checkpoint_patch-protocol-1165.yaml diff --git a/changelog/pending/20221215--backend-service--fixes-out-of-memory-issues-when-using-pulumi_optimized_checkpoint_patch-protocol-1165.yaml b/changelog/pending/20221215--backend-service--fixes-out-of-memory-issues-when-using-pulumi_optimized_checkpoint_patch-protocol-1165.yaml new file mode 100644 index 000000000000..3b05bfb6c656 --- /dev/null +++ b/changelog/pending/20221215--backend-service--fixes-out-of-memory-issues-when-using-pulumi_optimized_checkpoint_patch-protocol-1165.yaml @@ -0,0 +1,4 @@ +changes: +- type: fix + scope: backend/service + description: "Fixes out-of-memory issues when using PULUMI_OPTIMIZED_CHECKPOINT_PATCH protocol (#1165)" From f699a0f5d65b24370213a34ef514152b2eb3404a Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Thu, 15 Dec 2022 11:52:46 -0500 Subject: [PATCH 3/6] CHANGELOG --- ...n-using-pulumi_optimized_checkpoint_patch-protocol-1165.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/pending/20221215--backend-service--fixes-out-of-memory-issues-when-using-pulumi_optimized_checkpoint_patch-protocol-1165.yaml b/changelog/pending/20221215--backend-service--fixes-out-of-memory-issues-when-using-pulumi_optimized_checkpoint_patch-protocol-1165.yaml index 3b05bfb6c656..ddffe345d291 100644 --- a/changelog/pending/20221215--backend-service--fixes-out-of-memory-issues-when-using-pulumi_optimized_checkpoint_patch-protocol-1165.yaml +++ b/changelog/pending/20221215--backend-service--fixes-out-of-memory-issues-when-using-pulumi_optimized_checkpoint_patch-protocol-1165.yaml @@ -1,4 +1,4 @@ changes: - type: fix scope: backend/service - description: "Fixes out-of-memory issues when using PULUMI_OPTIMIZED_CHECKPOINT_PATCH protocol (#1165)" + description: "Fixes out-of-memory issues when using PULUMI_OPTIMIZED_CHECKPOINT_PATCH protocol" From 7897d349a6aa8ca643db83eb615973e43ae76ef7 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Thu, 15 Dec 2022 12:08:17 -0500 Subject: [PATCH 4/6] Update pkg/backend/httpstate/client/marshal.go Co-authored-by: Robbie McKinstry --- pkg/backend/httpstate/client/marshal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/backend/httpstate/client/marshal.go b/pkg/backend/httpstate/client/marshal.go index 39a9a240287a..56666520c8c7 100644 --- a/pkg/backend/httpstate/client/marshal.go +++ b/pkg/backend/httpstate/client/marshal.go @@ -36,7 +36,7 @@ var jsonIterConfig = jsoniter.Config{SortMapKeys: true}.Froze() // result in max 8MB memory use by the algorithm. func MarshalUntypedDeployment(deployment *apitype.DeploymentV3) (json.RawMessage, error) { var buf bytes.Buffer - md := &marshalUntypedDeployment{deployment} + var md = &marshalUntypedDeployment{deployment} if err := md.Write(&buf); err != nil { return nil, err } From 44934929d7e32864fe109dd8eac5484dc921b8ac Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Thu, 15 Dec 2022 12:12:32 -0500 Subject: [PATCH 5/6] Feedback --- pkg/backend/httpstate/client/marshal.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/backend/httpstate/client/marshal.go b/pkg/backend/httpstate/client/marshal.go index 56666520c8c7..1217f59a2190 100644 --- a/pkg/backend/httpstate/client/marshal.go +++ b/pkg/backend/httpstate/client/marshal.go @@ -25,6 +25,8 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/common/apitype" ) +const maxNewLines = 1024 + var jsonIterConfig = jsoniter.Config{SortMapKeys: true}.Froze() // Marshals to canonical JSON in the apitype.UntypedDeployment format. @@ -32,8 +34,8 @@ var jsonIterConfig = jsoniter.Config{SortMapKeys: true}.Froze() // Optimized for large checkpoints. // // Injects newlines to allow efficient textual diffs over the JSON. Textual diffs currently use O(N^2) memory in the -// number of newlines, so the injection needs to be conservative. Currently it limits to up to 1024 newlines which would -// result in max 8MB memory use by the algorithm. +// number of newlines, so the injection needs to be conservative. Currently it limits to up to maxNewLines newlines +// which would result in max 8MB memory use by the algorithm. func MarshalUntypedDeployment(deployment *apitype.DeploymentV3) (json.RawMessage, error) { var buf bytes.Buffer var md = &marshalUntypedDeployment{deployment} @@ -62,13 +64,13 @@ func (c *marshalUntypedDeployment) Write(w io.Writer) error { } func (c *marshalUntypedDeployment) writeToStream(stream *jsoniter.Stream) error { - stream.WriteObjectStart() - stream.WriteObjectField("version") + stream.WriteObjectStart() // writes `{` + stream.WriteObjectField("version") // writes `"version":` stream.WriteInt(3) - stream.WriteMore() + stream.WriteMore() // writes `,` stream.WriteObjectField("deployment") c.writeDeploymentV3(stream) - stream.WriteObjectEnd() + stream.WriteObjectEnd() // writes `}` return stream.Flush() } @@ -87,8 +89,8 @@ func (c *marshalUntypedDeployment) writeDeploymentV3(stream *jsoniter.Stream) (e } nResources := len(deployment.Resources) - maxNewlines := 1024 - 2 - newlinePeriod := int(math.Ceil(float64(nResources) / float64(maxNewlines))) + maxNL := maxNewLines - 2 + newlinePeriod := int(math.Ceil(float64(nResources) / float64(maxNL))) if nResources > 0 { stream.WriteMore() @@ -97,7 +99,7 @@ func (c *marshalUntypedDeployment) writeDeploymentV3(stream *jsoniter.Stream) (e for i, r := range deployment.Resources { if i > 0 { stream.WriteRaw(",") - if (nResources <= maxNewlines) || (i%newlinePeriod == 0) { + if (nResources <= maxNL) || (i%newlinePeriod == 0) { stream.WriteRaw("\n") } } From 45681576f70bfb8f63faeb949490eca40c476221 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Thu, 15 Dec 2022 12:14:51 -0500 Subject: [PATCH 6/6] Lint --- pkg/backend/httpstate/client/marshal.go | 5 ++++- pkg/backend/httpstate/snapshot_test.go | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/backend/httpstate/client/marshal.go b/pkg/backend/httpstate/client/marshal.go index 1217f59a2190..99af9784b51d 100644 --- a/pkg/backend/httpstate/client/marshal.go +++ b/pkg/backend/httpstate/client/marshal.go @@ -69,7 +69,10 @@ func (c *marshalUntypedDeployment) writeToStream(stream *jsoniter.Stream) error stream.WriteInt(3) stream.WriteMore() // writes `,` stream.WriteObjectField("deployment") - c.writeDeploymentV3(stream) + err := c.writeDeploymentV3(stream) + if err != nil { + return err + } stream.WriteObjectEnd() // writes `}` return stream.Flush() } diff --git a/pkg/backend/httpstate/snapshot_test.go b/pkg/backend/httpstate/snapshot_test.go index fd36b8ec47cf..eb26462128d7 100644 --- a/pkg/backend/httpstate/snapshot_test.go +++ b/pkg/backend/httpstate/snapshot_test.go @@ -54,7 +54,7 @@ func TestCloudSnapshotPersisterUseOfDiffProtocol(t *testing.T) { t.Cleanup(func() { bytes, err := json.MarshalIndent(expectations, "", " ") require.NoError(t, err) - err = os.WriteFile(expectationsFile, bytes, 0700) + err = os.WriteFile(expectationsFile, bytes, 0600) require.NoError(t, err) }) } else {