From 9aa30d9046938fdd3a0ca8ac5342088d82857b3f Mon Sep 17 00:00:00 2001 From: Fraser Waters Date: Tue, 6 Dec 2022 14:40:37 +0000 Subject: [PATCH] Add Output.JsonSerialize to dotnet sdk Plan is to add functions like this to _all_ the SDKs. JsonSerialization is _very_ language specific, dotnet for example uses System.Text.Json, go would use JsonMarshal, etc. So it's worth having it built into SDKs and then exposed as a PCL intrinsic (with the caveat that the cross-language result will be _valid_ JSON, but with no commmitment to formatting for example). This is just the first part of this work, to add it to the dotnet SDK (simply because I know that best). --- sdk/Pulumi.Tests/Core/OutputTests.cs | 135 +++++++++++++++++++++++++ sdk/Pulumi/Core/Output.cs | 143 ++++++++++++++++++++++++++- 2 files changed, 277 insertions(+), 1 deletion(-) diff --git a/sdk/Pulumi.Tests/Core/OutputTests.cs b/sdk/Pulumi.Tests/Core/OutputTests.cs index cba9f7ce..5de66721 100644 --- a/sdk/Pulumi.Tests/Core/OutputTests.cs +++ b/sdk/Pulumi.Tests/Core/OutputTests.cs @@ -1,5 +1,6 @@ // Copyright 2016-2019, Pulumi Corporation +using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; using System.Threading.Tasks; @@ -9,12 +10,30 @@ namespace Pulumi.Tests.Core { + // Simple struct used for JSON tests + public struct TestStructure { + public int X { get; set;} + + private int y; + + public string Z => (y+1).ToString(); + + public TestStructure(int x, int y) { + X = x; + this.y = y; + } + } + public class OutputTests : PulumiTest { private static Output CreateOutput(T value, bool isKnown, bool isSecret = false) => new Output(Task.FromResult(OutputData.Create( ImmutableHashSet.Empty, value, isKnown, isSecret))); + private static Output CreateOutput(IEnumerable resources, T value, bool isKnown, bool isSecret = false) + => new Output(Task.FromResult(OutputData.Create( + ImmutableHashSet.CreateRange(resources), value, isKnown, isSecret))); + public class PreviewTests { [Fact] @@ -618,6 +637,122 @@ public Task CreateSecretSetsSecret() Assert.True(data.IsSecret); Assert.Equal(0, data.Value); }); + + [Fact] + public Task JsonSerializeBasic() + => RunInNormal(async () => + { + var o1 = CreateOutput(new int[]{ 0, 1} , true); + var o2 = Output.JsonSerialize(o1); + var data = await o2.DataTask.ConfigureAwait(false); + Assert.True(data.IsKnown); + Assert.False(data.IsSecret); + Assert.Equal("[0,1]", data.Value); + }); + + [Fact] + public Task JsonSerializeNested() + => RunInNormal(async () => + { + var o1 = CreateOutput(new Output[] { + CreateOutput(0, true), + CreateOutput(1, true), + }, true); + var o2 = Output.JsonSerialize(o1); + var data = await o2.DataTask.ConfigureAwait(false); + Assert.True(data.IsKnown); + Assert.False(data.IsSecret); + Assert.Equal("[0,1]", data.Value); + }); + + [Fact] + public Task JsonSerializeNestedUnknown() + => RunInNormal(async () => + { + var o1 = CreateOutput(new Output[] { + CreateOutput(default, false), + CreateOutput(1, true), + }, true); + var o2 = Output.JsonSerialize(o1); + var data = await o2.DataTask.ConfigureAwait(false); + Assert.False(data.IsKnown); + Assert.False(data.IsSecret); + }); + + [Fact] + public Task JsonSerializeNestedSecret() + => RunInNormal(async () => + { + var o1 = CreateOutput(new Output[] { + CreateOutput(0, true, true), + CreateOutput(1, true), + }, true); + var o2 = Output.JsonSerialize(o1); + var data = await o2.DataTask.ConfigureAwait(false); + Assert.True(data.IsKnown); + Assert.True(data.IsSecret); + Assert.Equal("[0,1]", data.Value); + }); + + [Fact] + public Task JsonSerializeWithOptions() + => RunInNormal(async () => + { + var v = new System.Collections.Generic.Dictionary(); + v.Add("a", new TestStructure(1, 2)); + v.Add("b", new TestStructure(int.MinValue, int.MaxValue)); + var o1 = CreateOutput(v, true); + var options = new System.Text.Json.JsonSerializerOptions(); + options.WriteIndented = true; + var o2 = Output.JsonSerialize(o1, options); + var data = await o2.DataTask.ConfigureAwait(false); + Assert.True(data.IsKnown); + Assert.False(data.IsSecret); + var expected = @"{ + ""a"": { + ""X"": 1, + ""Z"": ""3"" + }, + ""b"": { + ""X"": -2147483648, + ""Z"": ""-2147483648"" + } +}"; + Assert.Equal(expected, data.Value); + }); + + [Fact] + public async Task JsonSerializeNestedDependencies() { + // We need a custom mock setup for this because new CustomResource will call into the + // deployment to try and register. + var runner = new Moq.Mock(Moq.MockBehavior.Strict); + runner.Setup(r => r.RegisterTask(Moq.It.IsAny(), Moq.It.IsAny())); + + var logger = new Moq.Mock(Moq.MockBehavior.Strict); + logger.Setup(l => l.DebugAsync(Moq.It.IsAny(), Moq.It.IsAny(), Moq.It.IsAny(), Moq.It.IsAny())).Returns(Task.CompletedTask); + + var mock = new Moq.Mock(Moq.MockBehavior.Strict); + mock.Setup(d => d.IsDryRun).Returns(false); + mock.Setup(d => d.Stack).Returns(() => null!); + mock.Setup(d => d.Runner).Returns(runner.Object); + mock.Setup(d => d.Logger).Returns(logger.Object); + mock.Setup(d => d.ReadOrRegisterResource(Moq.It.IsAny(), Moq.It.IsAny(), Moq.It.IsAny>(), Moq.It.IsAny(), Moq.It.IsAny())); + + Deployment.Instance = new DeploymentInstance(mock.Object); + + var resource = new CustomResource("type", "name", null); + + var o1 = CreateOutput(new Output[] { + CreateOutput(new Resource[] { resource}, 0, true, true), + CreateOutput(1, true), + }, true); + var o2 = Output.JsonSerialize(o1); + var data = await o2.DataTask.ConfigureAwait(false); + Assert.True(data.IsKnown); + Assert.True(data.IsSecret); + Assert.Contains(resource, data.Resources); + Assert.Equal("[0,1]", data.Value); + } } } } diff --git a/sdk/Pulumi/Core/Output.cs b/sdk/Pulumi/Core/Output.cs index 4523a7fa..6e7c5cc6 100644 --- a/sdk/Pulumi/Core/Output.cs +++ b/sdk/Pulumi/Core/Output.cs @@ -5,11 +5,88 @@ using System.Collections.Immutable; using System.Diagnostics; using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; using System.Threading.Tasks; using Pulumi.Serialization; namespace Pulumi { + /// + /// Internal class used for Output.JsonSerialize. + /// + sealed class OutputJsonConverter : System.Text.Json.Serialization.JsonConverterFactory + { + private sealed class OutputJsonConverterInner : System.Text.Json.Serialization.JsonConverter> + { + readonly OutputJsonConverter Parent; + readonly JsonConverter Converter; + + public OutputJsonConverterInner(OutputJsonConverter parent, JsonSerializerOptions options) { + Parent = parent; + Converter = (JsonConverter)options.GetConverter(typeof(T)); + } + + public override Output Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + throw new NotImplementedException("JsonSerialize only supports writing to JSON"); + } + + public override void Write(Utf8JsonWriter writer, Output value, JsonSerializerOptions options) + { + // Sadly we have to block here as converters aren't async + var result = value.DataTask.Result; + // Add the seen dependencies to the resources set + Parent.Resources.AddRange(result.Resources); + if (!result.IsKnown) + { + // If the result isn't known we can just write a null and flag the parent to reject this whole serialization + writer.WriteNullValue(); + Parent.SeenUnknown = true; + } + else + { + // The result is known we can just serialize the inner value, but flag the parent if we've seen a secret + Converter.Write(writer, result.Value, options); + Parent.SeenSecret |= result.IsSecret; + } + } + } + + public bool SeenUnknown {get; private set;} + public bool SeenSecret {get; private set;} + public ImmutableHashSet SeenResources => Resources.ToImmutableHashSet(); + private readonly HashSet Resources; + + public OutputJsonConverter() + { + Resources = new HashSet(); + } + + public override bool CanConvert(Type typeToConvert) + { + if (typeToConvert.IsGenericType) + { + var genericType = typeToConvert.GetGenericTypeDefinition(); + return genericType == typeof(Output<>); + } + return false; + } + + public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) + { + Type elementType = typeToConvert.GetGenericArguments()[0]; + JsonConverter converter = (JsonConverter)Activator.CreateInstance( + typeof(OutputJsonConverterInner<>).MakeGenericType( + new Type[] { elementType }), + System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public, + binder: null, + args: new object[] { this, options }, + culture: null)!; + return converter; + } + } + /// /// Useful static utility methods for both creating and working with s. /// @@ -106,6 +183,70 @@ public static Output Format(FormattableString formattableString) internal static Output> Concat(Output> values1, Output> values2) => Tuple(values1, values2).Apply(tuple => tuple.Item1.AddRange(tuple.Item2)); + + /// + /// Uses to serialize the given value into a JSON string. + /// + public static Output JsonSerialize(Output value, System.Text.Json.JsonSerializerOptions? options = null) + { + if (value == null) { + throw new ArgumentNullException("value"); + } + + async Task> GetData() + { + var result = await value.DataTask; + + if (!result.IsKnown) { + return new OutputData(result.Resources, "", false, result.IsSecret); + } + + var utf8 = new System.IO.MemoryStream(); + // This needs to handle nested potentially secret and unknown Output values, we do this by + // hooking options to handle any seen Output values. + + // TODO: This can be simplified in net6.0 to just new System.Text.Json.JsonSerializerOptions(options); + var internalOptions = new System.Text.Json.JsonSerializerOptions(); + internalOptions.AllowTrailingCommas = options?.AllowTrailingCommas ?? internalOptions.AllowTrailingCommas; + if (options != null) + { + foreach(var converter in options.Converters) + { + internalOptions.Converters.Add(converter); + } + } + internalOptions.DefaultBufferSize = options?.DefaultBufferSize ?? internalOptions.DefaultBufferSize; + internalOptions.DictionaryKeyPolicy = options?.DictionaryKeyPolicy ?? internalOptions.DictionaryKeyPolicy; + internalOptions.Encoder = options?.Encoder ?? internalOptions.Encoder; + internalOptions.IgnoreNullValues = options?.IgnoreNullValues ?? internalOptions.IgnoreNullValues; + internalOptions.IgnoreReadOnlyProperties = options?.IgnoreReadOnlyProperties ?? internalOptions.IgnoreReadOnlyProperties; + internalOptions.MaxDepth = options?.MaxDepth ?? internalOptions.MaxDepth; + internalOptions.PropertyNameCaseInsensitive = options?.PropertyNameCaseInsensitive ?? internalOptions.PropertyNameCaseInsensitive; + internalOptions.PropertyNamingPolicy = options?.PropertyNamingPolicy ?? internalOptions.PropertyNamingPolicy; + internalOptions.ReadCommentHandling = options?.ReadCommentHandling ?? internalOptions.ReadCommentHandling; + internalOptions.WriteIndented = options?.WriteIndented ?? internalOptions.WriteIndented; + + // Add the magic converter to allow us to do nested outputs + var outputConverter = new OutputJsonConverter(); + internalOptions.Converters.Add(outputConverter); + + await System.Text.Json.JsonSerializer.SerializeAsync(utf8, result.Value, internalOptions); + + // Check if the result is valid or not, that is if we saw any nulls we can just throw away the json string made and return unknown + if (outputConverter.SeenUnknown) { + return new OutputData(result.Resources.Union(outputConverter.SeenResources), "", false, result.IsSecret | outputConverter.SeenSecret); + } + + // GetBuffer returns the entire byte array backing the MemoryStream, wrapping a span of the + // correct length around that rather than just calling ToArray() saves an array copy. + var json = System.Text.Encoding.UTF8.GetString(new ReadOnlySpan(utf8.GetBuffer(), 0, (int)utf8.Length)); + + return new OutputData(result.Resources.Union(outputConverter.SeenResources), json, true, result.IsSecret | outputConverter.SeenSecret); + } + + return new Output(GetData()); + } } /// @@ -128,7 +269,7 @@ internal interface IOutput /// s are a key part of how Pulumi tracks dependencies between s. Because the values of outputs are not available until resources are /// created, these are represented using the special s type, which - /// internally represents two things: an eventually available value of the output and + /// internally represents two things: an eventually available value of the output and /// the dependency on the source(s) of the output value. /// In fact, s is quite similar to . /// Additionally, they carry along dependency information.