From a2cbd5a8246e7203b27cb938bc170c29e33269ec Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 6 Apr 2020 16:40:50 +0200 Subject: [PATCH 1/4] serialization benchmark improvements --- ...ionConfig.cs => BenchmarkDatasetConfig.cs} | 14 ++++---- ...Benchmark.cs => GoogleMessageBenchmark.cs} | 32 +++++++++++-------- 2 files changed, 25 insertions(+), 21 deletions(-) rename csharp/src/Google.Protobuf.Benchmarks/{SerializationConfig.cs => BenchmarkDatasetConfig.cs} (89%) rename csharp/src/Google.Protobuf.Benchmarks/{SerializationBenchmark.cs => GoogleMessageBenchmark.cs} (74%) diff --git a/csharp/src/Google.Protobuf.Benchmarks/SerializationConfig.cs b/csharp/src/Google.Protobuf.Benchmarks/BenchmarkDatasetConfig.cs similarity index 89% rename from csharp/src/Google.Protobuf.Benchmarks/SerializationConfig.cs rename to csharp/src/Google.Protobuf.Benchmarks/BenchmarkDatasetConfig.cs index 679f16cb9a87..c0754190b6c9 100644 --- a/csharp/src/Google.Protobuf.Benchmarks/SerializationConfig.cs +++ b/csharp/src/Google.Protobuf.Benchmarks/BenchmarkDatasetConfig.cs @@ -43,20 +43,20 @@ namespace Google.Protobuf.Benchmarks /// /// The configuration for a single serialization test, loaded from a dataset. /// - public class SerializationConfig + public class BenchmarkDatasetConfig { private static readonly Dictionary parsersByMessageName = - typeof(SerializationBenchmark).Assembly.GetTypes() + typeof(GoogleMessageBenchmark).Assembly.GetTypes() .Where(t => typeof(IMessage).IsAssignableFrom(t)) .ToDictionary( t => ((MessageDescriptor) t.GetProperty("Descriptor", BindingFlags.Static | BindingFlags.Public).GetValue(null)).FullName, t => ((MessageParser) t.GetProperty("Parser", BindingFlags.Static | BindingFlags.Public).GetValue(null))); public MessageParser Parser { get; } - public IEnumerable Payloads { get; } + public List Payloads { get; } public string Name { get; } - public SerializationConfig(string resource) + public BenchmarkDatasetConfig(string resource, string shortName = null) { var data = LoadData(resource); var dataset = BenchmarkDataset.Parser.ParseFrom(data); @@ -66,13 +66,13 @@ public SerializationConfig(string resource) throw new ArgumentException($"No parser for message {dataset.MessageName} in this assembly"); } Parser = parser; - Payloads = dataset.Payload; - Name = dataset.Name; + Payloads = new List(dataset.Payload.Select(p => p.ToByteArray())); + Name = shortName ?? dataset.Name; } private static byte[] LoadData(string resource) { - using (var stream = typeof(SerializationBenchmark).Assembly.GetManifestResourceStream($"Google.Protobuf.Benchmarks.{resource}")) + using (var stream = typeof(GoogleMessageBenchmark).Assembly.GetManifestResourceStream($"Google.Protobuf.Benchmarks.{resource}")) { if (stream == null) { diff --git a/csharp/src/Google.Protobuf.Benchmarks/SerializationBenchmark.cs b/csharp/src/Google.Protobuf.Benchmarks/GoogleMessageBenchmark.cs similarity index 74% rename from csharp/src/Google.Protobuf.Benchmarks/SerializationBenchmark.cs rename to csharp/src/Google.Protobuf.Benchmarks/GoogleMessageBenchmark.cs index d8c2ec11d73e..132967e00a29 100644 --- a/csharp/src/Google.Protobuf.Benchmarks/SerializationBenchmark.cs +++ b/csharp/src/Google.Protobuf.Benchmarks/GoogleMessageBenchmark.cs @@ -38,23 +38,27 @@ namespace Google.Protobuf.Benchmarks { /// - /// Benchmark for serializing (to a MemoryStream) and deserializing (from a ByteString). + /// Benchmark for serializing and deserializing of standard datasets that are also + /// measured by benchmarks in other languages. /// Over time we may wish to test the various different approaches to serialization and deserialization separately. + /// See https://github.com/protocolbuffers/protobuf/blob/master/benchmarks/README.md + /// See https://github.com/protocolbuffers/protobuf/blob/master/docs/performance.md /// [MemoryDiagnoser] - public class SerializationBenchmark + public class GoogleMessageBenchmark { /// - /// All the configurations to be tested. Add more datasets to the array as they're available. + /// All the datasets to be tested. Add more datasets to the array as they're available. /// (When C# supports proto2, this will increase significantly.) /// - public static SerializationConfig[] Configurations => new[] + public static BenchmarkDatasetConfig[] DatasetConfigurations => new[] { - new SerializationConfig("dataset.google_message1_proto3.pb") + // short name is specified to make results table more readable + new BenchmarkDatasetConfig("dataset.google_message1_proto3.pb", "goog_msg1_proto3") }; - [ParamsSource(nameof(Configurations))] - public SerializationConfig Configuration { get; set; } + [ParamsSource(nameof(DatasetConfigurations))] + public BenchmarkDatasetConfig Dataset { get; set; } private MessageParser parser; /// @@ -67,8 +71,8 @@ public class SerializationBenchmark [GlobalSetup] public void GlobalSetup() { - parser = Configuration.Parser; - subTests = Configuration.Payloads.Select(p => new SubTest(p, parser.ParseFrom(p))).ToList(); + parser = Dataset.Parser; + subTests = Dataset.Payloads.Select(p => new SubTest(p, parser.ParseFrom(p))).ToList(); } [Benchmark] @@ -78,7 +82,7 @@ public void GlobalSetup() public void ToByteArray() => subTests.ForEach(item => item.ToByteArray()); [Benchmark] - public void ParseFromByteString() => subTests.ForEach(item => item.ParseFromByteString(parser)); + public void ParseFromByteArray() => subTests.ForEach(item => item.ParseFromByteArray(parser)); [Benchmark] public void ParseFromStream() => subTests.ForEach(item => item.ParseFromStream(parser)); @@ -87,13 +91,13 @@ private class SubTest { private readonly Stream destinationStream; private readonly Stream sourceStream; - private readonly ByteString data; + private readonly byte[] data; private readonly IMessage message; - public SubTest(ByteString data, IMessage message) + public SubTest(byte[] data, IMessage message) { destinationStream = new MemoryStream(data.Length); - sourceStream = new MemoryStream(data.ToByteArray()); + sourceStream = new MemoryStream(data); this.data = data; this.message = message; } @@ -108,7 +112,7 @@ public void WriteToStream() public void ToByteArray() => message.ToByteArray(); - public void ParseFromByteString(MessageParser parser) => parser.ParseFrom(data); + public void ParseFromByteArray(MessageParser parser) => parser.ParseFrom(data); public void ParseFromStream(MessageParser parser) { From 79cfc73293e3c1d86af8937f3747e1ee1d015873 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 6 Apr 2020 18:24:13 +0200 Subject: [PATCH 2/4] refactor WrapperBechmark --- ...Benchmark.cs => ParseMessagesBenchmark.cs} | 72 +++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-) rename csharp/src/Google.Protobuf.Benchmarks/{WrapperBenchmark.cs => ParseMessagesBenchmark.cs} (59%) diff --git a/csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmark.cs b/csharp/src/Google.Protobuf.Benchmarks/ParseMessagesBenchmark.cs similarity index 59% rename from csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmark.cs rename to csharp/src/Google.Protobuf.Benchmarks/ParseMessagesBenchmark.cs index ae17c1819af0..02310aac7ee8 100644 --- a/csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmark.cs +++ b/csharp/src/Google.Protobuf.Benchmarks/ParseMessagesBenchmark.cs @@ -34,37 +34,87 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Buffers; +using Google.Protobuf.WellKnownTypes; namespace Google.Protobuf.Benchmarks { /// - /// Benchmark that tests serialization/deserialization of wrapper fields. + /// Benchmark that tests parsing performance for various messages. /// [MemoryDiagnoser] - public class WrapperBenchmark + public class ParseMessagesBenchmark { + const int MaxMessages = 100; + byte[] manyWrapperFieldsData; byte[] manyPrimitiveFieldsData; + byte[] manyWrapperFieldsMultipleMessagesData; + byte[] manyPrimitiveFieldsMultipleMessagesData; + + byte[] emptyData = new byte[0]; + + public IEnumerable MessageCountValues => new[] { 10, 100 }; + [GlobalSetup] public void GlobalSetup() { manyWrapperFieldsData = CreateManyWrapperFieldsMessage().ToByteArray(); manyPrimitiveFieldsData = CreateManyPrimitiveFieldsMessage().ToByteArray(); + manyWrapperFieldsMultipleMessagesData = CreateBufferWithMultipleMessages(CreateManyWrapperFieldsMessage(), MaxMessages); + manyPrimitiveFieldsMultipleMessagesData = CreateBufferWithMultipleMessages(CreateManyPrimitiveFieldsMessage(), MaxMessages); } [Benchmark] - public ManyWrapperFieldsMessage ParseWrapperFields() + public ManyWrapperFieldsMessage ManyWrapperFieldsMessage_ParseFromByteArray() { return ManyWrapperFieldsMessage.Parser.ParseFrom(manyWrapperFieldsData); } [Benchmark] - public ManyPrimitiveFieldsMessage ParsePrimitiveFields() + public ManyPrimitiveFieldsMessage ManyPrimitiveFieldsMessage_ParseFromByteArray() { return ManyPrimitiveFieldsMessage.Parser.ParseFrom(manyPrimitiveFieldsData); } + [Benchmark] + public Empty EmptyMessage_ParseFromByteArray() + { + return Empty.Parser.ParseFrom(emptyData); + } + + [Benchmark] + [ArgumentsSource(nameof(MessageCountValues))] + public long ManyWrapperFieldsMessage_ParseDelimitedMessagesFromByteArray(int messageCount) + { + long sum = 0; + var input = new CodedInputStream(manyWrapperFieldsMultipleMessagesData); + for (int i = 0; i < messageCount; i++) + { + var msg = new ManyWrapperFieldsMessage(); + input.ReadMessage(msg); + sum += msg.Int64Field19.Value; + } + return sum; + } + + [Benchmark] + [ArgumentsSource(nameof(MessageCountValues))] + public long ManyPrimitiveFieldsMessage_ParseDelimitedMessagesFromByteArray(int messageCount) + { + long sum = 0; + var input = new CodedInputStream(manyPrimitiveFieldsMultipleMessagesData); + for (int i = 0; i < messageCount; i++) + { + var msg = new ManyPrimitiveFieldsMessage(); + input.ReadMessage(msg); + sum += msg.Int64Field19; + } + return sum; + } + + private static ManyWrapperFieldsMessage CreateManyWrapperFieldsMessage() { // Example data match data of an internal benchmarks @@ -98,5 +148,17 @@ private static ManyPrimitiveFieldsMessage CreateManyPrimitiveFieldsMessage() DoubleField50 = 2.45 }; } + + private static byte[] CreateBufferWithMultipleMessages(IMessage msg, int msgCount) + { + var ms = new MemoryStream(); + var cos = new CodedOutputStream(ms); + for (int i = 0; i < msgCount; i++) + { + cos.WriteMessage(msg); + } + cos.Flush(); + return ms.ToArray(); + } } -} +} \ No newline at end of file From 4116e659843082e2544f83b362b0a16f1470b555 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 7 Apr 2020 11:56:37 +0200 Subject: [PATCH 3/4] improve ParseMessageBenchmark maintainability --- .../ParseMessagesBenchmark.cs | 98 ++++++++++--------- 1 file changed, 52 insertions(+), 46 deletions(-) diff --git a/csharp/src/Google.Protobuf.Benchmarks/ParseMessagesBenchmark.cs b/csharp/src/Google.Protobuf.Benchmarks/ParseMessagesBenchmark.cs index 02310aac7ee8..cbc47328fa1b 100644 --- a/csharp/src/Google.Protobuf.Benchmarks/ParseMessagesBenchmark.cs +++ b/csharp/src/Google.Protobuf.Benchmarks/ParseMessagesBenchmark.cs @@ -31,6 +31,7 @@ #endregion using BenchmarkDotNet.Attributes; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -47,74 +48,49 @@ public class ParseMessagesBenchmark { const int MaxMessages = 100; - byte[] manyWrapperFieldsData; - byte[] manyPrimitiveFieldsData; - - byte[] manyWrapperFieldsMultipleMessagesData; - byte[] manyPrimitiveFieldsMultipleMessagesData; - - byte[] emptyData = new byte[0]; + SubTest manyWrapperFieldsTest = new SubTest(CreateManyWrapperFieldsMessage(), ManyWrapperFieldsMessage.Parser, () => new ManyWrapperFieldsMessage(), MaxMessages); + SubTest manyPrimitiveFieldsTest = new SubTest(CreateManyPrimitiveFieldsMessage(), ManyPrimitiveFieldsMessage.Parser, () => new ManyPrimitiveFieldsMessage(), MaxMessages); + SubTest emptyMessageTest = new SubTest(new Empty(), Empty.Parser, () => new Empty(), MaxMessages); public IEnumerable MessageCountValues => new[] { 10, 100 }; [GlobalSetup] public void GlobalSetup() { - manyWrapperFieldsData = CreateManyWrapperFieldsMessage().ToByteArray(); - manyPrimitiveFieldsData = CreateManyPrimitiveFieldsMessage().ToByteArray(); - manyWrapperFieldsMultipleMessagesData = CreateBufferWithMultipleMessages(CreateManyWrapperFieldsMessage(), MaxMessages); - manyPrimitiveFieldsMultipleMessagesData = CreateBufferWithMultipleMessages(CreateManyPrimitiveFieldsMessage(), MaxMessages); } [Benchmark] - public ManyWrapperFieldsMessage ManyWrapperFieldsMessage_ParseFromByteArray() + public IMessage ManyWrapperFieldsMessage_ParseFromByteArray() { - return ManyWrapperFieldsMessage.Parser.ParseFrom(manyWrapperFieldsData); + return manyWrapperFieldsTest.ParseFromByteArray(); } [Benchmark] - public ManyPrimitiveFieldsMessage ManyPrimitiveFieldsMessage_ParseFromByteArray() + public IMessage ManyPrimitiveFieldsMessage_ParseFromByteArray() { - return ManyPrimitiveFieldsMessage.Parser.ParseFrom(manyPrimitiveFieldsData); + return manyPrimitiveFieldsTest.ParseFromByteArray(); } [Benchmark] - public Empty EmptyMessage_ParseFromByteArray() + public IMessage EmptyMessage_ParseFromByteArray() { - return Empty.Parser.ParseFrom(emptyData); + return emptyMessageTest.ParseFromByteArray(); } [Benchmark] [ArgumentsSource(nameof(MessageCountValues))] - public long ManyWrapperFieldsMessage_ParseDelimitedMessagesFromByteArray(int messageCount) + public void ManyWrapperFieldsMessage_ParseDelimitedMessagesFromByteArray(int messageCount) { - long sum = 0; - var input = new CodedInputStream(manyWrapperFieldsMultipleMessagesData); - for (int i = 0; i < messageCount; i++) - { - var msg = new ManyWrapperFieldsMessage(); - input.ReadMessage(msg); - sum += msg.Int64Field19.Value; - } - return sum; + manyWrapperFieldsTest.ParseDelimitedMessagesFromByteArray(messageCount); } [Benchmark] [ArgumentsSource(nameof(MessageCountValues))] - public long ManyPrimitiveFieldsMessage_ParseDelimitedMessagesFromByteArray(int messageCount) + public void ManyPrimitiveFieldsMessage_ParseDelimitedMessagesFromByteArray(int messageCount) { - long sum = 0; - var input = new CodedInputStream(manyPrimitiveFieldsMultipleMessagesData); - for (int i = 0; i < messageCount; i++) - { - var msg = new ManyPrimitiveFieldsMessage(); - input.ReadMessage(msg); - sum += msg.Int64Field19; - } - return sum; + manyPrimitiveFieldsTest.ParseDelimitedMessagesFromByteArray(messageCount); } - private static ManyWrapperFieldsMessage CreateManyWrapperFieldsMessage() { // Example data match data of an internal benchmarks @@ -149,16 +125,46 @@ private static ManyPrimitiveFieldsMessage CreateManyPrimitiveFieldsMessage() }; } - private static byte[] CreateBufferWithMultipleMessages(IMessage msg, int msgCount) + private class SubTest { - var ms = new MemoryStream(); - var cos = new CodedOutputStream(ms); - for (int i = 0; i < msgCount; i++) + private readonly IMessage message; + private readonly MessageParser parser; + private readonly Func factory; + private readonly byte[] data; + private readonly byte[] multipleMessagesData; + + public SubTest(IMessage message, MessageParser parser, Func factory, int maxMessageCount) + { + this.message = message; + this.parser = parser; + this.factory = factory; + this.data = message.ToByteArray(); + this.multipleMessagesData = CreateBufferWithMultipleMessages(message, maxMessageCount); + } + + public IMessage ParseFromByteArray() => parser.ParseFrom(data); + + public void ParseDelimitedMessagesFromByteArray(int messageCount) + { + var input = new CodedInputStream(multipleMessagesData); + for (int i = 0; i < messageCount; i++) + { + var msg = factory(); + input.ReadMessage(msg); + } + } + + private static byte[] CreateBufferWithMultipleMessages(IMessage msg, int msgCount) { - cos.WriteMessage(msg); + var ms = new MemoryStream(); + var cos = new CodedOutputStream(ms); + for (int i = 0; i < msgCount; i++) + { + cos.WriteMessage(msg); + } + cos.Flush(); + return ms.ToArray(); } - cos.Flush(); - return ms.ToArray(); } } -} \ No newline at end of file +} From c762a3bb2a8ea865df2fd3c8f832a19b67aee4c4 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 7 Apr 2020 12:01:29 +0200 Subject: [PATCH 4/4] update Makefile.am --- Makefile.am | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index 9b0e68bc7678..d01afea52bc9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -85,15 +85,15 @@ csharp_EXTRA_DIST= \ csharp/src/AddressBook/ListPeople.cs \ csharp/src/AddressBook/Program.cs \ csharp/src/AddressBook/SampleUsage.cs \ + csharp/src/Google.Protobuf.Benchmarks/BenchmarkDatasetConfig.cs \ csharp/src/Google.Protobuf.Benchmarks/BenchmarkMessage1Proto3.cs \ csharp/src/Google.Protobuf.Benchmarks/Benchmarks.cs \ csharp/src/Google.Protobuf.Benchmarks/Google.Protobuf.Benchmarks.csproj \ + csharp/src/Google.Protobuf.Benchmarks/GoogleMessageBenchmark.cs \ csharp/src/Google.Protobuf.Benchmarks/ParseRawPrimitivesBenchmark.cs \ + csharp/src/Google.Protobuf.Benchmarks/ParseMessagesBenchmark.cs \ csharp/src/Google.Protobuf.Benchmarks/Program.cs \ - csharp/src/Google.Protobuf.Benchmarks/SerializationBenchmark.cs \ - csharp/src/Google.Protobuf.Benchmarks/SerializationConfig.cs \ csharp/src/Google.Protobuf.Benchmarks/wrapper_benchmark_messages.proto \ - csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmark.cs \ csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmarkMessages.cs \ csharp/src/Google.Protobuf.Conformance/Conformance.cs \ csharp/src/Google.Protobuf.Conformance/Google.Protobuf.Conformance.csproj \