Skip to content

Commit

Permalink
Merge pull request #7576 from jtattermusch/protobuf_buffer_serialization
Browse files Browse the repository at this point in the history 
New Span-based serialization logic (followup for #7351)
  • Loading branch information
@jtattermusch
jtattermusch committed Jun 29, 2020
2 parents 093faeb + 4f0afc7 commit 206b973
Show file tree
Hide file tree
Showing 80 changed files with 11,022 additions and 628 deletions.
8 changes: 8 additions & 0 deletions Makefile.am
View file
Expand Up @@ -96,11 +96,14 @@ csharp_EXTRA_DIST= \
csharp/src/Google.Protobuf.Benchmarks/Program.cs \
csharp/src/Google.Protobuf.Benchmarks/wrapper_benchmark_messages.proto \
csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmarkMessages.cs \
csharp/src/Google.Protobuf.Benchmarks/WriteMessagesBenchmark.cs \
csharp/src/Google.Protobuf.Benchmarks/WriteRawPrimitivesBenchmark.cs \
csharp/src/Google.Protobuf.Conformance/Conformance.cs \
csharp/src/Google.Protobuf.Conformance/Google.Protobuf.Conformance.csproj \
csharp/src/Google.Protobuf.Conformance/Program.cs \
csharp/src/Google.Protobuf.JsonDump/Google.Protobuf.JsonDump.csproj \
csharp/src/Google.Protobuf.JsonDump/Program.cs \
csharp/src/Google.Protobuf.Test/Buffers/ArrayBufferWriter.cs \
csharp/src/Google.Protobuf.Test/ByteStringTest.cs \
csharp/src/Google.Protobuf.Test/CodedInputStreamExtensions.cs \
csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs \
Expand Down Expand Up @@ -259,6 +262,11 @@ csharp_EXTRA_DIST= \
csharp/src/Google.Protobuf/WellKnownTypes/Wrappers.cs \
csharp/src/Google.Protobuf/WellKnownTypes/WrappersPartial.cs \
csharp/src/Google.Protobuf/WireFormat.cs \
csharp/src/Google.Protobuf/WritingPrimitivesMessages.cs \
csharp/src/Google.Protobuf/WritingPrimitives.cs \
csharp/src/Google.Protobuf/WriterInternalState.cs \
csharp/src/Google.Protobuf/WriteContext.cs \
csharp/src/Google.Protobuf/WriteBufferHelper.cs \
csharp/src/Google.Protobuf/UnknownField.cs \
csharp/src/Google.Protobuf/UnknownFieldSet.cs

Expand Down
64 changes: 32 additions & 32 deletions csharp/compatibility_tests/v3.0.0/src/Google.Protobuf.Test/CodedOutputStreamTest.cs
View file
Expand Up @@ -211,62 +211,62 @@ public void WriteWholeMessage_VaryingBlockSizes()
[Test]
public void EncodeZigZag32()
{
Assert.AreEqual(0u, CodedOutputStream.EncodeZigZag32(0));
Assert.AreEqual(1u, CodedOutputStream.EncodeZigZag32(-1));
Assert.AreEqual(2u, CodedOutputStream.EncodeZigZag32(1));
Assert.AreEqual(3u, CodedOutputStream.EncodeZigZag32(-2));
Assert.AreEqual(0x7FFFFFFEu, CodedOutputStream.EncodeZigZag32(0x3FFFFFFF));
Assert.AreEqual(0x7FFFFFFFu, CodedOutputStream.EncodeZigZag32(unchecked((int) 0xC0000000)));
Assert.AreEqual(0xFFFFFFFEu, CodedOutputStream.EncodeZigZag32(0x7FFFFFFF));
Assert.AreEqual(0xFFFFFFFFu, CodedOutputStream.EncodeZigZag32(unchecked((int) 0x80000000)));
Assert.AreEqual(0u, WritingPrimitives.EncodeZigZag32(0));
Assert.AreEqual(1u, WritingPrimitives.EncodeZigZag32(-1));
Assert.AreEqual(2u, WritingPrimitives.EncodeZigZag32(1));
Assert.AreEqual(3u, WritingPrimitives.EncodeZigZag32(-2));
Assert.AreEqual(0x7FFFFFFEu, WritingPrimitives.EncodeZigZag32(0x3FFFFFFF));
Assert.AreEqual(0x7FFFFFFFu, WritingPrimitives.EncodeZigZag32(unchecked((int) 0xC0000000)));
Assert.AreEqual(0xFFFFFFFEu, WritingPrimitives.EncodeZigZag32(0x7FFFFFFF));
Assert.AreEqual(0xFFFFFFFFu, WritingPrimitives.EncodeZigZag32(unchecked((int) 0x80000000)));
}

[Test]
public void EncodeZigZag64()
{
Assert.AreEqual(0u, CodedOutputStream.EncodeZigZag64(0));
Assert.AreEqual(1u, CodedOutputStream.EncodeZigZag64(-1));
Assert.AreEqual(2u, CodedOutputStream.EncodeZigZag64(1));
Assert.AreEqual(3u, CodedOutputStream.EncodeZigZag64(-2));
Assert.AreEqual(0u, WritingPrimitives.EncodeZigZag64(0));
Assert.AreEqual(1u, WritingPrimitives.EncodeZigZag64(-1));
Assert.AreEqual(2u, WritingPrimitives.EncodeZigZag64(1));
Assert.AreEqual(3u, WritingPrimitives.EncodeZigZag64(-2));
Assert.AreEqual(0x000000007FFFFFFEuL,
CodedOutputStream.EncodeZigZag64(unchecked((long) 0x000000003FFFFFFFUL)));
WritingPrimitives.EncodeZigZag64(unchecked((long) 0x000000003FFFFFFFUL)));
Assert.AreEqual(0x000000007FFFFFFFuL,
CodedOutputStream.EncodeZigZag64(unchecked((long) 0xFFFFFFFFC0000000UL)));
WritingPrimitives.EncodeZigZag64(unchecked((long) 0xFFFFFFFFC0000000UL)));
Assert.AreEqual(0x00000000FFFFFFFEuL,
CodedOutputStream.EncodeZigZag64(unchecked((long) 0x000000007FFFFFFFUL)));
WritingPrimitives.EncodeZigZag64(unchecked((long) 0x000000007FFFFFFFUL)));
Assert.AreEqual(0x00000000FFFFFFFFuL,
CodedOutputStream.EncodeZigZag64(unchecked((long) 0xFFFFFFFF80000000UL)));
WritingPrimitives.EncodeZigZag64(unchecked((long) 0xFFFFFFFF80000000UL)));
Assert.AreEqual(0xFFFFFFFFFFFFFFFEL,
CodedOutputStream.EncodeZigZag64(unchecked((long) 0x7FFFFFFFFFFFFFFFUL)));
WritingPrimitives.EncodeZigZag64(unchecked((long) 0x7FFFFFFFFFFFFFFFUL)));
Assert.AreEqual(0xFFFFFFFFFFFFFFFFL,
CodedOutputStream.EncodeZigZag64(unchecked((long) 0x8000000000000000UL)));
WritingPrimitives.EncodeZigZag64(unchecked((long) 0x8000000000000000UL)));
}

[Test]
public void RoundTripZigZag32()
{
// Some easier-to-verify round-trip tests. The inputs (other than 0, 1, -1)
// were chosen semi-randomly via keyboard bashing.
Assert.AreEqual(0, ParsingPrimitives.DecodeZigZag32(CodedOutputStream.EncodeZigZag32(0)));
Assert.AreEqual(1, ParsingPrimitives.DecodeZigZag32(CodedOutputStream.EncodeZigZag32(1)));
Assert.AreEqual(-1, ParsingPrimitives.DecodeZigZag32(CodedOutputStream.EncodeZigZag32(-1)));
Assert.AreEqual(14927, ParsingPrimitives.DecodeZigZag32(CodedOutputStream.EncodeZigZag32(14927)));
Assert.AreEqual(-3612, ParsingPrimitives.DecodeZigZag32(CodedOutputStream.EncodeZigZag32(-3612)));
Assert.AreEqual(0, ParsingPrimitives.DecodeZigZag32(WritingPrimitives.EncodeZigZag32(0)));
Assert.AreEqual(1, ParsingPrimitives.DecodeZigZag32(WritingPrimitives.EncodeZigZag32(1)));
Assert.AreEqual(-1, ParsingPrimitives.DecodeZigZag32(WritingPrimitives.EncodeZigZag32(-1)));
Assert.AreEqual(14927, ParsingPrimitives.DecodeZigZag32(WritingPrimitives.EncodeZigZag32(14927)));
Assert.AreEqual(-3612, ParsingPrimitives.DecodeZigZag32(WritingPrimitives.EncodeZigZag32(-3612)));
}

[Test]
public void RoundTripZigZag64()
{
Assert.AreEqual(0, ParsingPrimitives.DecodeZigZag64(CodedOutputStream.EncodeZigZag64(0)));
Assert.AreEqual(1, ParsingPrimitives.DecodeZigZag64(CodedOutputStream.EncodeZigZag64(1)));
Assert.AreEqual(-1, ParsingPrimitives.DecodeZigZag64(CodedOutputStream.EncodeZigZag64(-1)));
Assert.AreEqual(14927, ParsingPrimitives.DecodeZigZag64(CodedOutputStream.EncodeZigZag64(14927)));
Assert.AreEqual(-3612, ParsingPrimitives.DecodeZigZag64(CodedOutputStream.EncodeZigZag64(-3612)));
Assert.AreEqual(0, ParsingPrimitives.DecodeZigZag64(WritingPrimitives.EncodeZigZag64(0)));
Assert.AreEqual(1, ParsingPrimitives.DecodeZigZag64(WritingPrimitives.EncodeZigZag64(1)));
Assert.AreEqual(-1, ParsingPrimitives.DecodeZigZag64(WritingPrimitives.EncodeZigZag64(-1)));
Assert.AreEqual(14927, ParsingPrimitives.DecodeZigZag64(WritingPrimitives.EncodeZigZag64(14927)));
Assert.AreEqual(-3612, ParsingPrimitives.DecodeZigZag64(WritingPrimitives.EncodeZigZag64(-3612)));

Assert.AreEqual(856912304801416L,
ParsingPrimitives.DecodeZigZag64(CodedOutputStream.EncodeZigZag64(856912304801416L)));
ParsingPrimitives.DecodeZigZag64(WritingPrimitives.EncodeZigZag64(856912304801416L)));
Assert.AreEqual(-75123905439571256L,
ParsingPrimitives.DecodeZigZag64(CodedOutputStream.EncodeZigZag64(-75123905439571256L)));
ParsingPrimitives.DecodeZigZag64(WritingPrimitives.EncodeZigZag64(-75123905439571256L)));
}

[Test]
Expand Down Expand Up @@ -395,7 +395,7 @@ public void Dispose_DisposesUnderlyingStream()
Assert.IsTrue(memoryStream.CanWrite);
using (var cos = new CodedOutputStream(memoryStream))
{
cos.WriteRawByte(0);
cos.WriteRawBytes(new byte[] {0});
Assert.AreEqual(0, memoryStream.Position); // Not flushed yet
}
Assert.AreEqual(1, memoryStream.ToArray().Length); // Flushed data from CodedOutputStream to MemoryStream
Expand All @@ -409,7 +409,7 @@ public void Dispose_WithLeaveOpen()
Assert.IsTrue(memoryStream.CanWrite);
using (var cos = new CodedOutputStream(memoryStream, true))
{
cos.WriteRawByte(0);
cos.WriteRawBytes(new byte[] {0});
Assert.AreEqual(0, memoryStream.Position); // Not flushed yet
}
Assert.AreEqual(1, memoryStream.Position); // Flushed data from CodedOutputStream to MemoryStream
Expand Down
22 changes: 20 additions & 2 deletions csharp/compatibility_tests/v3.0.0/src/Google.Protobuf.Test/FieldCodecTest.cs
View file
Expand Up @@ -124,7 +124,16 @@ public void TestRoundTripRaw()
{
var stream = new MemoryStream();
var codedOutput = new CodedOutputStream(stream);
codec.ValueWriter(codedOutput, sampleValue);
WriteContext.Initialize(codedOutput, out WriteContext ctx);
try
{
// only write the value using the codec
codec.ValueWriter(ref ctx, sampleValue);
}
finally
{
ctx.CopyStateTo(codedOutput);
}
codedOutput.Flush();
stream.Position = 0;
var codedInput = new CodedInputStream(stream);
Expand Down Expand Up @@ -172,7 +181,16 @@ public void TestDefaultValue()
if (codec.DefaultValue != null) // This part isn't appropriate for message types.
{
codedOutput = new CodedOutputStream(stream);
codec.ValueWriter(codedOutput, codec.DefaultValue);
WriteContext.Initialize(codedOutput, out WriteContext ctx);
try
{
// only write the value using the codec
codec.ValueWriter(ref ctx, codec.DefaultValue);
}
finally
{
ctx.CopyStateTo(codedOutput);
}
codedOutput.Flush();
Assert.AreNotEqual(0, stream.Position);
Assert.AreEqual(stream.Position, codec.ValueSizeCalculator(codec.DefaultValue));
Expand Down
65 changes: 65 additions & 0 deletions csharp/src/AddressBook/Addressbook.cs
View file
Expand Up @@ -190,6 +190,9 @@ public sealed partial class Person : pb::IMessage<Person>

[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
public void WriteTo(pb::CodedOutputStream output) {
#if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE
output.WriteRawMessage(this);
#else
if (Name.Length != 0) {
output.WriteRawTag(10);
output.WriteString(Name);
Expand All @@ -210,8 +213,35 @@ public sealed partial class Person : pb::IMessage<Person>
if (_unknownFields != null) {
_unknownFields.WriteTo(output);
}
#endif
}

#if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) {
if (Name.Length != 0) {
output.WriteRawTag(10);
output.WriteString(Name);
}
if (Id != 0) {
output.WriteRawTag(16);
output.WriteInt32(Id);
}
if (Email.Length != 0) {
output.WriteRawTag(26);
output.WriteString(Email);
}
phones_.WriteTo(ref output, _repeated_phones_codec);
if (lastUpdated_ != null) {
output.WriteRawTag(42);
output.WriteMessage(LastUpdated);
}
if (_unknownFields != null) {
_unknownFields.WriteTo(ref output);
}
}
#endif

[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
public int CalculateSize() {
int size = 0;
Expand Down Expand Up @@ -441,6 +471,9 @@ public sealed partial class PhoneNumber : pb::IMessage<PhoneNumber>

[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
public void WriteTo(pb::CodedOutputStream output) {
#if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE
output.WriteRawMessage(this);
#else
if (Number.Length != 0) {
output.WriteRawTag(10);
output.WriteString(Number);
Expand All @@ -452,8 +485,26 @@ public sealed partial class PhoneNumber : pb::IMessage<PhoneNumber>
if (_unknownFields != null) {
_unknownFields.WriteTo(output);
}
#endif
}

#if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) {
if (Number.Length != 0) {
output.WriteRawTag(10);
output.WriteString(Number);
}
if (Type != global::Google.Protobuf.Examples.AddressBook.Person.Types.PhoneType.Mobile) {
output.WriteRawTag(16);
output.WriteEnum((int) Type);
}
if (_unknownFields != null) {
_unknownFields.WriteTo(ref output);
}
}
#endif

[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
public int CalculateSize() {
int size = 0;
Expand Down Expand Up @@ -621,12 +672,26 @@ public sealed partial class AddressBook : pb::IMessage<AddressBook>

[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
public void WriteTo(pb::CodedOutputStream output) {
#if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE
output.WriteRawMessage(this);
#else
people_.WriteTo(output, _repeated_people_codec);
if (_unknownFields != null) {
_unknownFields.WriteTo(output);
}
#endif
}

#if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) {
people_.WriteTo(ref output, _repeated_people_codec);
if (_unknownFields != null) {
_unknownFields.WriteTo(ref output);
}
}
#endif

[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
public int CalculateSize() {
int size = 0;
Expand Down

0 comments on commit 206b973

Please sign in to comment.