Merge pull request #591 from golang/master-merge

Merge remote-tracking branch 'origin/dev' into master

.gitignore

@@ -12,5 +12,6 @@ core
 _obj
 _test
 _testmain.go
-protoc-gen-go/testdata/multi/*.pb.go
-_conformance/_conformance
+
+# Conformance test output and transient files.
+conformance/failing_tests.txt

.travis.yml

@@ -2,17 +2,29 @@ sudo: false
 language: go
 go:
 - 1.6.x
-- 1.7.x
-- 1.8.x
-- 1.9.x
+- 1.10.x
+- 1.x
 
 install:
   - go get -v -d -t github.com/golang/protobuf/...
-  - curl -L https://github.com/google/protobuf/releases/download/v3.3.0/protoc-3.3.0-linux-x86_64.zip -o /tmp/protoc.zip
-  - unzip /tmp/protoc.zip -d $HOME/protoc
+  - curl -L https://github.com/google/protobuf/releases/download/v3.5.1/protoc-3.5.1-linux-x86_64.zip -o /tmp/protoc.zip
+  - unzip /tmp/protoc.zip -d "$HOME"/protoc
+  - mkdir -p "$HOME"/src && ln -s "$HOME"/protoc "$HOME"/src/protobuf
 
 env:
   - PATH=$HOME/protoc/bin:$PATH
 
 script:
-  - make all test
+  - make all
+  - make regenerate
+  # TODO(tamird): When https://github.com/travis-ci/gimme/pull/130 is
+  # released, make this look for "1.x".
+  - if [[ "$TRAVIS_GO_VERSION" == 1.10* ]]; then
+      if [[ "$(git status --porcelain 2>&1)" != "" ]]; then
+        git status >&2;
+        git diff -a >&2;
+        exit 1;
+      fi;
+      echo "git status is clean.";
+    fi;
+  - make test

Makefile

@@ -29,16 +29,14 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-
 all:	install
 
 install:
-	go install ./proto ./jsonpb ./ptypes
-	go install ./protoc-gen-go
+	go install ./proto ./jsonpb ./ptypes ./protoc-gen-go
 
 test:
-	go test ./proto ./jsonpb ./ptypes
-	make -C protoc-gen-go/testdata test
+	go test ./... ./protoc-gen-go/testdata
+	make -C conformance test
 
 clean:
 	go clean ./...
@@ -47,9 +45,4 @@ nuke:
 	go clean -i ./...
 
 regenerate:
-	make -C protoc-gen-go/descriptor regenerate
-	make -C protoc-gen-go/plugin regenerate
-	make -C protoc-gen-go/testdata regenerate
-	make -C proto/testdata regenerate
-	make -C jsonpb/jsonpb_test_proto regenerate
-	make -C _conformance regenerate
+	./regenerate.sh

README.md

@@ -7,7 +7,7 @@ Google's data interchange format.
 Copyright 2010 The Go Authors.
 https://github.com/golang/protobuf
 
-This package and the code it generates requires at least Go 1.4.
+This package and the code it generates requires at least Go 1.6.
 
 This software implements Go bindings for protocol buffers.  For
 information about protocol buffers themselves, see
@@ -56,13 +56,49 @@ parameter set to the directory you want to output the Go code to.
 The generated files will be suffixed .pb.go.  See the Test code below
 for an example using such a file.
 
+## Packages and input paths ##
+
+The protocol buffer language has a concept of "packages" which does not
+correspond well to the Go notion of packages. In generated Go code,
+each source `.proto` file is associated with a single Go package. The
+name and import path for this package is specified with the `go_package`
+proto option:
+
+	option go_package = "github.com/golang/protobuf/ptypes/any";
+
+The protocol buffer compiler will attempt to derive a package name and
+import path if a `go_package` option is not present, but it is
+best to always specify one explicitly.
+
+There is a one-to-one relationship between source `.proto` files and
+generated `.pb.go` files, but any number of `.pb.go` files may be
+contained in the same Go package.
+
+The output name of a generated file is produced by replacing the
+`.proto` suffix with `.pb.go` (e.g., `foo.proto` produces `foo.pb.go`).
+However, the output directory is selected in one of two ways.  Let
+us say we have `inputs/x.proto` with a `go_package` option of
+`github.com/golang/protobuf/p`. The corresponding output file may
+be:
+
+- Relative to the import path:
+
+	protoc --go_out=. inputs/x.proto
+	# writes ./github.com/golang/protobuf/p/x.pb.go
+
+  (This can work well with `--go_out=$GOPATH`.)
+
+- Relative to the input file:
+
+	protoc --go_out=paths=source_relative:. inputs/x.proto
+	# generate ./inputs/x.pb.go
+
+## Generated code ##
 
 The package comment for the proto library contains text describing
 the interface provided in Go for protocol buffers. Here is an edited
 version.
 
-==========
-
 The proto package converts data structures to and from the
 wire format of protocol buffers.  It works in concert with the
 Go source code generated for .proto files by the protocol compiler.
@@ -114,9 +150,9 @@ Consider file test.proto, containing
 ```proto
 	syntax = "proto2";
 	package example;
-	
+
 	enum FOO { X = 17; };
-	
+
 	message Test {
 	  required string label = 1;
 	  optional int32 type = 2 [default=77];
@@ -170,22 +206,25 @@ To create and play with a Test object from the example package,
 To pass extra parameters to the plugin, use a comma-separated
 parameter list separated from the output directory by a colon:
 
-
 	protoc --go_out=plugins=grpc,import_path=mypackage:. *.proto
 
-
-- `import_prefix=xxx` - a prefix that is added onto the beginning of
-  all imports. Useful for things like generating protos in a
-  subdirectory, or regenerating vendored protobufs in-place.
-- `import_path=foo/bar` - used as the package if no input files
-  declare `go_package`. If it contains slashes, everything up to the
-  rightmost slash is ignored.
+- `paths=(import | source_relative)` - specifies how the paths of
+  generated files are structured. See the "Packages and imports paths"
+  section above. The default is `import`.
 - `plugins=plugin1+plugin2` - specifies the list of sub-plugins to
   load. The only plugin in this repo is `grpc`.
 - `Mfoo/bar.proto=quux/shme` - declares that foo/bar.proto is
   associated with Go package quux/shme.  This is subject to the
   import_prefix parameter.
 
+The following parameters are deprecated and should not be used:
+
+- `import_prefix=xxx` - a prefix that is added onto the beginning of
+  all imports.
+- `import_path=foo/bar` - used as the package if no input files
+  declare `go_package`. If it contains slashes, everything up to the
+  rightmost slash is ignored.
+
 ## gRPC Support ##
 
 If a proto file specifies RPC services, protoc-gen-go can be instructed to

_conformance/Makefile → conformance/Makefile

@@ -29,5 +29,21 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-regenerate:
-	protoc --go_out=Mgoogle/protobuf/any.proto=github.com/golang/protobuf/ptypes/any,Mgoogle/protobuf/duration.proto=github.com/golang/protobuf/ptypes/duration,Mgoogle/protobuf/struct.proto=github.com/golang/protobuf/ptypes/struct,Mgoogle/protobuf/timestamp.proto=github.com/golang/protobuf/ptypes/timestamp,Mgoogle/protobuf/wrappers.proto=github.com/golang/protobuf/ptypes/wrappers,Mgoogle/protobuf/field_mask.proto=google.golang.org/genproto/protobuf:. conformance_proto/conformance.proto
+PROTOBUF_ROOT=$(HOME)/src/protobuf
+
+all:
+	@echo To run the tests in this directory, acquire the main protobuf
+	@echo distribution from:
+	@echo
+	@echo '   https://github.com/google/protobuf'
+	@echo
+	@echo Build the test runner with:
+	@echo
+	@echo '   cd conformance && make conformance-test-runner'
+	@echo
+	@echo And run the tests in this directory with:
+	@echo
+	@echo '   make test PROTOBUF_ROOT=<protobuf distribution>'
+
+test:
+	./test.sh $(PROTOBUF_ROOT)

_conformance/conformance.go → conformance/conformance.go

@@ -39,7 +39,7 @@ import (
 	"io"
 	"os"
 
-	pb "github.com/golang/protobuf/_conformance/conformance_proto"
+	pb "github.com/golang/protobuf/conformance/internal/conformance_proto"
 	"github.com/golang/protobuf/jsonpb"
 	"github.com/golang/protobuf/proto"
 )
@@ -101,13 +101,6 @@ func handle(req *pb.ConformanceRequest) *pb.ConformanceResponse {
 		err = proto.Unmarshal(p.ProtobufPayload, &msg)
 	case *pb.ConformanceRequest_JsonPayload:
 		err = jsonpb.UnmarshalString(p.JsonPayload, &msg)
-		if err != nil && err.Error() == "unmarshaling Any not supported yet" {
-			return &pb.ConformanceResponse{
-				Result: &pb.ConformanceResponse_Skipped{
-					Skipped: err.Error(),
-				},
-			}
-		}
 	default:
 		return &pb.ConformanceResponse{
 			Result: &pb.ConformanceResponse_RuntimeError{

conformance/conformance.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+cd $(dirname $0)
+exec go run conformance.go $*

conformance/failure_list_go.txt

@@ -0,0 +1,61 @@
+# This is the list of conformance tests that are known ot fail right now.
+# TODO: These should be fixed.
+
+DurationProtoInputTooLarge.JsonOutput
+DurationProtoInputTooSmall.JsonOutput
+FieldMaskNumbersDontRoundTrip.JsonOutput
+FieldMaskPathsDontRoundTrip.JsonOutput
+FieldMaskTooManyUnderscore.JsonOutput
+JsonInput.AnyWithFieldMask.JsonOutput
+JsonInput.AnyWithFieldMask.ProtobufOutput
+JsonInput.DoubleFieldQuotedValue.JsonOutput
+JsonInput.DoubleFieldQuotedValue.ProtobufOutput
+JsonInput.DurationHas3FractionalDigits.Validator
+JsonInput.DurationHas6FractionalDigits.Validator
+JsonInput.DurationHas9FractionalDigits.Validator
+JsonInput.DurationHasZeroFractionalDigit.Validator
+JsonInput.DurationMaxValue.JsonOutput
+JsonInput.DurationMaxValue.ProtobufOutput
+JsonInput.DurationMinValue.JsonOutput
+JsonInput.DurationMinValue.ProtobufOutput
+JsonInput.EnumFieldUnknownValue.Validator
+JsonInput.FieldMask.JsonOutput
+JsonInput.FieldMask.ProtobufOutput
+JsonInput.FieldNameInLowerCamelCase.Validator
+JsonInput.FieldNameWithMixedCases.JsonOutput
+JsonInput.FieldNameWithMixedCases.ProtobufOutput
+JsonInput.FieldNameWithMixedCases.Validator
+JsonInput.FieldNameWithNumbers.Validator
+JsonInput.FloatFieldQuotedValue.JsonOutput
+JsonInput.FloatFieldQuotedValue.ProtobufOutput
+JsonInput.Int32FieldExponentialFormat.JsonOutput
+JsonInput.Int32FieldExponentialFormat.ProtobufOutput
+JsonInput.Int32FieldFloatTrailingZero.JsonOutput
+JsonInput.Int32FieldFloatTrailingZero.ProtobufOutput
+JsonInput.Int32FieldMaxFloatValue.JsonOutput
+JsonInput.Int32FieldMaxFloatValue.ProtobufOutput
+JsonInput.Int32FieldMinFloatValue.JsonOutput
+JsonInput.Int32FieldMinFloatValue.ProtobufOutput
+JsonInput.Int32FieldStringValue.JsonOutput
+JsonInput.Int32FieldStringValue.ProtobufOutput
+JsonInput.Int32FieldStringValueEscaped.JsonOutput
+JsonInput.Int32FieldStringValueEscaped.ProtobufOutput
+JsonInput.Int64FieldBeString.Validator
+JsonInput.MapFieldValueIsNull
+JsonInput.OneofFieldDuplicate
+JsonInput.RepeatedFieldMessageElementIsNull
+JsonInput.RepeatedFieldPrimitiveElementIsNull
+JsonInput.StringFieldSurrogateInWrongOrder
+JsonInput.StringFieldUnpairedHighSurrogate
+JsonInput.StringFieldUnpairedLowSurrogate
+JsonInput.TimestampHas3FractionalDigits.Validator
+JsonInput.TimestampHas6FractionalDigits.Validator
+JsonInput.TimestampHas9FractionalDigits.Validator
+JsonInput.TimestampHasZeroFractionalDigit.Validator
+JsonInput.TimestampJsonInputTooSmall
+JsonInput.TimestampZeroNormalized.Validator
+JsonInput.Uint32FieldMaxFloatValue.JsonOutput
+JsonInput.Uint32FieldMaxFloatValue.ProtobufOutput
+JsonInput.Uint64FieldBeString.Validator
+TimestampProtoInputTooLarge.JsonOutput
+TimestampProtoInputTooSmall.JsonOutput