diff --git a/.gitignore b/.gitignore index d38133b822d0..4e52414a2cd0 100644 --- a/.gitignore +++ b/.gitignore @@ -183,7 +183,6 @@ js/testproto_libs2.js /bazel-* # ruby test output -ruby/lib/ ruby/tests/basic_test_pb.rb ruby/tests/basic_test_proto2_pb.rb ruby/tests/generated_code_pb.rb diff --git a/Makefile.am b/Makefile.am index a71f4d117b87..ddbe08b2f051 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1138,6 +1138,7 @@ ruby_EXTRA_DIST= \ ruby/ext/google/protobuf_c/ruby-upb.h \ ruby/ext/google/protobuf_c/wrap_memcpy.c \ ruby/google-protobuf.gemspec \ + ruby/lib/google/protobuf/descriptor_dsl.rb \ ruby/lib/google/protobuf/message_exts.rb \ ruby/lib/google/protobuf/repeated_field.rb \ ruby/lib/google/protobuf/well_known_types.rb \ diff --git a/ruby/Rakefile b/ruby/Rakefile index 221e9b507f52..523d4de458b7 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -8,6 +8,7 @@ spec = Gem::Specification.load("google-protobuf.gemspec") well_known_protos = %w[ google/protobuf/any.proto google/protobuf/api.proto + google/protobuf/descriptor.proto google/protobuf/duration.proto google/protobuf/empty.proto google/protobuf/field_mask.proto diff --git a/ruby/ext/google/protobuf_c/defs.c b/ruby/ext/google/protobuf_c/defs.c index 992f54bc4d61..fd32cce66587 100644 --- a/ruby/ext/google/protobuf_c/defs.c +++ b/ruby/ext/google/protobuf_c/defs.c @@ -36,13 +36,6 @@ #include "message.h" #include "protobuf.h" -static VALUE Builder_build(VALUE _self); - -static VALUE cMessageBuilderContext; -static VALUE cOneofBuilderContext; -static VALUE cEnumBuilderContext; -static VALUE cBuilder; - // ----------------------------------------------------------------------------- // Global map from upb {msg,enum}defs to wrapper Descriptor/EnumDescriptor // instances. @@ -75,167 +68,6 @@ static VALUE rb_str_maybe_null(const char* s) { return rb_str_new2(s); } -// ----------------------------------------------------------------------------- -// Backward compatibility code. -// ----------------------------------------------------------------------------- - -static void rewrite_enum_default(const upb_symtab* symtab, - google_protobuf_FileDescriptorProto* file, - google_protobuf_FieldDescriptorProto* field) { - upb_strview defaultval; - const char *type_name_str; - char *end; - long val; - const upb_enumdef *e; - upb_strview type_name; - - /* Look for TYPE_ENUM fields that have a default. */ - if (google_protobuf_FieldDescriptorProto_type(field) != - google_protobuf_FieldDescriptorProto_TYPE_ENUM || - !google_protobuf_FieldDescriptorProto_has_default_value(field) || - !google_protobuf_FieldDescriptorProto_has_type_name(field)) { - return; - } - - defaultval = google_protobuf_FieldDescriptorProto_default_value(field); - type_name = google_protobuf_FieldDescriptorProto_type_name(field); - - if (defaultval.size == 0 || !isdigit(defaultval.data[0])) { - return; - } - - if (type_name.size == 0 || type_name.data[0] != '.') { - return; - } - - type_name_str = type_name.data + 1; - - errno = 0; - val = strtol(defaultval.data, &end, 10); - - if (errno != 0 || *end != 0 || val < INT32_MIN || val > INT32_MAX) { - return; - } - - /* Now find the corresponding enum definition. */ - e = upb_symtab_lookupenum(symtab, type_name_str); - if (e) { - /* Look in previously loaded files. */ - const char *label = upb_enumdef_iton(e, val); - if (!label) { - return; - } - google_protobuf_FieldDescriptorProto_set_default_value( - field, upb_strview_makez(label)); - } else { - /* Look in enums defined in this file. */ - const google_protobuf_EnumDescriptorProto* matching_enum = NULL; - size_t i, n; - const google_protobuf_EnumDescriptorProto* const* enums = - google_protobuf_FileDescriptorProto_enum_type(file, &n); - const google_protobuf_EnumValueDescriptorProto* const* values; - - for (i = 0; i < n; i++) { - if (upb_strview_eql(google_protobuf_EnumDescriptorProto_name(enums[i]), - upb_strview_makez(type_name_str))) { - matching_enum = enums[i]; - break; - } - } - - if (!matching_enum) { - return; - } - - values = google_protobuf_EnumDescriptorProto_value(matching_enum, &n); - for (i = 0; i < n; i++) { - if (google_protobuf_EnumValueDescriptorProto_number(values[i]) == val) { - google_protobuf_FieldDescriptorProto_set_default_value( - field, google_protobuf_EnumValueDescriptorProto_name(values[i])); - return; - } - } - - /* We failed to find an enum default. But we'll just leave the enum - * untouched and let the normal def-building code catch it. */ - } -} - -/* Historically we allowed enum defaults to be specified as a number. In - * retrospect this was a mistake as descriptors require defaults to be - * specified as a label. This can make a difference if multiple labels have the - * same number. - * - * Here we do a pass over all enum defaults and rewrite numeric defaults by - * looking up their labels. This is complicated by the fact that the enum - * definition can live in either the symtab or the file_proto. - * */ -static void rewrite_enum_defaults( - const upb_symtab* symtab, google_protobuf_FileDescriptorProto* file_proto) { - size_t i, n; - google_protobuf_DescriptorProto** msgs = - google_protobuf_FileDescriptorProto_mutable_message_type(file_proto, &n); - - for (i = 0; i < n; i++) { - size_t j, m; - google_protobuf_FieldDescriptorProto** fields = - google_protobuf_DescriptorProto_mutable_field(msgs[i], &m); - for (j = 0; j < m; j++) { - rewrite_enum_default(symtab, file_proto, fields[j]); - } - } -} - -static void remove_path(upb_strview *name) { - const char* last = strrchr(name->data, '.'); - if (last) { - size_t remove = last - name->data + 1; - name->data += remove; - name->size -= remove; - } -} - -static void rewrite_nesting(VALUE msg_ent, google_protobuf_DescriptorProto* msg, - google_protobuf_DescriptorProto* const* msgs, - google_protobuf_EnumDescriptorProto* const* enums, - upb_arena *arena) { - VALUE submsgs = rb_hash_aref(msg_ent, ID2SYM(rb_intern("msgs"))); - VALUE enum_pos = rb_hash_aref(msg_ent, ID2SYM(rb_intern("enums"))); - int submsg_count; - int enum_count; - int i; - google_protobuf_DescriptorProto** msg_msgs; - google_protobuf_EnumDescriptorProto** msg_enums; - - Check_Type(submsgs, T_ARRAY); - Check_Type(enum_pos, T_ARRAY); - - submsg_count = RARRAY_LEN(submsgs); - enum_count = RARRAY_LEN(enum_pos); - - msg_msgs = google_protobuf_DescriptorProto_resize_nested_type( - msg, submsg_count, arena); - msg_enums = - google_protobuf_DescriptorProto_resize_enum_type(msg, enum_count, arena); - - for (i = 0; i < submsg_count; i++) { - VALUE submsg_ent = RARRAY_PTR(submsgs)[i]; - VALUE pos = rb_hash_aref(submsg_ent, ID2SYM(rb_intern("pos"))); - upb_strview name; - - msg_msgs[i] = msgs[NUM2INT(pos)]; - name = google_protobuf_DescriptorProto_name(msg_msgs[i]); - remove_path(&name); - google_protobuf_DescriptorProto_set_name(msg_msgs[i], name); - rewrite_nesting(submsg_ent, msg_msgs[i], msgs, enums, arena); - } - - for (i = 0; i < enum_count; i++) { - VALUE pos = RARRAY_PTR(enum_pos)[i]; - msg_enums[i] = enums[NUM2INT(pos)]; - } -} - // ----------------------------------------------------------------------------- // DescriptorPool. // ----------------------------------------------------------------------------- @@ -302,20 +134,32 @@ static VALUE DescriptorPool_alloc(VALUE klass) { /* * call-seq: - * DescriptorPool.build(&block) + * DescriptorPool.add_serialized_file(serialized_file_proto) * - * Invokes the block with a Builder instance as self. All message and enum types - * added within the block are committed to the pool atomically, and may refer - * (co)recursively to each other. The user should call Builder#add_message and - * Builder#add_enum within the block as appropriate. This is the recommended, - * idiomatic way to define new message and enum types. + * Adds the given serialized FileDescriptorProto to the pool. */ -static VALUE DescriptorPool_build(int argc, VALUE* argv, VALUE _self) { - VALUE ctx = rb_class_new_instance(1, &_self, cBuilder); - VALUE block = rb_block_proc(); - rb_funcall_with_block(ctx, rb_intern("instance_eval"), 0, NULL, block); - Builder_build(ctx); - return Qnil; +VALUE DescriptorPool_add_serialized_file(VALUE _self, + VALUE serialized_file_proto) { + DescriptorPool* self = ruby_to_DescriptorPool(_self); + Check_Type(serialized_file_proto, T_STRING); + VALUE arena_rb = Arena_new(); + upb_arena *arena = Arena_get(arena_rb); + google_protobuf_FileDescriptorProto* file_proto = + google_protobuf_FileDescriptorProto_parse( + RSTRING_PTR(serialized_file_proto), + RSTRING_LEN(serialized_file_proto), arena); + if (!file_proto) { + rb_raise(rb_eArgError, "Unable to parse FileDescriptorProto"); + } + upb_status status; + upb_status_clear(&status); + const upb_filedef* filedef = + upb_symtab_addfile(self->symtab, file_proto, &status); + if (!filedef) { + rb_raise(cTypeError, "Unable to build file to DescriptorPool: %s", + upb_status_errmsg(&status)); + } + return get_filedef_obj(_self, filedef); } /* @@ -361,7 +205,8 @@ static void DescriptorPool_register(VALUE module) { VALUE klass = rb_define_class_under( module, "DescriptorPool", rb_cObject); rb_define_alloc_func(klass, DescriptorPool_alloc); - rb_define_method(klass, "build", DescriptorPool_build, -1); + rb_define_method(klass, "add_serialized_file", + DescriptorPool_add_serialized_file, 1); rb_define_method(klass, "lookup", DescriptorPool_lookup, 1); rb_define_singleton_method(klass, "generated_pool", DescriptorPool_generated_pool, 0); @@ -773,41 +618,6 @@ upb_fieldtype_t ruby_to_fieldtype(VALUE type) { return 0; } -static upb_descriptortype_t ruby_to_descriptortype(VALUE type) { - if (TYPE(type) != T_SYMBOL) { - rb_raise(rb_eArgError, "Expected symbol for field type."); - } - -#define CONVERT(upb, ruby) \ - if (SYM2ID(type) == rb_intern( # ruby )) { \ - return UPB_DESCRIPTOR_TYPE_ ## upb; \ - } - - CONVERT(FLOAT, float); - CONVERT(DOUBLE, double); - CONVERT(BOOL, bool); - CONVERT(STRING, string); - CONVERT(BYTES, bytes); - CONVERT(MESSAGE, message); - CONVERT(GROUP, group); - CONVERT(ENUM, enum); - CONVERT(INT32, int32); - CONVERT(INT64, int64); - CONVERT(UINT32, uint32); - CONVERT(UINT64, uint64); - CONVERT(SINT32, sint32); - CONVERT(SINT64, sint64); - CONVERT(FIXED32, fixed32); - CONVERT(FIXED64, fixed64); - CONVERT(SFIXED32, sfixed32); - CONVERT(SFIXED64, sfixed64); - -#undef CONVERT - - rb_raise(rb_eArgError, "Unknown field type."); - return 0; -} - static VALUE descriptortype_to_ruby(upb_descriptortype_t type) { switch (type) { #define CONVERT(upb, ruby) \ @@ -1353,1111 +1163,6 @@ static void EnumDescriptor_register(VALUE module) { cEnumDescriptor = klass; } -// ----------------------------------------------------------------------------- -// FileBuilderContext. -// ----------------------------------------------------------------------------- - -typedef struct { - upb_arena *arena; - google_protobuf_FileDescriptorProto* file_proto; - VALUE descriptor_pool; -} FileBuilderContext; - -static VALUE cFileBuilderContext = Qnil; - -static void FileBuilderContext_mark(void* _self) { - FileBuilderContext* self = _self; - rb_gc_mark(self->descriptor_pool); -} - -static void FileBuilderContext_free(void* _self) { - FileBuilderContext* self = _self; - upb_arena_free(self->arena); - xfree(self); -} - -static const rb_data_type_t FileBuilderContext_type = { - "Google::Protobuf::Internal::FileBuilderContext", - {FileBuilderContext_mark, FileBuilderContext_free, NULL}, - .flags = RUBY_TYPED_FREE_IMMEDIATELY, -}; - -static FileBuilderContext* ruby_to_FileBuilderContext(VALUE val) { - FileBuilderContext* ret; - TypedData_Get_Struct(val, FileBuilderContext, &FileBuilderContext_type, ret); - return ret; -} - -static upb_strview FileBuilderContext_strdup2(VALUE _self, const char *str) { - FileBuilderContext* self = ruby_to_FileBuilderContext(_self); - upb_strview ret; - char *data; - - ret.size = strlen(str); - data = upb_malloc(upb_arena_alloc(self->arena), ret.size + 1); - ret.data = data; - memcpy(data, str, ret.size); - /* Null-terminate required by rewrite_enum_defaults() above. */ - data[ret.size] = '\0'; - return ret; -} - -static upb_strview FileBuilderContext_strdup(VALUE _self, VALUE rb_str) { - return FileBuilderContext_strdup2(_self, get_str(rb_str)); -} - -static upb_strview FileBuilderContext_strdup_sym(VALUE _self, VALUE rb_sym) { - Check_Type(rb_sym, T_SYMBOL); - return FileBuilderContext_strdup(_self, rb_id2str(SYM2ID(rb_sym))); -} - -static VALUE FileBuilderContext_alloc(VALUE klass) { - FileBuilderContext* self = ALLOC(FileBuilderContext); - VALUE ret = TypedData_Wrap_Struct(klass, &FileBuilderContext_type, self); - self->arena = upb_arena_new(); - self->file_proto = google_protobuf_FileDescriptorProto_new(self->arena); - self->descriptor_pool = Qnil; - return ret; -} - -/* - * call-seq: - * FileBuilderContext.new(descriptor_pool) => context - * - * Create a new file builder context for the given file descriptor and - * builder context. This class is intended to serve as a DSL context to be used - * with #instance_eval. - */ -static VALUE FileBuilderContext_initialize(VALUE _self, VALUE descriptor_pool, - VALUE name, VALUE options) { - FileBuilderContext* self = ruby_to_FileBuilderContext(_self); - self->descriptor_pool = descriptor_pool; - - google_protobuf_FileDescriptorProto_set_name( - self->file_proto, FileBuilderContext_strdup(_self, name)); - - // Default syntax for Ruby is proto3. - google_protobuf_FileDescriptorProto_set_syntax( - self->file_proto, - FileBuilderContext_strdup(_self, rb_str_new2("proto3"))); - - if (options != Qnil) { - VALUE syntax; - - Check_Type(options, T_HASH); - syntax = rb_hash_lookup2(options, ID2SYM(rb_intern("syntax")), Qnil); - - if (syntax != Qnil) { - VALUE syntax_str; - - Check_Type(syntax, T_SYMBOL); - syntax_str = rb_id2str(SYM2ID(syntax)); - google_protobuf_FileDescriptorProto_set_syntax( - self->file_proto, FileBuilderContext_strdup(_self, syntax_str)); - } - } - - return Qnil; -} - -static void MessageBuilderContext_add_synthetic_oneofs(VALUE _self); - -/* - * call-seq: - * FileBuilderContext.add_message(name, &block) - * - * Creates a new, empty descriptor with the given name, and invokes the block in - * the context of a MessageBuilderContext on that descriptor. The block can then - * call, e.g., MessageBuilderContext#optional and MessageBuilderContext#repeated - * methods to define the message fields. - * - * This is the recommended, idiomatic way to build message definitions. - */ -static VALUE FileBuilderContext_add_message(VALUE _self, VALUE name) { - VALUE args[2] = { _self, name }; - VALUE ctx = rb_class_new_instance(2, args, cMessageBuilderContext); - VALUE block = rb_block_proc(); - rb_funcall_with_block(ctx, rb_intern("instance_eval"), 0, NULL, block); - MessageBuilderContext_add_synthetic_oneofs(ctx); - return Qnil; -} - -/* We have to do some relatively complicated logic here for backward - * compatibility. - * - * In descriptor.proto, messages are nested inside other messages if that is - * what the original .proto file looks like. For example, suppose we have this - * foo.proto: - * - * package foo; - * message Bar { - * message Baz {} - * } - * - * The descriptor for this must look like this: - * - * file { - * name: "test.proto" - * package: "foo" - * message_type { - * name: "Bar" - * nested_type { - * name: "Baz" - * } - * } - * } - * - * However, the Ruby generated code has always generated messages in a flat, - * non-nested way: - * - * Google::Protobuf::DescriptorPool.generated_pool.build do - * add_message "foo.Bar" do - * end - * add_message "foo.Bar.Baz" do - * end - * end - * - * Here we need to do a translation where we turn this generated code into the - * above descriptor. We need to infer that "foo" is the package name, and not - * a message itself. - * - * We delegate to Ruby to compute the transformation, for more concice and - * readable code than we can do in C */ -static void rewrite_names(VALUE _file_builder, - google_protobuf_FileDescriptorProto* file_proto) { - FileBuilderContext* file_builder = ruby_to_FileBuilderContext(_file_builder); - upb_arena *arena = file_builder->arena; - // Build params (package, msg_names, enum_names). - VALUE package = Qnil; - VALUE msg_names = rb_ary_new(); - VALUE enum_names = rb_ary_new(); - size_t msg_count, enum_count, i; - VALUE new_package, nesting, msg_ents, enum_ents; - google_protobuf_DescriptorProto** msgs; - google_protobuf_EnumDescriptorProto** enums; - - if (google_protobuf_FileDescriptorProto_has_package(file_proto)) { - upb_strview package_str = - google_protobuf_FileDescriptorProto_package(file_proto); - package = rb_str_new(package_str.data, package_str.size); - } - - msgs = google_protobuf_FileDescriptorProto_mutable_message_type(file_proto, - &msg_count); - for (i = 0; i < msg_count; i++) { - upb_strview name = google_protobuf_DescriptorProto_name(msgs[i]); - rb_ary_push(msg_names, rb_str_new(name.data, name.size)); - } - - enums = google_protobuf_FileDescriptorProto_mutable_enum_type(file_proto, - &enum_count); - for (i = 0; i < enum_count; i++) { - upb_strview name = google_protobuf_EnumDescriptorProto_name(enums[i]); - rb_ary_push(enum_names, rb_str_new(name.data, name.size)); - } - - { - // Call Ruby code to calculate package name and nesting. - VALUE args[3] = { package, msg_names, enum_names }; - VALUE internal = rb_eval_string("Google::Protobuf::Internal"); - VALUE ret = rb_funcallv(internal, rb_intern("fixup_descriptor"), 3, args); - - new_package = rb_ary_entry(ret, 0); - nesting = rb_ary_entry(ret, 1); - } - - // Rewrite package and names. - if (new_package != Qnil) { - upb_strview new_package_str = - FileBuilderContext_strdup(_file_builder, new_package); - google_protobuf_FileDescriptorProto_set_package(file_proto, - new_package_str); - } - - for (i = 0; i < msg_count; i++) { - upb_strview name = google_protobuf_DescriptorProto_name(msgs[i]); - remove_path(&name); - google_protobuf_DescriptorProto_set_name(msgs[i], name); - } - - for (i = 0; i < enum_count; i++) { - upb_strview name = google_protobuf_EnumDescriptorProto_name(enums[i]); - remove_path(&name); - google_protobuf_EnumDescriptorProto_set_name(enums[i], name); - } - - // Rewrite nesting. - msg_ents = rb_hash_aref(nesting, ID2SYM(rb_intern("msgs"))); - enum_ents = rb_hash_aref(nesting, ID2SYM(rb_intern("enums"))); - - Check_Type(msg_ents, T_ARRAY); - Check_Type(enum_ents, T_ARRAY); - - for (i = 0; i < (size_t)RARRAY_LEN(msg_ents); i++) { - VALUE msg_ent = rb_ary_entry(msg_ents, i); - VALUE pos = rb_hash_aref(msg_ent, ID2SYM(rb_intern("pos"))); - msgs[i] = msgs[NUM2INT(pos)]; - rewrite_nesting(msg_ent, msgs[i], msgs, enums, arena); - } - - for (i = 0; i < (size_t)RARRAY_LEN(enum_ents); i++) { - VALUE enum_pos = rb_ary_entry(enum_ents, i); - enums[i] = enums[NUM2INT(enum_pos)]; - } - - google_protobuf_FileDescriptorProto_resize_message_type( - file_proto, RARRAY_LEN(msg_ents), arena); - google_protobuf_FileDescriptorProto_resize_enum_type( - file_proto, RARRAY_LEN(enum_ents), arena); -} - -/* - * call-seq: - * FileBuilderContext.add_enum(name, &block) - * - * Creates a new, empty enum descriptor with the given name, and invokes the - * block in the context of an EnumBuilderContext on that descriptor. The block - * can then call EnumBuilderContext#add_value to define the enum values. - * - * This is the recommended, idiomatic way to build enum definitions. - */ -static VALUE FileBuilderContext_add_enum(VALUE _self, VALUE name) { - VALUE args[2] = { _self, name }; - VALUE ctx = rb_class_new_instance(2, args, cEnumBuilderContext); - VALUE block = rb_block_proc(); - rb_funcall_with_block(ctx, rb_intern("instance_eval"), 0, NULL, block); - return Qnil; -} - -static void FileBuilderContext_build(VALUE _self) { - FileBuilderContext* self = ruby_to_FileBuilderContext(_self); - DescriptorPool* pool = ruby_to_DescriptorPool(self->descriptor_pool); - upb_status status; - - rewrite_enum_defaults(pool->symtab, self->file_proto); - rewrite_names(_self, self->file_proto); - - upb_status_clear(&status); - if (!upb_symtab_addfile(pool->symtab, self->file_proto, &status)) { - rb_raise(cTypeError, "Unable to add defs to DescriptorPool: %s", - upb_status_errmsg(&status)); - } -} - -static void FileBuilderContext_register(VALUE module) { - VALUE klass = rb_define_class_under(module, "FileBuilderContext", rb_cObject); - rb_define_alloc_func(klass, FileBuilderContext_alloc); - rb_define_method(klass, "initialize", FileBuilderContext_initialize, 3); - rb_define_method(klass, "add_message", FileBuilderContext_add_message, 1); - rb_define_method(klass, "add_enum", FileBuilderContext_add_enum, 1); - rb_gc_register_address(&cFileBuilderContext); - cFileBuilderContext = klass; -} - -// ----------------------------------------------------------------------------- -// MessageBuilderContext. -// ----------------------------------------------------------------------------- - -typedef struct { - google_protobuf_DescriptorProto* msg_proto; - VALUE file_builder; -} MessageBuilderContext; - -static VALUE cMessageBuilderContext = Qnil; - -static void MessageBuilderContext_mark(void* _self) { - MessageBuilderContext* self = _self; - rb_gc_mark(self->file_builder); -} - -static const rb_data_type_t MessageBuilderContext_type = { - "Google::Protobuf::Internal::MessageBuilderContext", - {MessageBuilderContext_mark, RUBY_DEFAULT_FREE, NULL}, - .flags = RUBY_TYPED_FREE_IMMEDIATELY, -}; - -static MessageBuilderContext* ruby_to_MessageBuilderContext(VALUE val) { - MessageBuilderContext* ret; - TypedData_Get_Struct(val, MessageBuilderContext, &MessageBuilderContext_type, - ret); - return ret; -} - -static VALUE MessageBuilderContext_alloc(VALUE klass) { - MessageBuilderContext* self = ALLOC(MessageBuilderContext); - VALUE ret = TypedData_Wrap_Struct(klass, &MessageBuilderContext_type, self); - self->file_builder = Qnil; - return ret; -} - -/* - * call-seq: - * MessageBuilderContext.new(file_builder, name) => context - * - * Create a new message builder context around the given message descriptor and - * builder context. This class is intended to serve as a DSL context to be used - * with #instance_eval. - */ -static VALUE MessageBuilderContext_initialize(VALUE _self, VALUE _file_builder, - VALUE name) { - MessageBuilderContext* self = ruby_to_MessageBuilderContext(_self); - FileBuilderContext* file_builder = ruby_to_FileBuilderContext(_file_builder); - google_protobuf_FileDescriptorProto* file_proto = file_builder->file_proto; - - self->file_builder = _file_builder; - self->msg_proto = google_protobuf_FileDescriptorProto_add_message_type( - file_proto, file_builder->arena); - - google_protobuf_DescriptorProto_set_name( - self->msg_proto, FileBuilderContext_strdup(_file_builder, name)); - - return Qnil; -} - -static void msgdef_add_field(VALUE msgbuilder_rb, upb_label_t label, VALUE name, - VALUE type, VALUE number, VALUE type_class, - VALUE options, int oneof_index, - bool proto3_optional) { - MessageBuilderContext* self = ruby_to_MessageBuilderContext(msgbuilder_rb); - FileBuilderContext* file_context = - ruby_to_FileBuilderContext(self->file_builder); - google_protobuf_FieldDescriptorProto* field_proto; - VALUE name_str; - - field_proto = google_protobuf_DescriptorProto_add_field(self->msg_proto, - file_context->arena); - - Check_Type(name, T_SYMBOL); - name_str = rb_id2str(SYM2ID(name)); - - google_protobuf_FieldDescriptorProto_set_name( - field_proto, FileBuilderContext_strdup(self->file_builder, name_str)); - google_protobuf_FieldDescriptorProto_set_number(field_proto, NUM2INT(number)); - google_protobuf_FieldDescriptorProto_set_label(field_proto, (int)label); - google_protobuf_FieldDescriptorProto_set_type( - field_proto, (int)ruby_to_descriptortype(type)); - - if (proto3_optional) { - google_protobuf_FieldDescriptorProto_set_proto3_optional(field_proto, true); - } - - if (type_class != Qnil) { - Check_Type(type_class, T_STRING); - - // Make it an absolute type name by prepending a dot. - type_class = rb_str_append(rb_str_new2("."), type_class); - google_protobuf_FieldDescriptorProto_set_type_name( - field_proto, FileBuilderContext_strdup(self->file_builder, type_class)); - } - - if (options != Qnil) { - Check_Type(options, T_HASH); - - if (rb_funcall(options, rb_intern("key?"), 1, - ID2SYM(rb_intern("default"))) == Qtrue) { - VALUE default_value = - rb_hash_lookup(options, ID2SYM(rb_intern("default"))); - - /* Call #to_s since all defaults are strings in the descriptor. */ - default_value = rb_funcall(default_value, rb_intern("to_s"), 0); - - google_protobuf_FieldDescriptorProto_set_default_value( - field_proto, - FileBuilderContext_strdup(self->file_builder, default_value)); - } - - if (rb_funcall(options, rb_intern("key?"), 1, - ID2SYM(rb_intern("json_name"))) == Qtrue) { - VALUE json_name = - rb_hash_lookup(options, ID2SYM(rb_intern("json_name"))); - - google_protobuf_FieldDescriptorProto_set_json_name( - field_proto, - FileBuilderContext_strdup(self->file_builder, json_name)); - } - } - - if (oneof_index >= 0) { - google_protobuf_FieldDescriptorProto_set_oneof_index(field_proto, - oneof_index); - } -} - -#if RUBY_API_VERSION_CODE >= 20700 -static VALUE make_mapentry(VALUE _message_builder, VALUE types, int argc, - const VALUE* argv, VALUE blockarg) { - (void)blockarg; -#else -static VALUE make_mapentry(VALUE _message_builder, VALUE types, int argc, - VALUE* argv) { -#endif - MessageBuilderContext* message_builder = - ruby_to_MessageBuilderContext(_message_builder); - VALUE type_class = rb_ary_entry(types, 2); - FileBuilderContext* file_context = - ruby_to_FileBuilderContext(message_builder->file_builder); - google_protobuf_MessageOptions* options = - google_protobuf_DescriptorProto_mutable_options( - message_builder->msg_proto, file_context->arena); - - google_protobuf_MessageOptions_set_map_entry(options, true); - - // optional key = 1; - rb_funcall(_message_builder, rb_intern("optional"), 3, - ID2SYM(rb_intern("key")), rb_ary_entry(types, 0), INT2NUM(1)); - - // optional value = 2; - if (type_class == Qnil) { - rb_funcall(_message_builder, rb_intern("optional"), 3, - ID2SYM(rb_intern("value")), rb_ary_entry(types, 1), INT2NUM(2)); - } else { - rb_funcall(_message_builder, rb_intern("optional"), 4, - ID2SYM(rb_intern("value")), rb_ary_entry(types, 1), INT2NUM(2), - type_class); - } - - return Qnil; -} - -/* - * call-seq: - * MessageBuilderContext.optional(name, type, number, type_class = nil, - * options = nil) - * - * Defines a new optional field on this message type with the given type, tag - * number, and type class (for message and enum fields). The type must be a Ruby - * symbol (as accepted by FieldDescriptor#type=) and the type_class must be a - * string, if present (as accepted by FieldDescriptor#submsg_name=). - */ -VALUE MessageBuilderContext_optional(int argc, VALUE* argv, VALUE _self) { - VALUE name, type, number; - VALUE type_class, options = Qnil; - - rb_scan_args(argc, argv, "32", &name, &type, &number, &type_class, &options); - - // Allow passing (name, type, number, options) or - // (name, type, number, type_class, options) - if (argc == 4 && RB_TYPE_P(type_class, T_HASH)) { - options = type_class; - type_class = Qnil; - } - - msgdef_add_field(_self, UPB_LABEL_OPTIONAL, name, type, number, type_class, - options, -1, false); - - return Qnil; -} - -/* - * call-seq: - * MessageBuilderContext.proto3_optional(name, type, number, - * type_class = nil, options = nil) - * - * Defines a true proto3 optional field (that tracks presence) on this message - * type with the given type, tag number, and type class (for message and enum - * fields). The type must be a Ruby symbol (as accepted by - * FieldDescriptor#type=) and the type_class must be a string, if present (as - * accepted by FieldDescriptor#submsg_name=). - */ -static VALUE MessageBuilderContext_proto3_optional(int argc, VALUE* argv, - VALUE _self) { - VALUE name, type, number; - VALUE type_class, options = Qnil; - - rb_scan_args(argc, argv, "32", &name, &type, &number, &type_class, &options); - - // Allow passing (name, type, number, options) or - // (name, type, number, type_class, options) - if (argc == 4 && RB_TYPE_P(type_class, T_HASH)) { - options = type_class; - type_class = Qnil; - } - - msgdef_add_field(_self, UPB_LABEL_OPTIONAL, name, type, number, type_class, - options, -1, true); - - return Qnil; -} - -/* - * call-seq: - * MessageBuilderContext.required(name, type, number, type_class = nil, - * options = nil) - * - * Defines a new required field on this message type with the given type, tag - * number, and type class (for message and enum fields). The type must be a Ruby - * symbol (as accepted by FieldDescriptor#type=) and the type_class must be a - * string, if present (as accepted by FieldDescriptor#submsg_name=). - * - * Proto3 does not have required fields, but this method exists for - * completeness. Any attempt to add a message type with required fields to a - * pool will currently result in an error. - */ -static VALUE MessageBuilderContext_required(int argc, VALUE* argv, - VALUE _self) { - VALUE name, type, number; - VALUE type_class, options = Qnil; - - rb_scan_args(argc, argv, "32", &name, &type, &number, &type_class, &options); - - // Allow passing (name, type, number, options) or - // (name, type, number, type_class, options) - if (argc == 4 && RB_TYPE_P(type_class, T_HASH)) { - options = type_class; - type_class = Qnil; - } - - msgdef_add_field(_self, UPB_LABEL_REQUIRED, name, type, number, type_class, - options, -1, false); - - return Qnil; -} - -/* - * call-seq: - * MessageBuilderContext.repeated(name, type, number, type_class = nil) - * - * Defines a new repeated field on this message type with the given type, tag - * number, and type class (for message and enum fields). The type must be a Ruby - * symbol (as accepted by FieldDescriptor#type=) and the type_class must be a - * string, if present (as accepted by FieldDescriptor#submsg_name=). - */ -static VALUE MessageBuilderContext_repeated(int argc, VALUE* argv, - VALUE _self) { - VALUE name, type, number; - VALUE type_class, options = Qnil; - - rb_scan_args(argc, argv, "32", &name, &type, &number, &type_class, &options); - - // Allow passing (name, type, number, options) or - // (name, type, number, type_class, options) - if (argc == 4 && RB_TYPE_P(type_class, T_HASH)) { - options = type_class; - type_class = Qnil; - } - - msgdef_add_field(_self, UPB_LABEL_REPEATED, name, type, number, type_class, - options, -1, false); - - return Qnil; -} - -/* - * call-seq: - * MessageBuilderContext.map(name, key_type, value_type, number, - * value_type_class = nil) - * - * Defines a new map field on this message type with the given key and value - * types, tag number, and type class (for message and enum value types). The key - * type must be :int32/:uint32/:int64/:uint64, :bool, or :string. The value type - * type must be a Ruby symbol (as accepted by FieldDescriptor#type=) and the - * type_class must be a string, if present (as accepted by - * FieldDescriptor#submsg_name=). - */ -static VALUE MessageBuilderContext_map(int argc, VALUE* argv, VALUE _self) { - MessageBuilderContext* self = ruby_to_MessageBuilderContext(_self); - VALUE name, key_type, value_type, number, type_class; - VALUE mapentry_desc_name; - FileBuilderContext* file_builder; - upb_strview msg_name; - - if (argc < 4) { - rb_raise(rb_eArgError, "Expected at least 4 arguments."); - } - name = argv[0]; - key_type = argv[1]; - value_type = argv[2]; - number = argv[3]; - type_class = (argc > 4) ? argv[4] : Qnil; - - // Validate the key type. We can't accept enums, messages, or floats/doubles - // as map keys. (We exclude these explicitly, and the field-descriptor setter - // below then ensures that the type is one of the remaining valid options.) - if (SYM2ID(key_type) == rb_intern("float") || - SYM2ID(key_type) == rb_intern("double") || - SYM2ID(key_type) == rb_intern("enum") || - SYM2ID(key_type) == rb_intern("message")) { - rb_raise(rb_eArgError, - "Cannot add a map field with a float, double, enum, or message " - "type."); - } - - file_builder = ruby_to_FileBuilderContext(self->file_builder); - - // Create a new message descriptor for the map entry message, and create a - // repeated submessage field here with that type. - msg_name = google_protobuf_DescriptorProto_name(self->msg_proto); - mapentry_desc_name = rb_str_new(msg_name.data, msg_name.size); - mapentry_desc_name = rb_str_cat2(mapentry_desc_name, "_MapEntry_"); - mapentry_desc_name = - rb_str_cat2(mapentry_desc_name, rb_id2name(SYM2ID(name))); - - { - // message _MapEntry_ { /* ... */ } - VALUE args[1] = {mapentry_desc_name}; - VALUE types = rb_ary_new3(3, key_type, value_type, type_class); - rb_block_call(self->file_builder, rb_intern("add_message"), 1, args, - make_mapentry, types); - } - - // If this file is in a package, we need to qualify the map entry type. - if (google_protobuf_FileDescriptorProto_has_package(file_builder->file_proto)) { - upb_strview package_view = - google_protobuf_FileDescriptorProto_package(file_builder->file_proto); - VALUE package = rb_str_new(package_view.data, package_view.size); - package = rb_str_cat2(package, "."); - mapentry_desc_name = rb_str_concat(package, mapentry_desc_name); - } - - // repeated MapEntry = ; - rb_funcall(_self, rb_intern("repeated"), 4, name, - ID2SYM(rb_intern("message")), number, mapentry_desc_name); - - return Qnil; -} - -/* - * call-seq: - * MessageBuilderContext.oneof(name, &block) => nil - * - * Creates a new OneofDescriptor with the given name, creates a - * OneofBuilderContext attached to that OneofDescriptor, evaluates the given - * block in the context of that OneofBuilderContext with #instance_eval, and - * then adds the oneof to the message. - * - * This is the recommended, idiomatic way to build oneof definitions. - */ -static VALUE MessageBuilderContext_oneof(VALUE _self, VALUE name) { - MessageBuilderContext* self = ruby_to_MessageBuilderContext(_self); - size_t oneof_count; - FileBuilderContext* file_context = - ruby_to_FileBuilderContext(self->file_builder); - google_protobuf_OneofDescriptorProto* oneof_proto; - - // Existing oneof_count becomes oneof_index. - google_protobuf_DescriptorProto_oneof_decl(self->msg_proto, &oneof_count); - - // Create oneof_proto and set its name. - oneof_proto = google_protobuf_DescriptorProto_add_oneof_decl( - self->msg_proto, file_context->arena); - google_protobuf_OneofDescriptorProto_set_name( - oneof_proto, FileBuilderContext_strdup_sym(self->file_builder, name)); - - // Evaluate the block with the builder as argument. - { - VALUE args[2] = { INT2NUM(oneof_count), _self }; - VALUE ctx = rb_class_new_instance(2, args, cOneofBuilderContext); - VALUE block = rb_block_proc(); - rb_funcall_with_block(ctx, rb_intern("instance_eval"), 0, NULL, block); - } - - return Qnil; -} - -static void MessageBuilderContext_add_synthetic_oneofs(VALUE _self) { - MessageBuilderContext* self = ruby_to_MessageBuilderContext(_self); - FileBuilderContext* file_context = - ruby_to_FileBuilderContext(self->file_builder); - size_t field_count, oneof_count; - google_protobuf_FieldDescriptorProto** fields = - google_protobuf_DescriptorProto_mutable_field(self->msg_proto, &field_count); - const google_protobuf_OneofDescriptorProto*const* oneofs = - google_protobuf_DescriptorProto_oneof_decl(self->msg_proto, &oneof_count); - VALUE names = rb_hash_new(); - VALUE underscore = rb_str_new2("_"); - size_t i; - - // We have to build a set of all names, to ensure that synthetic oneofs are - // not creating conflicts. - for (i = 0; i < field_count; i++) { - upb_strview name = google_protobuf_FieldDescriptorProto_name(fields[i]); - rb_hash_aset(names, rb_str_new(name.data, name.size), Qtrue); - } - for (i = 0; i < oneof_count; i++) { - upb_strview name = google_protobuf_OneofDescriptorProto_name(oneofs[i]); - rb_hash_aset(names, rb_str_new(name.data, name.size), Qtrue); - } - - for (i = 0; i < field_count; i++) { - google_protobuf_OneofDescriptorProto* oneof_proto; - VALUE oneof_name; - upb_strview field_name; - - if (!google_protobuf_FieldDescriptorProto_proto3_optional(fields[i])) { - continue; - } - - // Prepend '_' until we are no longer conflicting. - field_name = google_protobuf_FieldDescriptorProto_name(fields[i]); - oneof_name = rb_str_new(field_name.data, field_name.size); - while (rb_hash_lookup(names, oneof_name) != Qnil) { - oneof_name = rb_str_plus(underscore, oneof_name); - } - - rb_hash_aset(names, oneof_name, Qtrue); - google_protobuf_FieldDescriptorProto_set_oneof_index(fields[i], - oneof_count++); - oneof_proto = google_protobuf_DescriptorProto_add_oneof_decl( - self->msg_proto, file_context->arena); - google_protobuf_OneofDescriptorProto_set_name( - oneof_proto, FileBuilderContext_strdup(self->file_builder, oneof_name)); - } -} - -static void MessageBuilderContext_register(VALUE module) { - VALUE klass = rb_define_class_under( - module, "MessageBuilderContext", rb_cObject); - rb_define_alloc_func(klass, MessageBuilderContext_alloc); - rb_define_method(klass, "initialize", - MessageBuilderContext_initialize, 2); - rb_define_method(klass, "optional", MessageBuilderContext_optional, -1); - rb_define_method(klass, "proto3_optional", MessageBuilderContext_proto3_optional, -1); - rb_define_method(klass, "required", MessageBuilderContext_required, -1); - rb_define_method(klass, "repeated", MessageBuilderContext_repeated, -1); - rb_define_method(klass, "map", MessageBuilderContext_map, -1); - rb_define_method(klass, "oneof", MessageBuilderContext_oneof, 1); - rb_gc_register_address(&cMessageBuilderContext); - cMessageBuilderContext = klass; -} - -// ----------------------------------------------------------------------------- -// OneofBuilderContext. -// ----------------------------------------------------------------------------- - -typedef struct { - int oneof_index; - VALUE message_builder; -} OneofBuilderContext; - -static VALUE cOneofBuilderContext = Qnil; - -void OneofBuilderContext_mark(void* _self) { - OneofBuilderContext* self = _self; - rb_gc_mark(self->message_builder); -} - -static const rb_data_type_t OneofBuilderContext_type = { - "Google::Protobuf::Internal::OneofBuilderContext", - {OneofBuilderContext_mark, RUBY_DEFAULT_FREE, NULL}, - .flags = RUBY_TYPED_FREE_IMMEDIATELY, -}; - -static OneofBuilderContext* ruby_to_OneofBuilderContext(VALUE val) { - OneofBuilderContext* ret; - TypedData_Get_Struct(val, OneofBuilderContext, &OneofBuilderContext_type, - ret); - return ret; -} - -static VALUE OneofBuilderContext_alloc(VALUE klass) { - OneofBuilderContext* self = ALLOC(OneofBuilderContext); - VALUE ret = TypedData_Wrap_Struct(klass, &OneofBuilderContext_type, self); - self->oneof_index = 0; - self->message_builder = Qnil; - return ret; -} - -/* - * call-seq: - * OneofBuilderContext.new(oneof_index, message_builder) => context - * - * Create a new oneof builder context around the given oneof descriptor and - * builder context. This class is intended to serve as a DSL context to be used - * with #instance_eval. - */ -static VALUE OneofBuilderContext_initialize(VALUE _self, VALUE oneof_index, - VALUE message_builder) { - OneofBuilderContext* self = ruby_to_OneofBuilderContext(_self); - self->oneof_index = NUM2INT(oneof_index); - self->message_builder = message_builder; - return Qnil; -} - -/* - * call-seq: - * OneofBuilderContext.optional(name, type, number, type_class = nil, - * default_value = nil) - * - * Defines a new optional field in this oneof with the given type, tag number, - * and type class (for message and enum fields). The type must be a Ruby symbol - * (as accepted by FieldDescriptor#type=) and the type_class must be a string, - * if present (as accepted by FieldDescriptor#submsg_name=). - */ -static VALUE OneofBuilderContext_optional(int argc, VALUE* argv, VALUE _self) { - OneofBuilderContext* self = ruby_to_OneofBuilderContext(_self); - VALUE name, type, number; - VALUE type_class, options = Qnil; - - rb_scan_args(argc, argv, "32", &name, &type, &number, &type_class, &options); - - msgdef_add_field(self->message_builder, UPB_LABEL_OPTIONAL, name, type, - number, type_class, options, self->oneof_index, false); - - return Qnil; -} - -static void OneofBuilderContext_register(VALUE module) { - VALUE klass = rb_define_class_under( - module, "OneofBuilderContext", rb_cObject); - rb_define_alloc_func(klass, OneofBuilderContext_alloc); - rb_define_method(klass, "initialize", - OneofBuilderContext_initialize, 2); - rb_define_method(klass, "optional", OneofBuilderContext_optional, -1); - rb_gc_register_address(&cOneofBuilderContext); - cOneofBuilderContext = klass; -} - -// ----------------------------------------------------------------------------- -// EnumBuilderContext. -// ----------------------------------------------------------------------------- - -typedef struct { - google_protobuf_EnumDescriptorProto* enum_proto; - VALUE file_builder; -} EnumBuilderContext; - -static VALUE cEnumBuilderContext = Qnil; - -void EnumBuilderContext_mark(void* _self) { - EnumBuilderContext* self = _self; - rb_gc_mark(self->file_builder); -} - -static const rb_data_type_t EnumBuilderContext_type = { - "Google::Protobuf::Internal::EnumBuilderContext", - {EnumBuilderContext_mark, RUBY_DEFAULT_FREE, NULL}, - .flags = RUBY_TYPED_FREE_IMMEDIATELY, -}; - -static EnumBuilderContext* ruby_to_EnumBuilderContext(VALUE val) { - EnumBuilderContext* ret; - TypedData_Get_Struct(val, EnumBuilderContext, &EnumBuilderContext_type, ret); - return ret; -} - -static VALUE EnumBuilderContext_alloc(VALUE klass) { - EnumBuilderContext* self = ALLOC(EnumBuilderContext); - VALUE ret = TypedData_Wrap_Struct(klass, &EnumBuilderContext_type, self); - self->enum_proto = NULL; - self->file_builder = Qnil; - return ret; -} - -/* - * call-seq: - * EnumBuilderContext.new(file_builder) => context - * - * Create a new builder context around the given enum descriptor. This class is - * intended to serve as a DSL context to be used with #instance_eval. - */ -static VALUE EnumBuilderContext_initialize(VALUE _self, VALUE _file_builder, - VALUE name) { - EnumBuilderContext* self = ruby_to_EnumBuilderContext(_self); - FileBuilderContext* file_builder = ruby_to_FileBuilderContext(_file_builder); - google_protobuf_FileDescriptorProto* file_proto = file_builder->file_proto; - - self->file_builder = _file_builder; - self->enum_proto = google_protobuf_FileDescriptorProto_add_enum_type( - file_proto, file_builder->arena); - - google_protobuf_EnumDescriptorProto_set_name( - self->enum_proto, FileBuilderContext_strdup(_file_builder, name)); - - return Qnil; -} - -/* - * call-seq: - * EnumBuilder.add_value(name, number) - * - * Adds the given name => number mapping to the enum type. Name must be a Ruby - * symbol. - */ -static VALUE EnumBuilderContext_value(VALUE _self, VALUE name, VALUE number) { - EnumBuilderContext* self = ruby_to_EnumBuilderContext(_self); - FileBuilderContext* file_builder = - ruby_to_FileBuilderContext(self->file_builder); - google_protobuf_EnumValueDescriptorProto* enum_value; - - enum_value = google_protobuf_EnumDescriptorProto_add_value( - self->enum_proto, file_builder->arena); - - google_protobuf_EnumValueDescriptorProto_set_name( - enum_value, FileBuilderContext_strdup_sym(self->file_builder, name)); - google_protobuf_EnumValueDescriptorProto_set_number(enum_value, - NUM2INT(number)); - - return Qnil; -} - -static void EnumBuilderContext_register(VALUE module) { - VALUE klass = rb_define_class_under( - module, "EnumBuilderContext", rb_cObject); - rb_define_alloc_func(klass, EnumBuilderContext_alloc); - rb_define_method(klass, "initialize", EnumBuilderContext_initialize, 2); - rb_define_method(klass, "value", EnumBuilderContext_value, 2); - rb_gc_register_address(&cEnumBuilderContext); - cEnumBuilderContext = klass; -} - -// ----------------------------------------------------------------------------- -// Builder. -// ----------------------------------------------------------------------------- - -typedef struct { - VALUE descriptor_pool; - VALUE default_file_builder; -} Builder; - -static VALUE cBuilder = Qnil; - -static void Builder_mark(void* _self) { - Builder* self = _self; - rb_gc_mark(self->descriptor_pool); - rb_gc_mark(self->default_file_builder); -} - -static const rb_data_type_t Builder_type = { - "Google::Protobuf::Internal::Builder", - {Builder_mark, RUBY_DEFAULT_FREE, NULL}, - .flags = RUBY_TYPED_FREE_IMMEDIATELY, -}; - -static Builder* ruby_to_Builder(VALUE val) { - Builder* ret; - TypedData_Get_Struct(val, Builder, &Builder_type, ret); - return ret; -} - -static VALUE Builder_alloc(VALUE klass) { - Builder* self = ALLOC(Builder); - VALUE ret = TypedData_Wrap_Struct(klass, &Builder_type, self); - self->descriptor_pool = Qnil; - self->default_file_builder = Qnil; - return ret; -} - -/* - * call-seq: - * Builder.new(descriptor_pool) => builder - * - * Creates a new Builder. A Builder can accumulate a set of new message and enum - * descriptors and atomically register them into a pool in a way that allows for - * (co)recursive type references. - */ -static VALUE Builder_initialize(VALUE _self, VALUE pool) { - Builder* self = ruby_to_Builder(_self); - self->descriptor_pool = pool; - self->default_file_builder = Qnil; // Created lazily if needed. - return Qnil; -} - -/* - * call-seq: - * Builder.add_file(name, options = nil, &block) - * - * Creates a new, file descriptor with the given name and options and invokes - * the block in the context of a FileBuilderContext on that descriptor. The - * block can then call FileBuilderContext#add_message or - * FileBuilderContext#add_enum to define new messages or enums, respectively. - * - * This is the recommended, idiomatic way to build file descriptors. - */ -static VALUE Builder_add_file(int argc, VALUE* argv, VALUE _self) { - Builder* self = ruby_to_Builder(_self); - VALUE name, options; - VALUE ctx; - VALUE block; - - rb_scan_args(argc, argv, "11", &name, &options); - - { - VALUE args[3] = { self->descriptor_pool, name, options }; - ctx = rb_class_new_instance(3, args, cFileBuilderContext); - } - - block = rb_block_proc(); - rb_funcall_with_block(ctx, rb_intern("instance_eval"), 0, NULL, block); - FileBuilderContext_build(ctx); - - return Qnil; -} - -static VALUE Builder_get_default_file(VALUE _self) { - Builder* self = ruby_to_Builder(_self); - - /* Lazily create only if legacy builder-level methods are called. */ - if (self->default_file_builder == Qnil) { - VALUE name = rb_str_new2("ruby_default_file.proto"); - VALUE args [3] = { self->descriptor_pool, name, rb_hash_new() }; - self->default_file_builder = - rb_class_new_instance(3, args, cFileBuilderContext); - } - - return self->default_file_builder; -} - -/* - * call-seq: - * Builder.add_message(name, &block) - * - * Old and deprecated way to create a new descriptor. - * See FileBuilderContext.add_message for the recommended way. - * - * Exists for backwards compatibility to allow building descriptor pool for - * files generated by protoc which don't add messages within "add_file" block. - * Descriptors created this way get assigned to a default empty FileDescriptor. - */ -static VALUE Builder_add_message(VALUE _self, VALUE name) { - VALUE file_builder = Builder_get_default_file(_self); - rb_funcall_with_block(file_builder, rb_intern("add_message"), 1, &name, - rb_block_proc()); - return Qnil; -} - -/* - * call-seq: - * Builder.add_enum(name, &block) - * - * Old and deprecated way to create a new enum descriptor. - * See FileBuilderContext.add_enum for the recommended way. - * - * Exists for backwards compatibility to allow building descriptor pool for - * files generated by protoc which don't add enums within "add_file" block. - * Enum descriptors created this way get assigned to a default empty - * FileDescriptor. - */ -static VALUE Builder_add_enum(VALUE _self, VALUE name) { - VALUE file_builder = Builder_get_default_file(_self); - rb_funcall_with_block(file_builder, rb_intern("add_enum"), 1, &name, - rb_block_proc()); - return Qnil; -} - -/* This method is hidden from Ruby, and only called directly from - * DescriptorPool_build(). */ -static VALUE Builder_build(VALUE _self) { - Builder* self = ruby_to_Builder(_self); - - if (self->default_file_builder != Qnil) { - FileBuilderContext_build(self->default_file_builder); - self->default_file_builder = Qnil; - } - - return Qnil; -} - -static void Builder_register(VALUE module) { - VALUE klass = rb_define_class_under(module, "Builder", rb_cObject); - rb_define_alloc_func(klass, Builder_alloc); - rb_define_method(klass, "initialize", Builder_initialize, 1); - rb_define_method(klass, "add_file", Builder_add_file, -1); - rb_define_method(klass, "add_message", Builder_add_message, 1); - rb_define_method(klass, "add_enum", Builder_add_enum, 1); - rb_gc_register_address(&cBuilder); - cBuilder = klass; -} - static VALUE get_def_obj(VALUE _descriptor_pool, const void* ptr, VALUE klass) { DescriptorPool* descriptor_pool = ruby_to_DescriptorPool(_descriptor_pool); VALUE key = ULL2NUM((intptr_t)ptr); @@ -2573,11 +1278,6 @@ void Defs_register(VALUE module) { FieldDescriptor_register(module); OneofDescriptor_register(module); EnumDescriptor_register(module); - FileBuilderContext_register(module); - MessageBuilderContext_register(module); - OneofBuilderContext_register(module); - EnumBuilderContext_register(module); - Builder_register(module); rb_gc_register_address(&c_only_cookie); c_only_cookie = rb_class_new_instance(0, NULL, rb_cObject); diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index 83ece4cfa17a..936876803faf 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -51,75 +51,7 @@ class TypeError < ::TypeError; end require 'google/protobuf_c' end - module Google - module Protobuf - module Internal - def self.infer_package(names) - # Package is longest common prefix ending in '.', if any. - if not names.empty? - min, max = names.minmax - last_common_dot = nil - min.size.times { |i| - if min[i] != max[i] then break end - if min[i] == ?. then last_common_dot = i end - } - if last_common_dot - return min.slice(0, last_common_dot) - end - end - - nil - end - - class NestingBuilder - def initialize(msg_names, enum_names) - @to_pos = {nil=>nil} - @msg_children = Hash.new { |hash, key| hash[key] = [] } - @enum_children = Hash.new { |hash, key| hash[key] = [] } - - msg_names.each_with_index { |name, idx| @to_pos[name] = idx } - enum_names.each_with_index { |name, idx| @to_pos[name] = idx } - - msg_names.each { |name| @msg_children[parent(name)] << name } - enum_names.each { |name| @enum_children[parent(name)] << name } - end - - def build(package) - return build_msg(package) - end - - private - def build_msg(msg) - return { - :pos => @to_pos[msg], - :msgs => @msg_children[msg].map { |child| build_msg(child) }, - :enums => @enum_children[msg].map { |child| @to_pos[child] }, - } - end - - private - def parent(name) - idx = name.rindex(?.) - if idx - return name.slice(0, idx) - else - return nil - end - end - end - - def self.fixup_descriptor(package, msg_names, enum_names) - if package.nil? - package = self.infer_package(msg_names + enum_names) - end - - nesting = NestingBuilder.new(msg_names, enum_names).build(package) - - return package, nesting - end - end - end - end + require 'google/protobuf/descriptor_dsl' end require 'google/protobuf/repeated_field' diff --git a/ruby/lib/google/protobuf/descriptor_dsl.rb b/ruby/lib/google/protobuf/descriptor_dsl.rb new file mode 100644 index 000000000000..3c1b8e41d07a --- /dev/null +++ b/ruby/lib/google/protobuf/descriptor_dsl.rb @@ -0,0 +1,439 @@ +#!/usr/bin/ruby +# +# Code that implements the DSL for defining proto messages. + +require 'google/protobuf/descriptor_pb' + +module Google + module Protobuf + module Internal + class Builder + def initialize(pool) + @pool = pool + @default_file = nil # Constructed lazily + end + + def add_file(name, options={}, &block) + builder = FileBuilder.new(@pool, name, options) + builder.instance_eval(&block) + internal_add_file(builder) + end + + def add_message(name, &block) + internal_default_file.add_message(name, &block) + end + + def add_enum(name, &block) + internal_default_file.add_enum(name, &block) + end + + # ---- Internal methods, not part of the DSL ---- + + def build + if @default_file + internal_add_file(@default_file) + end + end + + private def internal_add_file(file_builder) + proto = file_builder.build + serialized = Google::Protobuf::FileDescriptorProto.encode(proto) + @pool.add_serialized_file(serialized) + end + + private def internal_default_file + @default_file ||= FileBuilder.new(@pool, "ruby_default_file.proto") + end + end + + class FileBuilder + def initialize(pool, name, options={}) + @pool = pool + @file_proto = Google::Protobuf::FileDescriptorProto.new( + name: name, + syntax: options.fetch(:syntax, "proto3") + ) + end + + def add_message(name, &block) + builder = MessageBuilder.new(name, self, @file_proto) + builder.instance_eval(&block) + builder.internal_add_synthetic_oneofs + end + + def add_enum(name, &block) + EnumBuilder.new(name, @file_proto).instance_eval(&block) + end + + # ---- Internal methods, not part of the DSL ---- + + # These methods fix up the file descriptor to account for differences + # between the DSL and FileDescriptorProto. + + # The DSL can omit a package name; here we infer what the package is if + # was not specified. + def infer_package(names) + # Package is longest common prefix ending in '.', if any. + if not names.empty? + min, max = names.minmax + last_common_dot = nil + min.size.times { |i| + if min[i] != max[i] then break end + if min[i] == "." then last_common_dot = i end + } + if last_common_dot + return min.slice(0, last_common_dot) + end + end + + nil + end + + def rewrite_enum_default(field) + if field.type != :TYPE_ENUM or !field.has_default_value? or !field.has_type_name? + return + end + + value = field.default_value + type_name = field.type_name + + if value.empty? or value[0].ord < "0".ord or value[0].ord > "9".ord + return + end + + if type_name.empty? || type_name[0] != "." + return + end + + type_name = type_name[1..-1] + as_int = Integer(value) rescue return + + enum_desc = @pool.lookup(type_name) + if enum_desc.is_a?(Google::Protobuf::EnumDescriptor) + # Enum was defined in a previous file. + name = enum_desc.lookup_value(as_int) + if name + # Update the default value in the proto. + field.default_value = name + end + else + # See if enum was defined in this file. + @file_proto.enum_type.each { |enum_proto| + if enum_proto.name == type_name + enum_proto.value.each { |enum_value_proto| + if enum_value_proto.number == as_int + # Update the default value in the proto. + field.default_value = enum_value_proto.name + return + end + } + # We found the right enum, but no value matched. + return + end + } + end + end + + # Historically we allowed enum defaults to be specified as a number. + # In retrospect this was a mistake as descriptors require defaults to + # be specified as a label. This can make a difference if multiple + # labels have the same number. + # + # Here we do a pass over all enum defaults and rewrite numeric defaults + # by looking up their labels. This is complicated by the fact that the + # enum definition can live in either the symtab or the file_proto. + # + # We take advantage of the fact that this is called *before* enums or + # messages are nested in other messages, so we only have to iterate + # one level deep. + def rewrite_enum_defaults + @file_proto.message_type.each { |msg| + msg.field.each { |field| + rewrite_enum_default(field) + } + } + end + + # We have to do some relatively complicated logic here for backward + # compatibility. + # + # In descriptor.proto, messages are nested inside other messages if that is + # what the original .proto file looks like. For example, suppose we have this + # foo.proto: + # + # package foo; + # message Bar { + # message Baz {} + # } + # + # The descriptor for this must look like this: + # + # file { + # name: "test.proto" + # package: "foo" + # message_type { + # name: "Bar" + # nested_type { + # name: "Baz" + # } + # } + # } + # + # However, the Ruby generated code has always generated messages in a flat, + # non-nested way: + # + # Google::Protobuf::DescriptorPool.generated_pool.build do + # add_message "foo.Bar" do + # end + # add_message "foo.Bar.Baz" do + # end + # end + # + # Here we need to do a translation where we turn this generated code into the + # above descriptor. We need to infer that "foo" is the package name, and not + # a message itself. */ + + def split_parent_name(msg_or_enum) + name = msg_or_enum.name + idx = name.rindex(?.) + if idx + return name[0...idx], name[idx+1..-1] + else + return nil, name + end + end + + def get_parent_msg(msgs_by_name, name, parent_name) + parent_msg = msgs_by_name[parent_name] + if parent_msg.nil? + raise "To define name #{name}, there must be a message named #{parent_name} to enclose it" + end + return parent_msg + end + + def fix_nesting + # Calculate and update package. + msgs_by_name = @file_proto.message_type.map { |msg| [msg.name, msg] }.to_h + enum_names = @file_proto.enum_type.map { |enum_proto| enum_proto.name } + + package = infer_package(msgs_by_name.keys + enum_names) + if package + @file_proto.package = package + end + + # Update nesting based on package. + final_msgs = Google::Protobuf::RepeatedField.new(:message, Google::Protobuf::DescriptorProto) + final_enums = Google::Protobuf::RepeatedField.new(:message, Google::Protobuf::EnumDescriptorProto) + + # Note: We don't iterate over msgs_by_name.values because we want to + # preserve order as listed in the DSL. + @file_proto.message_type.each { |msg| + parent_name, msg.name = split_parent_name(msg) + if parent_name == package + final_msgs << msg + else + get_parent_msg(msgs_by_name, msg.name, parent_name).nested_type << msg + end + } + + @file_proto.enum_type.each { |enum| + parent_name, enum.name = split_parent_name(enum) + if parent_name == package + final_enums << enum + else + get_parent_msg(msgs_by_name, enum.name, parent_name).enum_type << enum + end + } + + @file_proto.message_type = final_msgs + @file_proto.enum_type = final_enums + end + + def internal_file_proto + @file_proto + end + + def build + rewrite_enum_defaults + fix_nesting + return @file_proto + end + end + + class MessageBuilder + def initialize(name, file_builder, file_proto) + @file_builder = file_builder + @msg_proto = Google::Protobuf::DescriptorProto.new( + :name => name + ) + file_proto.message_type << @msg_proto + end + + def optional(name, type, number, type_class=nil, options=nil) + internal_add_field(:LABEL_OPTIONAL, name, type, number, type_class, options) + end + + def proto3_optional(name, type, number, type_class=nil, options=nil) + internal_add_field(:LABEL_OPTIONAL, name, type, number, type_class, options, + proto3_optional: true) + end + + def required(name, type, number, type_class=nil, options=nil) + internal_add_field(:LABEL_REQUIRED, name, type, number, type_class, options) + end + + def repeated(name, type, number, type_class = nil) + internal_add_field(:LABEL_REPEATED, name, type, number, type_class, nil) + end + + def oneof(name, &block) + OneofBuilder.new(name, self).instance_eval(&block) + end + + # Defines a new map field on this message type with the given key and + # value types, tag number, and type class (for message and enum value + # types). The key type must be :int32/:uint32/:int64/:uint64, :bool, or + # :string. The value type type must be a Ruby symbol (as accepted by + # FieldDescriptor#type=) and the type_class must be a string, if + # present (as accepted by FieldDescriptor#submsg_name=). + def map(name, key_type, value_type, number, value_type_class = nil) + if key_type == :float or key_type == :double or key_type == :enum or + key_type == :message + raise ArgError, "Not an acceptable key type: " + key_type + end + entry_name = "#{@msg_proto.name}_MapEntry_#{name}" + + @file_builder.add_message entry_name do + optional :key, key_type, 1 + optional :value, value_type, 2, value_type_class + end + options = @file_builder.internal_file_proto.message_type.last.options ||= MessageOptions.new + options.map_entry = true + repeated name, :message, number, entry_name + end + + # ---- Internal methods, not part of the DSL ---- + + def internal_add_synthetic_oneofs + # We have to build a set of all names, to ensure that synthetic oneofs + # are not creating conflicts + names = {} + @msg_proto.field.each { |field| names[field.name] = true } + @msg_proto.oneof_decl.each { |oneof| names[oneof.name] = true } + + @msg_proto.field.each { |field| + if field.proto3_optional + # Prepend '_' until we are no longer conflicting. + oneof_name = field.name + while names[oneof_name] + oneof_name = "_" + oneof_name + end + names[oneof_name] = true + field.oneof_index = @msg_proto.oneof_decl.size + @msg_proto.oneof_decl << Google::Protobuf::OneofDescriptorProto.new( + name: oneof_name + ) + end + } + end + + def internal_add_field(label, name, type, number, type_class, options, + oneof_index: nil, proto3_optional: false) + # Allow passing either: + # - (name, type, number, options) or + # - (name, type, number, type_class, options) + if options.nil? and type_class.instance_of?(Hash) + options = type_class; + type_class = nil; + end + + field_proto = Google::Protobuf::FieldDescriptorProto.new( + :label => label, + :name => name, + :type => ("TYPE_" + type.to_s.upcase).to_sym, + :number => number + ) + + if type_class + # Make it an absolute type name by prepending a dot. + field_proto.type_name = "." + type_class + end + + if oneof_index + field_proto.oneof_index = oneof_index + end + + if proto3_optional + field_proto.proto3_optional = true + end + + if options + if options.key?(:default) + default = options[:default] + if !default.instance_of?(String) + # Call #to_s since all defaults are strings in the descriptor. + default = default.to_s + end + # XXX: we should be C-escaping bytes defaults. + field_proto.default_value = default.dup.force_encoding("UTF-8") + end + if options.key?(:json_name) + field_proto.json_name = options[:json_name] + end + end + + @msg_proto.field << field_proto + end + + def internal_msg_proto + @msg_proto + end + end + + class OneofBuilder + def initialize(name, msg_builder) + @msg_builder = msg_builder + oneof_proto = Google::Protobuf::OneofDescriptorProto.new( + :name => name + ) + msg_proto = msg_builder.internal_msg_proto + @oneof_index = msg_proto.oneof_decl.size + msg_proto.oneof_decl << oneof_proto + end + + def optional(name, type, number, type_class=nil, options=nil) + @msg_builder.internal_add_field( + :LABEL_OPTIONAL, name, type, number, type_class, options, + oneof_index: @oneof_index) + end + end + + class EnumBuilder + def initialize(name, file_proto) + @enum_proto = Google::Protobuf::EnumDescriptorProto.new( + :name => name + ) + file_proto.enum_type << @enum_proto + end + + def value(name, number) + enum_value_proto = Google::Protobuf::EnumValueDescriptorProto.new( + name: name, + number: number + ) + @enum_proto.value << enum_value_proto + end + end + + end + + # Re-open the class (the rest of the class is implemented in C) + class DescriptorPool + def build(&block) + builder = Internal::Builder.new(self) + builder.instance_eval(&block) + builder.build + end + end + end +end diff --git a/ruby/travis-test.sh b/ruby/travis-test.sh index faf0f9d0c733..8980395f78b0 100755 --- a/ruby/travis-test.sh +++ b/ruby/travis-test.sh @@ -18,7 +18,7 @@ test_version() { rake gc_test && cd ../conformance && make test_jruby && cd ../ruby/compatibility_tests/v3.0.0 && ./test.sh" - elif [ "$version" == "ruby-2.6.0" -o "$version" == "ruby-2.7.0" -o "$version" == "ruby-3.0.0" ] ; then + elif [ "$version" == "ruby-2.6.0" -o "$version" == "ruby-2.7.0" -o "$version" == "ruby-3.0.2" ] ; then bash --login -c \ "rvm install $version && rvm use $version && \ which ruby && \ diff --git a/src/google/protobuf/compiler/ruby/ruby_generator.cc b/src/google/protobuf/compiler/ruby/ruby_generator.cc index cca69de08725..2cee902d713b 100644 --- a/src/google/protobuf/compiler/ruby/ruby_generator.cc +++ b/src/google/protobuf/compiler/ruby/ruby_generator.cc @@ -527,6 +527,42 @@ bool MaybeEmitDependency(const FileDescriptor* import, } } +bool GenerateDslDescriptor(const FileDescriptor* file, io::Printer* printer, + std::string* error) { + printer->Print( + "require 'google/protobuf'\n\n"); + printer->Print("Google::Protobuf::DescriptorPool.generated_pool.build do\n"); + printer->Indent(); + printer->Print("add_file(\"$filename$\", :syntax => :$syntax$) do\n", + "filename", file->name(), "syntax", + StringifySyntax(file->syntax())); + printer->Indent(); + for (int i = 0; i < file->message_type_count(); i++) { + if (!GenerateMessage(file->message_type(i), printer, error)) { + return false; + } + } + for (int i = 0; i < file->enum_type_count(); i++) { + GenerateEnum(file->enum_type(i), printer); + } + printer->Outdent(); + printer->Print("end\n"); + printer->Outdent(); + printer->Print( + "end\n\n"); + return true; +} + +bool GenerateBinaryDescriptor(const FileDescriptor* file, io::Printer* printer, + std::string* error) { + printer->Print( + R"(descriptor_data = File.binread(__FILE__).split("\n__END__\n", 2)[1])"); + printer->Print( + "\nGoogle::Protobuf::DescriptorPool.generated_pool.add_serialized_file(" + "descriptor_data)\n\n"); + return true; +} + bool GenerateFile(const FileDescriptor* file, io::Printer* printer, std::string* error) { printer->Print( @@ -535,9 +571,6 @@ bool GenerateFile(const FileDescriptor* file, io::Printer* printer, "\n", "filename", file->name()); - printer->Print( - "require 'google/protobuf'\n\n"); - for (int i = 0; i < file->dependency_count(); i++) { if (!MaybeEmitDependency(file->dependency(i), file, printer, error)) { return false; @@ -550,25 +583,13 @@ bool GenerateFile(const FileDescriptor* file, io::Printer* printer, GOOGLE_LOG(WARNING) << "Extensions are not yet supported for proto2 .proto files."; } - printer->Print("Google::Protobuf::DescriptorPool.generated_pool.build do\n"); - printer->Indent(); - printer->Print("add_file(\"$filename$\", :syntax => :$syntax$) do\n", - "filename", file->name(), "syntax", - StringifySyntax(file->syntax())); - printer->Indent(); - for (int i = 0; i < file->message_type_count(); i++) { - if (!GenerateMessage(file->message_type(i), printer, error)) { - return false; - } - } - for (int i = 0; i < file->enum_type_count(); i++) { - GenerateEnum(file->enum_type(i), printer); + bool use_raw_descriptor = file->name() == "google/protobuf/descriptor.proto"; + + if (use_raw_descriptor) { + GenerateBinaryDescriptor(file, printer, error); + } else { + GenerateDslDescriptor(file, printer, error); } - printer->Outdent(); - printer->Print("end\n"); - printer->Outdent(); - printer->Print( - "end\n\n"); int levels = GeneratePackageModules(file, printer); for (int i = 0; i < file->message_type_count(); i++) { @@ -578,6 +599,15 @@ bool GenerateFile(const FileDescriptor* file, io::Printer* printer, GenerateEnumAssignment("", file->enum_type(i), printer); } EndPackageModules(levels, printer); + + if (use_raw_descriptor) { + printer->Print("\n__END__\n"); + FileDescriptorProto file_proto; + file->CopyTo(&file_proto); + std::string file_data; + file_proto.SerializeToString(&file_data); + printer->Print("$raw_descriptor$", "raw_descriptor", file_data); + } return true; } diff --git a/tests.sh b/tests.sh index f633881af8de..04d49c6ea16a 100755 --- a/tests.sh +++ b/tests.sh @@ -438,7 +438,7 @@ build_ruby27() { } build_ruby30() { internal_build_cpp # For conformance tests. - cd ruby && bash travis-test.sh ruby-3.0.0 && cd .. + cd ruby && bash travis-test.sh ruby-3.0.2 && cd .. } build_jruby() {