diff --git a/.golangci-new.yml b/.golangci-new.yml index 3f7396ffbce1..7f7ae8743c8f 100644 --- a/.golangci-new.yml +++ b/.golangci-new.yml @@ -81,6 +81,7 @@ linters: - lll - misspell - nolintlint + - sloglint - unused - whitespace diff --git a/.golangci.yml b/.golangci.yml index 90cf4448e303..7a64401ffcec 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -27,11 +27,11 @@ linters-settings: - pkg: github.com/jackc/pgx/v4 desc: use `github.com/jackc/pgx/v5` package instead fjson: + # TODO https://github.com/FerretDB/FerretDB/issues/4157 files: - $all - "!**/internal/bson/*_test.go" - "!**/internal/util/testutil/*.go" - - "!**/internal/wire/*.go" deny: - pkg: github.com/FerretDB/FerretDB/internal/types/fjson bson: @@ -209,6 +209,15 @@ linters-settings: ignore-generated-header: true severity: error rules: [] + sloglint: + no-mixed-args: true + kv-only: false + attr-only: true + context-only: false # https://github.com/go-simpler/sloglint/issues/29 + static-msg: false # TODO https://github.com/FerretDB/FerretDB/issues/3421 + no-raw-keys: false # TODO https://github.com/FerretDB/FerretDB/issues/3421 + key-naming-case: snake + args-on-sep-lines: false staticcheck: checks: - all @@ -241,6 +250,7 @@ linters: - misspell - nolintlint - revive + - sloglint - staticcheck - unused - whitespace diff --git a/integration/.golangci-new.yml b/integration/.golangci-new.yml index 1abc2af48ca1..7db5474d2fce 100644 --- a/integration/.golangci-new.yml +++ b/integration/.golangci-new.yml @@ -73,6 +73,7 @@ linters: - lll - misspell - nolintlint + - sloglint - unused - whitespace diff --git a/integration/.golangci.yml b/integration/.golangci.yml index 190e5bcb5cdf..7d8e2dce5345 100644 --- a/integration/.golangci.yml +++ b/integration/.golangci.yml @@ -107,6 +107,15 @@ linters-settings: ignore-generated-header: true severity: error rules: [] + sloglint: + no-mixed-args: true + kv-only: false + attr-only: true + context-only: true + static-msg: true + no-raw-keys: false # TODO https://github.com/FerretDB/FerretDB/issues/3421 + key-naming-case: snake + args-on-sep-lines: false staticcheck: checks: - all @@ -138,6 +147,7 @@ linters: - misspell - nolintlint - revive + - sloglint - staticcheck - unused - whitespace diff --git a/integration/basic_test.go b/integration/basic_test.go index 348e46af556b..f99665f59566 100644 --- a/integration/basic_test.go +++ b/integration/basic_test.go @@ -553,11 +553,11 @@ func TestDebugError(t *testing.T) { } func TestCheckingNestedDocuments(t *testing.T) { - t.Parallel() + t.Skip("https://github.com/FerretDB/FerretDB/issues/3759") for name, tc := range map[string]struct { doc any - err error + err *mongo.CommandError }{ "1ok": { doc: CreateNestedDocument(1), @@ -573,24 +573,29 @@ func TestCheckingNestedDocuments(t *testing.T) { }, "180fail": { doc: CreateNestedDocument(180), - err: fmt.Errorf("bson.Array.ReadFrom (document has exceeded the max supported nesting: 179."), + err: &mongo.CommandError{ + Message: "bson.Array.ReadFrom (document has exceeded the max supported nesting: 179.", + }, }, "180endedWithDocumentFail": { doc: bson.D{{"v", CreateNestedDocument(179)}}, - err: fmt.Errorf("bson.Document.ReadFrom (document has exceeded the max supported nesting: 179."), + err: &mongo.CommandError{ + Message: "bson.Document.ReadFrom (document has exceeded the max supported nesting: 179.", + }, }, "1000fail": { doc: CreateNestedDocument(1000), - err: fmt.Errorf("bson.Document.ReadFrom (document has exceeded the max supported nesting: 179."), + err: &mongo.CommandError{ + Message: "bson.Document.ReadFrom (document has exceeded the max supported nesting: 179.", + }, }, } { - name, tc := name, tc t.Run(name, func(t *testing.T) { - t.Parallel() ctx, collection := setup.Setup(t) + _, err := collection.InsertOne(ctx, tc.doc) if tc.err != nil { - require.Error(t, tc.err) + AssertEqualCommandError(t, *tc.err, err) return } @@ -699,23 +704,19 @@ func TestMutatingClientMetadata(t *testing.T) { }, }, } { - name, tc := name, tc t.Run(name, func(t *testing.T) { t.Parallel() - res := db.RunCommand(ctx, tc.command) + var res bson.D + err := db.RunCommand(ctx, tc.command).Decode(&res) if tc.err != nil { - err := res.Decode(&res) AssertEqualCommandError(t, *tc.err, err) return } - var actualRes bson.D - err := res.Decode(&actualRes) - require.NoError(t, err) - require.NotNil(t, actualRes) + require.NotNil(t, res) }) } } diff --git a/integration/create_test.go b/integration/create_test.go index 47b2f70bda79..8d951239bc32 100644 --- a/integration/create_test.go +++ b/integration/create_test.go @@ -176,8 +176,6 @@ func TestCreateOnInsertStressDiffCollection(t *testing.T) { } func TestCreateStressSameCollection(tt *testing.T) { - tt.Parallel() - t := setup.FailsForFerretDB(tt, "https://github.com/FerretDB/FerretDB/issues/3853") // It should be rewritten to use teststress.Stress. diff --git a/internal/bson/bson_test.go b/internal/bson/bson_test.go index 3e85f769ccd6..ac5da42290b9 100644 --- a/internal/bson/bson_test.go +++ b/internal/bson/bson_test.go @@ -206,6 +206,7 @@ func fuzzBinary(f *testing.F, testCases []testCase, newFunc func() bsontype) { t.Skip() } + // TODO https://github.com/FerretDB/FerretDB/issues/4157 mB, err := fjson.Marshal(fromBSON(v)) require.NoError(t, err) assert.NotEmpty(t, mB) diff --git a/internal/bson2/array.go b/internal/bson2/array.go index b64bdadbc359..be18204a71d2 100644 --- a/internal/bson2/array.go +++ b/internal/bson2/array.go @@ -31,6 +31,28 @@ type Array struct { elements []any } +// NewArray creates a new Array from the given values. +func NewArray(values ...any) (*Array, error) { + res := &Array{ + elements: make([]any, 0, len(values)), + } + + for i, v := range values { + if err := res.Add(v); err != nil { + return nil, lazyerrors.Errorf("%d: %w", i, err) + } + } + + return res, nil +} + +// MakeArray creates a new empty Array with the given capacity. +func MakeArray(cap int) *Array { + return &Array{ + elements: make([]any, 0, cap), + } +} + // ConvertArray converts [*types.Array] to Array. func ConvertArray(arr *types.Array) (*Array, error) { iter := arr.Iterator() @@ -80,6 +102,17 @@ func (arr *Array) Convert() (*types.Array, error) { return res, nil } +// Add adds a new element to the Array. +func (arr *Array) Add(value any) error { + if err := validBSONType(value); err != nil { + return lazyerrors.Error(err) + } + + arr.elements = append(arr.elements, value) + + return nil +} + // Encode encodes BSON array. // // TODO https://github.com/FerretDB/FerretDB/issues/3759 @@ -111,6 +144,13 @@ func (arr *Array) LogValue() slog.Value { return slogValue(arr) } +// LogMessage returns an indented representation as a string, +// somewhat similar (but not identical) to JSON or Go syntax. +// It may change over time. +func (arr *Array) LogMessage() string { + return logMessage(arr) +} + // check interfaces var ( _ slog.LogValuer = (*Array)(nil) diff --git a/internal/bson2/bson2.go b/internal/bson2/bson2.go index 8e98c2f0d405..c5bd6a9bd035 100644 --- a/internal/bson2/bson2.go +++ b/internal/bson2/bson2.go @@ -213,7 +213,7 @@ func convertToTypes(v any) (any, error) { case string: return v, nil case Binary: - // Special case to prevent it from being stored as null in sjson / logged as null in fjson. + // Special case to prevent it from being stored as null in sjson. // TODO https://github.com/FerretDB/FerretDB/issues/260 if v.B == nil { v.B = []byte{} diff --git a/internal/bson2/bson2_test.go b/internal/bson2/bson2_test.go index 4389cc539eef..d9826f73eae6 100644 --- a/internal/bson2/bson2_test.go +++ b/internal/bson2/bson2_test.go @@ -37,6 +37,7 @@ type normalTestCase struct { name string raw RawDocument tdoc *types.Document + m string } // decodeTestCase represents a single test case for unsuccessful decoding. @@ -55,6 +56,8 @@ type decodeTestCase struct { } // normalTestCases represents test cases for successful decoding/encoding. +// +//nolint:lll // for readability var normalTestCases = []normalTestCase{ { name: "handshake1", @@ -80,6 +83,23 @@ var normalTestCases = []normalTestCase{ "compression", must.NotFail(types.NewArray("none")), "loadBalanced", false, )), + m: ` + { + "ismaster": true, + "client": { + "driver": {"name": "nodejs", "version": "4.0.0-beta.6"}, + "os": { + "type": "Darwin", + "name": "darwin", + "architecture": "x64", + "version": "20.6.0", + }, + "platform": "Node.js v14.17.3, LE (unified)|Node.js v14.17.3, LE (unified)", + "application": {"name": "mongosh 1.0.1"}, + }, + "compression": ["none"], + "loadBalanced": false, + }`, }, { name: "handshake2", @@ -105,6 +125,23 @@ var normalTestCases = []normalTestCase{ "compression", must.NotFail(types.NewArray("none")), "loadBalanced", false, )), + m: ` + { + "ismaster": true, + "client": { + "driver": {"name": "nodejs", "version": "4.0.0-beta.6"}, + "os": { + "type": "Darwin", + "name": "darwin", + "architecture": "x64", + "version": "20.6.0", + }, + "platform": "Node.js v14.17.3, LE (unified)|Node.js v14.17.3, LE (unified)", + "application": {"name": "mongosh 1.0.1"}, + }, + "compression": ["none"], + "loadBalanced": false, + }`, }, { name: "handshake3", @@ -122,6 +159,12 @@ var normalTestCases = []normalTestCase{ )), "$db", "admin", )), + m: ` + { + "buildInfo": 1, + "lsid": {"id": Binary(uuid:oxnytKF1QMe456OjLsJWvg==)}, + "$db": "admin", + }`, }, { name: "handshake4", @@ -167,6 +210,37 @@ var normalTestCases = []normalTestCase{ "storageEngines", must.NotFail(types.NewArray("devnull", "ephemeralForTest", "wiredTiger")), "ok", float64(1), )), + m: ` + { + "version": "5.0.0", + "gitVersion": "1184f004a99660de6f5e745573419bda8a28c0e9", + "modules": [], + "allocator": "tcmalloc", + "javascriptEngine": "mozjs", + "sysInfo": "deprecated", + "versionArray": [5, 0, 0, 0], + "openssl": { + "running": "OpenSSL 1.1.1f 31 Mar 2020", + "compiled": "OpenSSL 1.1.1f 31 Mar 2020", + }, + "buildEnvironment": { + "distmod": "ubuntu2004", + "distarch": "x86_64", + "cc": "/opt/mongodbtoolchain/v3/bin/gcc: gcc (GCC) 8.5.0", + "ccflags": "-Werror -include mongo/platform/basic.h -fasynchronous-unwind-tables -ggdb -Wall -Wsign-compare -Wno-unknown-pragmas -Winvalid-pch -fno-omit-frame-pointer -fno-strict-aliasing -O2 -march=sandybridge -mtune=generic -mprefer-vector-width=128 -Wno-unused-local-typedefs -Wno-unused-function -Wno-deprecated-declarations -Wno-unused-const-variable -Wno-unused-but-set-variable -Wno-missing-braces -fstack-protector-strong -Wa,--nocompress-debug-sections -fno-builtin-memcmp", + "cxx": "/opt/mongodbtoolchain/v3/bin/g++: g++ (GCC) 8.5.0", + "cxxflags": "-Woverloaded-virtual -Wno-maybe-uninitialized -fsized-deallocation -std=c++17", + "linkflags": "-Wl,--fatal-warnings -pthread -Wl,-z,now -fuse-ld=gold -fstack-protector-strong -Wl,--no-threads -Wl,--build-id -Wl,--hash-style=gnu -Wl,-z,noexecstack -Wl,--warn-execstack -Wl,-z,relro -Wl,--compress-debug-sections=none -Wl,-z,origin -Wl,--enable-new-dtags", + "target_arch": "x86_64", + "target_os": "linux", + "cppdefines": "SAFEINT_USE_INTRINSICS 0 PCRE_STATIC NDEBUG _XOPEN_SOURCE 700 _GNU_SOURCE _REENTRANT 1 _FORTIFY_SOURCE 2 BOOST_THREAD_VERSION 5 BOOST_THREAD_USES_DATETIME BOOST_SYSTEM_NO_DEPRECATED BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS BOOST_ENABLE_ASSERT_DEBUG_HANDLER BOOST_LOG_NO_SHORTHAND_NAMES BOOST_LOG_USE_NATIVE_SYSLOG BOOST_LOG_WITHOUT_THREAD_ATTR ABSL_FORCE_ALIGNED_ACCESS", + }, + "bits": 64, + "debug": false, + "maxBsonObjectSize": 16777216, + "storageEngines": ["devnull", "ephemeralForTest", "wiredTiger"], + "ok": 1.0, + }`, }, { name: "all", @@ -196,6 +270,20 @@ var normalTestCases = []normalTestCase{ "string", must.NotFail(types.NewArray("foo", "")), "timestamp", must.NotFail(types.NewArray(types.Timestamp(42), types.Timestamp(0))), )), + m: ` + { + "array": [[""], ["foo"]], + "binary": [Binary(user:Qg==), Binary(generic:)], + "bool": [true, false], + "datetime": [2021-07-27T09:35:42.123Z, 0001-01-01T00:00:00Z], + "document": [{"foo": ""}, {"": "foo"}], + "double": [42.13, 0.0], + "int32": [42, 0], + "int64": [int64(42), int64(0)], + "objectID": [ObjectID(420000000000000000000000), ObjectID(000000000000000000000000)], + "string": ["foo", ""], + "timestamp": [Timestamp(42), Timestamp(0)], + }`, }, { name: "float64Doc", @@ -208,6 +296,7 @@ var normalTestCases = []normalTestCase{ tdoc: must.NotFail(types.NewDocument( "f", float64(3.141592653589793), )), + m: `{"f": 3.141592653589793}`, }, { name: "stringDoc", @@ -221,6 +310,7 @@ var normalTestCases = []normalTestCase{ tdoc: must.NotFail(types.NewDocument( "f", "v", )), + m: `{"f": "v"}`, }, { name: "binaryDoc", @@ -235,6 +325,7 @@ var normalTestCases = []normalTestCase{ tdoc: must.NotFail(types.NewDocument( "f", types.Binary{B: []byte("v"), Subtype: types.BinaryUser}, )), + m: `{"f": Binary(user:dg==)}`, }, { name: "objectIDDoc", @@ -247,6 +338,7 @@ var normalTestCases = []normalTestCase{ tdoc: must.NotFail(types.NewDocument( "f", types.ObjectID{0x62, 0x56, 0xc5, 0xba, 0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40}, )), + m: `{"f": ObjectID(6256c5ba182d4454fb210940)}`, }, { name: "boolDoc", @@ -259,6 +351,7 @@ var normalTestCases = []normalTestCase{ tdoc: must.NotFail(types.NewDocument( "f", true, )), + m: `{"f": true}`, }, { name: "timeDoc", @@ -271,6 +364,7 @@ var normalTestCases = []normalTestCase{ tdoc: must.NotFail(types.NewDocument( "f", time.Date(2024, 1, 17, 17, 40, 42, 123000000, time.UTC), )), + m: `{"f": 2024-01-17T17:40:42.123Z}`, }, { name: "nullDoc", @@ -282,6 +376,7 @@ var normalTestCases = []normalTestCase{ tdoc: must.NotFail(types.NewDocument( "f", types.Null, )), + m: `{"f": null}`, }, { name: "regexDoc", @@ -295,6 +390,7 @@ var normalTestCases = []normalTestCase{ tdoc: must.NotFail(types.NewDocument( "f", types.Regex{Pattern: "p", Options: "o"}, )), + m: `{"f": /p/o}`, }, { name: "int32Doc", @@ -307,6 +403,7 @@ var normalTestCases = []normalTestCase{ tdoc: must.NotFail(types.NewDocument( "f", int32(314159265), )), + m: `{"f": 314159265}`, }, { name: "timestampDoc", @@ -319,6 +416,7 @@ var normalTestCases = []normalTestCase{ tdoc: must.NotFail(types.NewDocument( "f", types.Timestamp(42), )), + m: `{"f": Timestamp(42)}`, }, { name: "int64Doc", @@ -331,6 +429,7 @@ var normalTestCases = []normalTestCase{ tdoc: must.NotFail(types.NewDocument( "f", int64(3141592653589793), )), + m: `{"f": int64(3141592653589793)}`, }, { name: "smallDoc", @@ -343,6 +442,7 @@ var normalTestCases = []normalTestCase{ tdoc: must.NotFail(types.NewDocument( "foo", must.NotFail(types.NewDocument()), )), + m: `{"foo": {}}`, }, { name: "smallArray", @@ -355,6 +455,7 @@ var normalTestCases = []normalTestCase{ tdoc: must.NotFail(types.NewDocument( "foo", must.NotFail(types.NewArray()), )), + m: `{"foo": []}`, }, { name: "duplicateKeys", @@ -368,6 +469,7 @@ var normalTestCases = []normalTestCase{ "", false, "", true, )), + m: `{"": false, "": true}`, }, } @@ -502,6 +604,8 @@ func TestNormal(t *testing.T) { assert.NotContains(t, ls, "panicked") assert.NotContains(t, ls, "called too many times") + assert.NotEmpty(t, tc.raw.LogMessage()) + l, err := FindRaw(tc.raw) require.NoError(t, err) require.Len(t, tc.raw, l) @@ -515,6 +619,8 @@ func TestNormal(t *testing.T) { assert.NotContains(t, ls, "panicked") assert.NotContains(t, ls, "called too many times") + assert.NotEmpty(t, doc.LogMessage()) + tdoc, err := doc.Convert() require.NoError(t, err) testutil.AssertEqual(t, tc.tdoc, tdoc) @@ -532,6 +638,8 @@ func TestNormal(t *testing.T) { assert.NotContains(t, ls, "panicked") assert.NotContains(t, ls, "called too many times") + assert.Equal(t, testutil.Unindent(t, tc.m), doc.LogMessage()) + tdoc, err := doc.Convert() require.NoError(t, err) testutil.AssertEqual(t, tc.tdoc, tdoc) @@ -580,6 +688,12 @@ func TestDecode(t *testing.T) { t.Run("bson2", func(t *testing.T) { t.Run("FindRaw", func(t *testing.T) { + ls := tc.raw.LogValue().Resolve().String() + assert.NotContains(t, ls, "panicked") + assert.NotContains(t, ls, "called too many times") + + assert.NotEmpty(t, tc.raw.LogMessage()) + l, err := FindRaw(tc.raw) if tc.findRawErr != nil { @@ -658,6 +772,7 @@ func BenchmarkDocument(b *testing.B) { b.Run("bson2", func(b *testing.B) { var doc *Document var raw []byte + var m string var err error b.Run("Decode", func(b *testing.B) { @@ -690,6 +805,22 @@ func BenchmarkDocument(b *testing.B) { assert.NotNil(b, raw) }) + b.Run("LogMessage", func(b *testing.B) { + doc, err = tc.raw.Decode() + require.NoError(b, err) + + b.ReportAllocs() + b.ResetTimer() + + for range b.N { + m = doc.LogMessage() + } + + b.StopTimer() + + assert.NotEmpty(b, m) + }) + b.Run("DecodeDeep", func(b *testing.B) { b.ReportAllocs() @@ -719,6 +850,22 @@ func BenchmarkDocument(b *testing.B) { require.NoError(b, err) assert.NotNil(b, raw) }) + + b.Run("LogMessageDeep", func(b *testing.B) { + doc, err = tc.raw.DecodeDeep() + require.NoError(b, err) + + b.ReportAllocs() + b.ResetTimer() + + for range b.N { + m = doc.LogMessage() + } + + b.StopTimer() + + assert.NotEmpty(b, m) + }) }) }) } @@ -735,6 +882,8 @@ func testRawDocument(t *testing.T, rawDoc RawDocument) { assert.NotContains(t, ls, "panicked") assert.NotContains(t, ls, "called too many times") + assert.NotEmpty(t, rawDoc.LogMessage()) + _, _ = FindRaw(rawDoc) }) @@ -751,6 +900,8 @@ func testRawDocument(t *testing.T, rawDoc RawDocument) { assert.NotContains(t, ls, "panicked") assert.NotContains(t, ls, "called too many times") + assert.NotEmpty(t, doc.LogMessage()) + _, _ = doc.Convert() raw, err := doc.Encode() @@ -769,6 +920,8 @@ func testRawDocument(t *testing.T, rawDoc RawDocument) { assert.NotContains(t, ls, "panicked") assert.NotContains(t, ls, "called too many times") + assert.NotEmpty(t, doc.LogMessage()) + _, err = doc.Convert() require.NoError(t, err) @@ -807,6 +960,8 @@ func testRawDocument(t *testing.T, rawDoc RawDocument) { assert.NotContains(t, ls, "panicked") assert.NotContains(t, ls, "called too many times") + assert.NotEmpty(t, bdoc2.LogMessage()) + tdoc1, err := types.ConvertDocument(&doc1) require.NoError(t, err) @@ -827,6 +982,8 @@ func testRawDocument(t *testing.T, rawDoc RawDocument) { assert.NotContains(t, ls, "panicked") assert.NotContains(t, ls, "called too many times") + assert.NotEmpty(t, doc2e.LogMessage()) + b1, err := doc1e.MarshalBinary() require.NoError(t, err) diff --git a/internal/bson2/document.go b/internal/bson2/document.go index 893803451a88..a1644c6f7e0f 100644 --- a/internal/bson2/document.go +++ b/internal/bson2/document.go @@ -55,7 +55,7 @@ func NewDocument(pairs ...any) (*Document, error) { value := pairs[i+1] - if err := res.add(name, value); err != nil { + if err := res.Add(name, value); err != nil { return nil, lazyerrors.Error(err) } } @@ -92,7 +92,7 @@ func ConvertDocument(doc *types.Document) (*Document, error) { return nil, lazyerrors.Error(err) } - if err = res.add(k, v); err != nil { + if err = res.Add(k, v); err != nil { return nil, lazyerrors.Error(err) } } @@ -133,8 +133,8 @@ func (doc *Document) Get(name string) any { return nil } -// add adds a new field to the Document. -func (doc *Document) add(name string, value any) error { +// Add adds a new field to the Document. +func (doc *Document) Add(name string, value any) error { if err := validBSONType(value); err != nil { return lazyerrors.Errorf("%q: %w", name, err) } @@ -178,6 +178,13 @@ func (doc *Document) LogValue() slog.Value { return slogValue(doc) } +// LogMessage returns an indented representation as a string, +// somewhat similar (but not identical) to JSON or Go syntax. +// It may change over time. +func (doc *Document) LogMessage() string { + return logMessage(doc) +} + // check interfaces var ( _ slog.LogValuer = (*Document)(nil) diff --git a/internal/bson2/logging.go b/internal/bson2/logging.go new file mode 100644 index 000000000000..29a823fd8d15 --- /dev/null +++ b/internal/bson2/logging.go @@ -0,0 +1,273 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bson2 + +import ( + "encoding/base64" + "encoding/hex" + "fmt" + "log/slog" + "math" + "strconv" + "strings" + "time" +) + +// flowLimit is the maximum length of a flow/inline/compact representation of a BSON value. +// It may be set to 0 to disable flow representation. +const flowLimit = 80 + +// nanBits is the most common pattern of a NaN float64 value, the same as math.Float64bits(math.NaN()). +const nanBits = 0b111111111111000000000000000000000000000000000000000000000000001 + +// slogValue returns a compact representation of any BSON value as [slog.Value]. +// It may change over time. +// +// The result is optimized for small values such as function parameters. +// Some information is lost; +// for example, both int32 and int64 values are returned with [slog.KindInt64], +// arrays are treated as documents, and empty documents are omitted. +// More information is subsequently lost in handlers output; +// for example, float64(42), int32(42), and int64(42) values would all look the same +// (`f64=42 i32=42 i64=42` or `{"f64":42,"i32":42,"i64":42}`). +func slogValue(v any) slog.Value { + switch v := v.(type) { + case *Document: + var attrs []slog.Attr + + for _, f := range v.fields { + attrs = append(attrs, slog.Attr{Key: f.name, Value: slogValue(f.value)}) + } + + return slog.GroupValue(attrs...) + + case RawDocument: + if v == nil { + return slog.StringValue("RawDocument") + } + + return slog.StringValue("RawDocument<" + strconv.Itoa(len(v)) + ">") + + case *Array: + var attrs []slog.Attr + + for i, v := range v.elements { + attrs = append(attrs, slog.Attr{Key: strconv.Itoa(i), Value: slogValue(v)}) + } + + return slog.GroupValue(attrs...) + + case RawArray: + if v == nil { + return slog.StringValue("RawArray") + } + + return slog.StringValue("RawArray<" + strconv.Itoa(len(v)) + ">") + + case float64: + // for JSON handler to work + switch { + case math.IsNaN(v): + return slog.StringValue("NaN") + case math.IsInf(v, 1): + return slog.StringValue("+Inf") + case math.IsInf(v, -1): + return slog.StringValue("-Inf") + } + + return slog.Float64Value(v) + + case string: + return slog.StringValue(v) + + case Binary: + return slog.StringValue(fmt.Sprintf("%#v", v)) + + case ObjectID: + return slog.StringValue("ObjectID(" + hex.EncodeToString(v[:]) + ")") + + case bool: + return slog.BoolValue(v) + + case time.Time: + return slog.TimeValue(v.Truncate(time.Millisecond).UTC()) + + case NullType: + return slog.Value{} + + case Regex: + return slog.StringValue(fmt.Sprintf("%#v", v)) + + case int32: + return slog.Int64Value(int64(v)) + + case Timestamp: + return slog.StringValue(fmt.Sprintf("%#v", v)) + + case int64: + return slog.Int64Value(v) + + default: + panic(fmt.Sprintf("invalid BSON type %T", v)) + } +} + +// logMessage returns an indented representation of any BSON value as a string, +// somewhat similar (but not identical) to JSON or Go syntax. +// It may change over time. +// +// The result is optimized for large values such as full request documents. +// All information is preserved. +func logMessage(v any) string { + return logMessageIndent(v, "") +} + +// logMessageIndent is a variant of [logMessage] with an indentation for recursive calls. +func logMessageIndent(v any, indent string) string { + switch v := v.(type) { + case *Document: + l := len(v.fields) + if l == 0 { + return "{}" + } + + if flowLimit > 0 { + res := "{" + + for i, f := range v.fields { + res += strconv.Quote(f.name) + `: ` + res += logMessageIndent(f.value, "") + + if i != l-1 { + res += ", " + } + } + + res += `}` + + if len(res) < flowLimit { + return res + } + } + + res := "{\n" + + for _, f := range v.fields { + res += indent + " " + res += strconv.Quote(f.name) + `: ` + res += logMessageIndent(f.value, indent+" ") + ",\n" + } + + res += indent + `}` + + return res + + case RawDocument: + return "RawDocument<" + strconv.FormatInt(int64(len(v)), 10) + ">" + + case *Array: + l := len(v.elements) + if l == 0 { + return "[]" + } + + if flowLimit > 0 { + res := "[" + + for i, e := range v.elements { + res += logMessageIndent(e, "") + + if i != l-1 { + res += ", " + } + } + + res += `]` + + if len(res) < flowLimit { + return res + } + } + + res := "[\n" + + for _, e := range v.elements { + res += indent + " " + res += logMessageIndent(e, indent+" ") + ",\n" + } + + res += indent + `]` + + return res + + case RawArray: + return "RawArray<" + strconv.FormatInt(int64(len(v)), 10) + ">" + + case float64: + switch { + case math.IsNaN(v): + if bits := math.Float64bits(v); bits != nanBits { + return fmt.Sprintf("NaN(%b)", bits) + } + + return "NaN" + + case math.IsInf(v, 1): + return "+Inf" + case math.IsInf(v, -1): + return "-Inf" + default: + res := strconv.FormatFloat(v, 'f', -1, 64) + if !strings.Contains(res, ".") { + res += ".0" + } + + return res + } + + case string: + return strconv.Quote(v) + + case Binary: + return "Binary(" + v.Subtype.String() + ":" + base64.StdEncoding.EncodeToString(v.B) + ")" + + case ObjectID: + return "ObjectID(" + hex.EncodeToString(v[:]) + ")" + + case bool: + return strconv.FormatBool(v) + + case time.Time: + return v.Truncate(time.Millisecond).UTC().Format(time.RFC3339Nano) + + case NullType: + return "null" + + case Regex: + return "/" + v.Pattern + "/" + v.Options + + case int32: + return strconv.FormatInt(int64(v), 10) + + case Timestamp: + return "Timestamp(" + strconv.FormatUint(uint64(v), 10) + ")" + + case int64: + return "int64(" + strconv.FormatInt(int64(v), 10) + ")" + + default: + panic(fmt.Sprintf("invalid BSON type %T", v)) + } +} diff --git a/internal/bson2/logging_test.go b/internal/bson2/logging_test.go new file mode 100644 index 000000000000..62c7ef172660 --- /dev/null +++ b/internal/bson2/logging_test.go @@ -0,0 +1,145 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bson2 + +import ( + "bytes" + "context" + "log/slog" + "math" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/FerretDB/FerretDB/internal/util/must" + "github.com/FerretDB/FerretDB/internal/util/testutil" +) + +func TestLogging(t *testing.T) { + opts := &slog.HandlerOptions{ + ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr { + if groups != nil { + return a + } + + if a.Key == "v" { + return a + } + + return slog.Attr{} + }, + } + + ctx := context.Background() + + var tbuf, jbuf bytes.Buffer + tlog := slog.New(slog.NewTextHandler(&tbuf, opts)) + jlog := slog.New(slog.NewJSONHandler(&jbuf, opts)) + + for _, tc := range []struct { + name string + v slog.LogValuer + t string + j string + m string + }{ + { + name: "Numbers", + v: must.NotFail(NewDocument( + "f64", 42.0, + "inf", float64(math.Inf(1)), + "neg_inf", float64(math.Inf(-1)), + "zero", math.Copysign(0, 1), + "neg_zero", math.Copysign(0, -1), + "nan", float64(math.NaN()), + "i32", int32(42), + "i64", int64(42), + )), + t: `v.f64=42 v.inf=+Inf v.neg_inf=-Inf v.zero=0 v.neg_zero=-0 v.nan=NaN v.i32=42 v.i64=42`, + j: `{"v":{"f64":42,"inf":"+Inf","neg_inf":"-Inf","zero":0,"neg_zero":-0,"nan":"NaN","i32":42,"i64":42}}`, + m: ` + { + "f64": 42.0, + "inf": +Inf, + "neg_inf": -Inf, + "zero": 0.0, + "neg_zero": -0.0, + "nan": NaN, + "i32": 42, + "i64": int64(42), + }`, + }, + { + name: "Scalars", + v: must.NotFail(NewDocument( + "null", Null, + "id", ObjectID{0x42}, + "bool", true, + "time", time.Date(2023, 3, 6, 13, 14, 42, 123456789, time.FixedZone("", int(4*time.Hour.Seconds()))), + )), + t: `v.null= v.id=ObjectID(420000000000000000000000) v.bool=true v.time=2023-03-06T09:14:42.123Z`, + j: `{"v":{"null":null,"id":"ObjectID(420000000000000000000000)","bool":true,"time":"2023-03-06T09:14:42.123Z"}}`, + m: ` + { + "null": null, + "id": ObjectID(420000000000000000000000), + "bool": true, + "time": 2023-03-06T09:14:42.123Z, + }`, + }, + { + name: "Composites", + v: must.NotFail(NewDocument( + "doc", must.NotFail(NewDocument( + "foo", "bar", + "baz", must.NotFail(NewDocument( + "qux", "quux", + )), + )), + "doc_raw", RawDocument{0x42}, + "doc_empty", must.NotFail(NewDocument()), + "array", must.NotFail(NewArray( + "foo", + "bar", + must.NotFail(NewArray("baz", "qux")), + )), + )), + t: `v.doc.foo=bar v.doc.baz.qux=quux v.doc_raw=RawDocument<1> ` + + `v.array.0=foo v.array.1=bar v.array.2.0=baz v.array.2.1=qux`, + j: `{"v":{"doc":{"foo":"bar","baz":{"qux":"quux"}},"doc_raw":"RawDocument<1>",` + + `"array":{"0":"foo","1":"bar","2":{"0":"baz","1":"qux"}}}}`, + m: ` + { + "doc": {"foo": "bar", "baz": {"qux": "quux"}}, + "doc_raw": RawDocument<1>, + "doc_empty": {}, + "array": ["foo", "bar", ["baz", "qux"]], + }`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + tlog.InfoContext(ctx, "", slog.Any("v", tc.v)) + assert.Equal(t, tc.t+"\n", tbuf.String()) + tbuf.Reset() + + jlog.InfoContext(ctx, "", slog.Any("v", tc.v)) + assert.Equal(t, tc.j+"\n", jbuf.String()) + jbuf.Reset() + + assert.Equal(t, testutil.Unindent(t, tc.m), logMessage(tc.v)) + }) + } +} diff --git a/internal/bson2/raw_array.go b/internal/bson2/raw_array.go index 504b89ccc5d0..5e0067a09d2a 100644 --- a/internal/bson2/raw_array.go +++ b/internal/bson2/raw_array.go @@ -27,11 +27,6 @@ import ( // It generally references a part of a larger slice, not a copy. type RawArray []byte -// LogValue implements slog.LogValuer interface. -func (arr RawArray) LogValue() slog.Value { - return slogValue(arr) -} - // Decode decodes a single BSON array that takes the whole byte slice. // // Only top-level elements are decoded; @@ -94,3 +89,19 @@ func (raw RawArray) decode(mode decodeMode) (*Array, error) { return res, nil } + +// LogValue implements slog.LogValuer interface. +func (doc RawArray) LogValue() slog.Value { + return slogValue(doc) +} + +// LogMessage returns a representation as a string. +// It may change over time. +func (doc RawArray) LogMessage() string { + return logMessage(doc) +} + +// check interfaces +var ( + _ slog.LogValuer = RawArray(nil) +) diff --git a/internal/bson2/raw_document.go b/internal/bson2/raw_document.go index 5f39b374d82a..19d2eb126aaf 100644 --- a/internal/bson2/raw_document.go +++ b/internal/bson2/raw_document.go @@ -27,11 +27,6 @@ import ( // It generally references a part of a larger slice, not a copy. type RawDocument []byte -// LogValue implements slog.LogValuer interface. -func (doc RawDocument) LogValue() slog.Value { - return slogValue(doc) -} - // Decode decodes a single BSON document that takes the whole byte slice. // // Only top-level fields are decoded; @@ -162,6 +157,22 @@ func (raw RawDocument) decode(mode decodeMode) (*Document, error) { return nil, lazyerrors.Error(err) } - must.NoError(res.add(name, v)) + must.NoError(res.Add(name, v)) } } + +// LogValue implements slog.LogValuer interface. +func (doc RawDocument) LogValue() slog.Value { + return slogValue(doc) +} + +// LogMessage returns a representation as a string. +// It may change over time. +func (doc RawDocument) LogMessage() string { + return logMessage(doc) +} + +// check interfaces +var ( + _ slog.LogValuer = RawDocument(nil) +) diff --git a/internal/bson2/slog.go b/internal/bson2/slog.go deleted file mode 100644 index 08458bde3768..000000000000 --- a/internal/bson2/slog.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2021 FerretDB Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bson2 - -import ( - "encoding/hex" - "fmt" - "log/slog" - "strconv" - "time" -) - -// slogValue converts any BSON value to [slog.Value]. -// -// TODO https://github.com/FerretDB/FerretDB/issues/3759 -// It is not clear if slog.Value represents is a good one and even if it is handler-independent. -func slogValue(v any) slog.Value { - switch v := v.(type) { - case *Document: - var attrs []slog.Attr - - for _, f := range v.fields { - attrs = append(attrs, slog.Attr{Key: f.name, Value: slogValue(f.value)}) - } - - return slog.GroupValue(attrs...) - - case RawDocument: - if v == nil { - return slog.StringValue("RawDocument(nil)") - } - - return slog.StringValue("RawDocument(" + strconv.Itoa(len(v)) + " bytes)") - - case *Array: - var attrs []slog.Attr - - for i, v := range v.elements { - attrs = append(attrs, slog.Attr{Key: strconv.Itoa(i), Value: slogValue(v)}) - } - - return slog.GroupValue(attrs...) - - case RawArray: - if v == nil { - return slog.StringValue("RawArray(nil)") - } - - return slog.StringValue("RawArray(" + strconv.Itoa(len(v)) + " bytes)") - - default: - return slogScalarValue(v) - } -} - -// slogScalarValue converts any scalar BSON value to [slog.Value]. -func slogScalarValue(v any) slog.Value { - switch v := v.(type) { - case float64: - return slog.StringValue(fmt.Sprintf("%[1]T(%[1]v)", v)) - case string: - return slog.StringValue(v) - case Binary: - return slog.AnyValue(v) - case ObjectID: - return slog.StringValue("ObjectID(" + hex.EncodeToString(v[:]) + ")") - case bool: - return slog.BoolValue(v) - case time.Time: - return slog.StringValue(fmt.Sprintf("%[1]T(%[1]v)", v)) - case NullType: - return slog.StringValue(fmt.Sprintf("%[1]T(%[1]v)", v)) - case Regex: - return slog.AnyValue(v) - case int32: - return slog.StringValue(fmt.Sprintf("%[1]T(%[1]v)", v)) - case Timestamp: - return slog.StringValue(fmt.Sprintf("%[1]T(%[1]v)", v)) - case int64: - return slog.StringValue(fmt.Sprintf("%[1]T(%[1]v)", v)) - default: - panic(fmt.Sprintf("invalid BSON type %T", v)) - } -} diff --git a/internal/driver/driver.go b/internal/driver/driver.go new file mode 100644 index 000000000000..f1c8d24b4a1f --- /dev/null +++ b/internal/driver/driver.go @@ -0,0 +1,171 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package driver provides low-level wire protocol driver for testing. +package driver + +import ( + "bufio" + "context" + "fmt" + "log/slog" + "net" + "net/url" + + "github.com/FerretDB/FerretDB/internal/util/lazyerrors" + "github.com/FerretDB/FerretDB/internal/wire" +) + +// Conn represents a single connection. +// +// It is not safe for concurrent use. +// +// Logger is used only with debug level. +type Conn struct { + c net.Conn + r *bufio.Reader + w *bufio.Writer + l *slog.Logger +} + +// Connect creates a new connection for the given MongoDB URI and logger. +// +// Context can be used to cancel the connection attempt. +// Canceling the context after the connection is established has no effect. +func Connect(ctx context.Context, uri string, l *slog.Logger) (*Conn, error) { + u, err := url.Parse(uri) + if err != nil { + return nil, lazyerrors.Error(err) + } + + if u.Scheme != "mongodb" { + return nil, lazyerrors.Errorf("invalid scheme %q", u.Scheme) + } + + if u.Opaque != "" { + return nil, lazyerrors.Errorf("invalid URI %q", uri) + } + + if _, _, err = net.SplitHostPort(u.Host); err != nil { + return nil, lazyerrors.Error(err) + } + + if u.User != nil { + return nil, lazyerrors.Errorf("authentication is not supported") + } + + if u.Path != "/" { + return nil, lazyerrors.Errorf("path %q is not supported", u.Path) + } + + for k := range u.Query() { + switch k { + default: + return nil, lazyerrors.Errorf("query parameter %q is not supported", k) + } + } + + l.DebugContext(ctx, "Connecting...", slog.String("uri", uri)) + + d := net.Dialer{} + + c, err := d.DialContext(ctx, "tcp", u.Host) + if err != nil { + return nil, lazyerrors.Error(err) + } + + return &Conn{ + c: c, + r: bufio.NewReader(c), + w: bufio.NewWriter(c), + l: l, + }, nil +} + +// Close closes the connection. +func (c *Conn) Close() error { + var err error + + c.l.Debug("Closing...") + + if e := c.w.Flush(); e != nil { + err = lazyerrors.Error(e) + } + + if e := c.c.Close(); e != nil && err == nil { + err = lazyerrors.Error(e) + } + + return err +} + +// Read reads the next message from the connection. +func (c *Conn) Read() (*wire.MsgHeader, wire.MsgBody, error) { + header, body, err := wire.ReadMessage(c.r) + if err != nil { + return nil, nil, lazyerrors.Error(err) + } + + c.l.Debug( + fmt.Sprintf("<<<\n%s", body.String()), + slog.Int("length", int(header.MessageLength)), + slog.Int("id", int(header.ResponseTo)), + slog.Int("response_to", int(header.ResponseTo)), + slog.String("opcode", header.OpCode.String()), + ) + + return header, body, nil +} + +// Write writes the given message to the connection. +func (c *Conn) Write(header *wire.MsgHeader, body wire.MsgBody) error { + c.l.Debug( + fmt.Sprintf(">>>\n%s", body.String()), + slog.Int("length", int(header.MessageLength)), + slog.Int("id", int(header.ResponseTo)), + slog.Int("response_to", int(header.ResponseTo)), + slog.String("opcode", header.OpCode.String()), + ) + + if err := wire.WriteMessage(c.w, header, body); err != nil { + return lazyerrors.Error(err) + } + + if err := c.w.Flush(); err != nil { + return lazyerrors.Error(err) + } + + return nil +} + +// WriteRaw writes the given raw bytes to the connection. +func (c *Conn) WriteRaw(b []byte) error { + c.l.Debug(fmt.Sprintf(">>> %d raw bytes", len(b))) + + if _, err := c.w.Write(b); err != nil { + return lazyerrors.Error(err) + } + + if err := c.w.Flush(); err != nil { + return lazyerrors.Error(err) + } + + return nil +} + +// Request sends the given request to the connection and returns the response. +func (c *Conn) Request(ctx context.Context, header *wire.MsgHeader, body wire.MsgBody) (*wire.MsgHeader, wire.MsgBody, error) { + // TODO https://github.com/FerretDB/FerretDB/issues/4146 + panic("not implemented") +} diff --git a/internal/driver/driver_test.go b/internal/driver/driver_test.go new file mode 100644 index 000000000000..a66555ee8b93 --- /dev/null +++ b/internal/driver/driver_test.go @@ -0,0 +1,38 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package driver + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/FerretDB/FerretDB/internal/util/testutil" +) + +func TestDriver(t *testing.T) { + if testing.Short() { + t.Skip("skipping in -short mode") + } + + ctx := testutil.Ctx(t) + + c, err := Connect(ctx, "mongodb://127.0.0.1:47017/", testutil.SLogger(t)) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, c.Close()) }) + + // TODO https://github.com/FerretDB/FerretDB/issues/4146 + _ = c +} diff --git a/internal/util/testutil/dump.go b/internal/util/testutil/dump.go index 05b30af32dc2..aed228ccb35a 100644 --- a/internal/util/testutil/dump.go +++ b/internal/util/testutil/dump.go @@ -17,6 +17,7 @@ package testutil import ( "bytes" "encoding/json" + "strings" "github.com/stretchr/testify/require" @@ -29,7 +30,8 @@ import ( func Dump[T types.Type](tb testtb.TB, o T) string { tb.Helper() - // We might switch to go-spew or something else later. + // We should switch to bson2's format. + // TODO https://github.com/FerretDB/FerretDB/issues/4157 b, err := fjson.Marshal(o) require.NoError(tb, err) @@ -40,7 +42,8 @@ func Dump[T types.Type](tb testtb.TB, o T) string { func DumpSlice[T types.Type](tb testtb.TB, s []T) string { tb.Helper() - // We might switch to go-spew or something else later. + // We should switch to bson2's format. + // TODO https://github.com/FerretDB/FerretDB/issues/4157 res := []byte("[") @@ -68,3 +71,27 @@ func IndentJSON(tb testtb.TB, b []byte) []byte { require.NoError(tb, err) return dst.Bytes() } + +// Unindent removes the common number of leading tabs from all lines in s. +func Unindent(tb testtb.TB, s string) string { + tb.Helper() + + require.NotEmpty(tb, s) + + parts := strings.Split(s, "\n") + require.Positive(tb, len(parts)) + + if parts[0] == "" { + parts = parts[1:] + } + + indent := len(parts[0]) - len(strings.TrimLeft(parts[0], "\t")) + require.GreaterOrEqual(tb, indent, 0) + + for i := range parts { + require.Greater(tb, len(parts[i]), indent, "line: %q", parts[i]) + parts[i] = parts[i][indent:] + } + + return strings.Join(parts, "\n") +} diff --git a/internal/wire/op_msg.go b/internal/wire/op_msg.go index eaf8297b9386..85019bc16d42 100644 --- a/internal/wire/op_msg.go +++ b/internal/wire/op_msg.go @@ -16,12 +16,10 @@ package wire import ( "encoding/binary" - "encoding/json" "fmt" "github.com/FerretDB/FerretDB/internal/bson2" "github.com/FerretDB/FerretDB/internal/types" - "github.com/FerretDB/FerretDB/internal/types/fjson" "github.com/FerretDB/FerretDB/internal/util/debugbuild" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" "github.com/FerretDB/FerretDB/internal/util/must" @@ -56,6 +54,16 @@ type OpMsg struct { checksum uint32 } +// NewOpMsg creates a message with a single section of kind 0 with a single raw document. +func NewOpMsg(raw bson2.RawDocument) (*OpMsg, error) { + var msg OpMsg + if err := msg.SetSections(OpMsgSection{documents: []bson2.RawDocument{raw}}); err != nil { + return nil, lazyerrors.Error(err) + } + + return &msg, nil +} + // checkSections checks given sections. func checkSections(sections []OpMsgSection) error { if len(sections) == 0 { @@ -188,6 +196,26 @@ func (msg *OpMsg) Document() (*types.Document, error) { return res, nil } +// RawSections returns the value of section with kind 0 and the value of all sections with kind 1. +func (msg *OpMsg) RawSections() (bson2.RawDocument, []byte) { + var spec bson2.RawDocument + var seq []byte + + for _, s := range msg.Sections() { + switch s.Kind { + case 0: + spec = s.documents[0] + + case 1: + for _, d := range s.documents { + seq = append(seq, d...) + } + } + } + + return spec, seq +} + // RawDocument returns the value of msg as a [bson2.RawDocument]. // // The error is returned if msg contains anything other than a single section of kind 0 @@ -394,51 +422,51 @@ func (msg *OpMsg) String() string { return "" } - m := map[string]any{ - "FlagBits": msg.Flags, - "Checksum": msg.checksum, - } + m := must.NotFail(bson2.NewDocument( + "FlagBits", msg.Flags.String(), + "Checksum", int64(msg.checksum), + )) - sections := make([]map[string]any, len(msg.sections)) - for i, section := range msg.sections { - s := map[string]any{ - "Kind": section.Kind, - } + sections := bson2.MakeArray(len(msg.sections)) + for _, section := range msg.sections { + s := must.NotFail(bson2.NewDocument( + "Kind", int32(section.Kind), + )) switch section.Kind { case 0: - doc, err := section.documents[0].Convert() + doc, err := section.documents[0].DecodeDeep() if err == nil { - s["Document"] = json.RawMessage(must.NotFail(fjson.Marshal(doc))) + must.NoError(s.Add("Document", doc)) } else { - s["DocumentError"] = err.Error() + must.NoError(s.Add("DocumentError", err.Error())) } case 1: - s["Identifier"] = section.Identifier - docs := make([]json.RawMessage, len(section.documents)) + must.NoError(s.Add("Identifier", section.Identifier)) + docs := bson2.MakeArray(len(section.documents)) - for j, d := range section.documents { - doc, err := d.Convert() + for _, d := range section.documents { + doc, err := d.DecodeDeep() if err == nil { - docs[j] = json.RawMessage(must.NotFail(fjson.Marshal(doc))) + must.NoError(docs.Add(doc)) } else { - docs[j] = must.NotFail(json.Marshal(map[string]string{"error": err.Error()})) + must.NoError(docs.Add(must.NotFail(bson2.NewDocument("error", err.Error())))) } } - s["Documents"] = docs + must.NoError(s.Add("Documents", docs)) default: panic(fmt.Sprintf("unknown kind %d", section.Kind)) } - sections[i] = s + must.NoError(sections.Add(s)) } - m["Sections"] = sections + must.NoError(m.Add("Sections", sections)) - return string(must.NotFail(json.MarshalIndent(m, "", " "))) + return m.LogMessage() } // check interfaces diff --git a/internal/wire/op_msg_test.go b/internal/wire/op_msg_test.go index 127a3968436f..ba9c09ff128e 100644 --- a/internal/wire/op_msg_test.go +++ b/internal/wire/op_msg_test.go @@ -24,6 +24,9 @@ import ( "github.com/FerretDB/FerretDB/internal/util/testutil" ) +// msgTestCases represents test cases for OP_MSG decoding/encoding. +// +//nolint:lll // for readability var msgTestCases = []testCase{ { name: "handshake5", @@ -52,6 +55,21 @@ var msgTestCases = []testCase{ }}, }, command: "buildInfo", + m: ` + { + "FlagBits": "[]", + "Checksum": int64(0), + "Sections": [ + { + "Kind": 0, + "Document": { + "buildInfo": 1, + "lsid": {"id": Binary(uuid:oxnytKF1QMe456OjLsJWvg==)}, + "$db": "admin", + }, + }, + ], + }`, }, { name: "handshake6", @@ -110,6 +128,46 @@ var msgTestCases = []testCase{ }}, }, command: "version", + m: ` + { + "FlagBits": "[]", + "Checksum": int64(0), + "Sections": [ + { + "Kind": 0, + "Document": { + "version": "5.0.0", + "gitVersion": "1184f004a99660de6f5e745573419bda8a28c0e9", + "modules": [], + "allocator": "tcmalloc", + "javascriptEngine": "mozjs", + "sysInfo": "deprecated", + "versionArray": [5, 0, 0, 0], + "openssl": { + "running": "OpenSSL 1.1.1f 31 Mar 2020", + "compiled": "OpenSSL 1.1.1f 31 Mar 2020", + }, + "buildEnvironment": { + "distmod": "ubuntu2004", + "distarch": "x86_64", + "cc": "/opt/mongodbtoolchain/v3/bin/gcc: gcc (GCC) 8.5.0", + "ccflags": "-Werror -include mongo/platform/basic.h -fasynchronous-unwind-tables -ggdb -Wall -Wsign-compare -Wno-unknown-pragmas -Winvalid-pch -fno-omit-frame-pointer -fno-strict-aliasing -O2 -march=sandybridge -mtune=generic -mprefer-vector-width=128 -Wno-unused-local-typedefs -Wno-unused-function -Wno-deprecated-declarations -Wno-unused-const-variable -Wno-unused-but-set-variable -Wno-missing-braces -fstack-protector-strong -Wa,--nocompress-debug-sections -fno-builtin-memcmp", + "cxx": "/opt/mongodbtoolchain/v3/bin/g++: g++ (GCC) 8.5.0", + "cxxflags": "-Woverloaded-virtual -Wno-maybe-uninitialized -fsized-deallocation -std=c++17", + "linkflags": "-Wl,--fatal-warnings -pthread -Wl,-z,now -fuse-ld=gold -fstack-protector-strong -Wl,--no-threads -Wl,--build-id -Wl,--hash-style=gnu -Wl,-z,noexecstack -Wl,--warn-execstack -Wl,-z,relro -Wl,--compress-debug-sections=none -Wl,-z,origin -Wl,--enable-new-dtags", + "target_arch": "x86_64", + "target_os": "linux", + "cppdefines": "SAFEINT_USE_INTRINSICS 0 PCRE_STATIC NDEBUG _XOPEN_SOURCE 700 _GNU_SOURCE _REENTRANT 1 _FORTIFY_SOURCE 2 BOOST_THREAD_VERSION 5 BOOST_THREAD_USES_DATETIME BOOST_SYSTEM_NO_DEPRECATED BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS BOOST_ENABLE_ASSERT_DEBUG_HANDLER BOOST_LOG_NO_SHORTHAND_NAMES BOOST_LOG_USE_NATIVE_SYSLOG BOOST_LOG_WITHOUT_THREAD_ATTR ABSL_FORCE_ALIGNED_ACCESS", + }, + "bits": 64, + "debug": false, + "maxBsonObjectSize": 16777216, + "storageEngines": ["devnull", "ephemeralForTest", "wiredTiger"], + "ok": 1.0, + }, + }, + ], + }`, }, { name: "import", @@ -154,6 +212,42 @@ var msgTestCases = []testCase{ }, }, command: "insert", + m: ` + { + "FlagBits": "[]", + "Checksum": int64(0), + "Sections": [ + { + "Kind": 0, + "Document": { + "insert": "actor", + "ordered": true, + "writeConcern": {"w": "majority"}, + "$db": "monila", + }, + }, + { + "Kind": 1, + "Identifier": "documents", + "Documents": [ + { + "_id": ObjectID(612ec2800000000100000001), + "actor_id": 1, + "first_name": "PENELOPE", + "last_name": "GUINESS", + "last_update": 2020-02-15T09:34:33Z, + }, + { + "_id": ObjectID(612ec2800000000200000002), + "actor_id": 2, + "first_name": "NICK", + "last_name": "WAHLBERG", + "last_update": 2020-02-15T09:34:33Z, + }, + ], + }, + ], + }`, }, { name: "msg_fuzz1", @@ -266,6 +360,22 @@ var msgTestCases = []testCase{ }, }, command: "insert", + m: ` + { + "FlagBits": "[]", + "Checksum": int64(0), + "Sections": [ + { + "Kind": 0, + "Document": {"insert": "TestInsertSimple", "ordered": true, "$db": "testinsertsimple"}, + }, + { + "Kind": 1, + "Identifier": "documents", + "Documents": [{"_id": ObjectID(637cfad88dc3cecde38e1e6b), "v": -0.0}], + }, + ], + }`, }, { name: "MultiSectionInsert", @@ -327,6 +437,19 @@ var msgTestCases = []testCase{ checksum: 1737537506, }, command: "insert", + m: ` + { + "FlagBits": "[checksumPresent]", + "Checksum": int64(1737537506), + "Sections": [ + { + "Kind": 1, + "Identifier": "documents", + "Documents": [{"_id": ObjectID(638cec46aa778bf370105429), "a": 3.0}], + }, + {"Kind": 0, "Document": {"insert": "foo", "ordered": true, "$db": "test"}}, + ], + }`, }, { name: "MultiSectionUpdate", @@ -411,6 +534,21 @@ var msgTestCases = []testCase{ checksum: 2932997361, }, command: "update", + m: ` + { + "FlagBits": "[checksumPresent]", + "Checksum": int64(2932997361), + "Sections": [ + { + "Kind": 1, + "Identifier": "updates", + "Documents": [ + {"q": {"a": 20.0}, "u": {"$inc": {"a": 1.0}}, "multi": false, "upsert": false}, + ], + }, + {"Kind": 0, "Document": {"update": "foo", "ordered": true, "$db": "test"}}, + ], + }`, }, { name: "InvalidChecksum", diff --git a/internal/wire/op_query.go b/internal/wire/op_query.go index c573d26849ae..07a28d63a9af 100644 --- a/internal/wire/op_query.go +++ b/internal/wire/op_query.go @@ -16,11 +16,9 @@ package wire import ( "encoding/binary" - "encoding/json" "github.com/FerretDB/FerretDB/internal/bson2" "github.com/FerretDB/FerretDB/internal/types" - "github.com/FerretDB/FerretDB/internal/types/fjson" "github.com/FerretDB/FerretDB/internal/util/debugbuild" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" "github.com/FerretDB/FerretDB/internal/util/must" @@ -159,30 +157,30 @@ func (query *OpQuery) String() string { return "" } - m := map[string]any{ - "Flags": query.Flags, - "FullCollectionName": query.FullCollectionName, - "NumberToSkip": query.NumberToSkip, - "NumberToReturn": query.NumberToReturn, - } + m := must.NotFail(bson2.NewDocument( + "Flags", query.Flags.String(), + "FullCollectionName", query.FullCollectionName, + "NumberToSkip", query.NumberToSkip, + "NumberToReturn", query.NumberToReturn, + )) - doc, err := query.query.Convert() + doc, err := query.query.DecodeDeep() if err == nil { - m["Query"] = json.RawMessage(must.NotFail(fjson.Marshal(doc))) + must.NoError(m.Add("Query", doc)) } else { - m["QueryError"] = err.Error() + must.NoError(m.Add("QueryError", err.Error())) } if query.returnFieldsSelector != nil { - doc, err = query.returnFieldsSelector.Convert() + doc, err = query.returnFieldsSelector.DecodeDeep() if err == nil { - m["ReturnFieldsSelector"] = json.RawMessage(must.NotFail(fjson.Marshal(doc))) + must.NoError(m.Add("ReturnFieldsSelector", doc)) } else { - m["ReturnFieldsSelectorError"] = err.Error() + must.NoError(m.Add("ReturnFieldsSelectorError", err.Error())) } } - return string(must.NotFail(json.MarshalIndent(m, "", " "))) + return m.LogMessage() } // check interfaces diff --git a/internal/wire/op_query_test.go b/internal/wire/op_query_test.go index 0d044e8773e6..df21e6248fe5 100644 --- a/internal/wire/op_query_test.go +++ b/internal/wire/op_query_test.go @@ -61,6 +61,29 @@ var queryTestCases = []testCase{ ), returnFieldsSelector: nil, }, + m: ` + { + "Flags": "[]", + "FullCollectionName": "admin.$cmd", + "NumberToSkip": 0, + "NumberToReturn": -1, + "Query": { + "ismaster": true, + "client": { + "driver": {"name": "nodejs", "version": "4.0.0-beta.6"}, + "os": { + "type": "Darwin", + "name": "darwin", + "architecture": "x64", + "version": "20.6.0", + }, + "platform": "Node.js v14.17.3, LE (unified)|Node.js v14.17.3, LE (unified)", + "application": {"name": "mongosh 1.0.1"}, + }, + "compression": ["none"], + "loadBalanced": false, + }, + }`, }, { name: "handshake3", @@ -100,6 +123,29 @@ var queryTestCases = []testCase{ ), returnFieldsSelector: nil, }, + m: ` + { + "Flags": "[]", + "FullCollectionName": "admin.$cmd", + "NumberToSkip": 0, + "NumberToReturn": -1, + "Query": { + "ismaster": true, + "client": { + "driver": {"name": "nodejs", "version": "4.0.0-beta.6"}, + "os": { + "type": "Darwin", + "name": "darwin", + "architecture": "x64", + "version": "20.6.0", + }, + "platform": "Node.js v14.17.3, LE (unified)|Node.js v14.17.3, LE (unified)", + "application": {"name": "mongosh 1.0.1"}, + }, + "compression": ["none"], + "loadBalanced": false, + }, + }`, }, } diff --git a/internal/wire/op_reply.go b/internal/wire/op_reply.go index 273c00d09d66..62864e07faac 100644 --- a/internal/wire/op_reply.go +++ b/internal/wire/op_reply.go @@ -16,11 +16,9 @@ package wire import ( "encoding/binary" - "encoding/json" "github.com/FerretDB/FerretDB/internal/bson2" "github.com/FerretDB/FerretDB/internal/types" - "github.com/FerretDB/FerretDB/internal/types/fjson" "github.com/FerretDB/FerretDB/internal/util/debugbuild" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" "github.com/FerretDB/FerretDB/internal/util/must" @@ -130,26 +128,26 @@ func (reply *OpReply) String() string { return "" } - m := map[string]any{ - "ResponseFlags": reply.Flags, - "CursorID": reply.CursorID, - "StartingFrom": reply.StartingFrom, - } + m := must.NotFail(bson2.NewDocument( + "ResponseFlags", reply.Flags.String(), + "CursorID", reply.CursorID, + "StartingFrom", reply.StartingFrom, + )) if reply.document == nil { - m["NumberReturned"] = 0 + must.NoError(m.Add("NumberReturned", int32(0))) } else { - m["NumberReturned"] = 1 + must.NoError(m.Add("NumberReturned", int32(1))) - doc, err := reply.document.Convert() + doc, err := reply.document.DecodeDeep() if err == nil { - m["Document"] = json.RawMessage(must.NotFail(fjson.Marshal(doc))) + must.NoError(m.Add("Document", doc)) } else { - m["DocumentError"] = err.Error() + must.NoError(m.Add("DocumentError", err.Error())) } } - return string(must.NotFail(json.MarshalIndent(m, "", " "))) + return m.LogMessage() } // check interfaces diff --git a/internal/wire/op_reply_test.go b/internal/wire/op_reply_test.go index 8927c0bcbf5b..d22c06b4325a 100644 --- a/internal/wire/op_reply_test.go +++ b/internal/wire/op_reply_test.go @@ -56,6 +56,27 @@ var replyTestCases = []testCase{ "ok", float64(1), ), }, + m: ` + { + "ResponseFlags": "[AwaitCapable]", + "CursorID": int64(0), + "StartingFrom": 0, + "NumberReturned": 1, + "Document": { + "ismaster": true, + "topologyVersion": {"processId": ObjectID(60fbed5371fe1bae70339505), "counter": int64(0)}, + "maxBsonObjectSize": 16777216, + "maxMessageSizeBytes": 48000000, + "maxWriteBatchSize": 100000, + "localTime": 2021-07-24T12:54:41.571Z, + "logicalSessionTimeoutMinutes": 30, + "connectionId": 28, + "minWireVersion": 0, + "maxWireVersion": 13, + "readOnly": false, + "ok": 1.0, + }, + }`, }, { name: "handshake4", @@ -89,6 +110,27 @@ var replyTestCases = []testCase{ "ok", float64(1), ), }, + m: ` + { + "ResponseFlags": "[AwaitCapable]", + "CursorID": int64(0), + "StartingFrom": 0, + "NumberReturned": 1, + "Document": { + "ismaster": true, + "topologyVersion": {"processId": ObjectID(60fbed5371fe1bae70339505), "counter": int64(0)}, + "maxBsonObjectSize": 16777216, + "maxMessageSizeBytes": 48000000, + "maxWriteBatchSize": 100000, + "localTime": 2021-07-24T12:54:41.592Z, + "logicalSessionTimeoutMinutes": 30, + "connectionId": 29, + "minWireVersion": 0, + "maxWireVersion": 13, + "readOnly": false, + "ok": 1.0, + }, + }`, }, } diff --git a/internal/wire/wire_test.go b/internal/wire/wire_test.go index a3388a7a1f2b..57d19645be48 100644 --- a/internal/wire/wire_test.go +++ b/internal/wire/wire_test.go @@ -28,6 +28,7 @@ import ( "github.com/FerretDB/FerretDB/internal/bson2" "github.com/FerretDB/FerretDB/internal/types" "github.com/FerretDB/FerretDB/internal/util/must" + "github.com/FerretDB/FerretDB/internal/util/testutil" "github.com/FerretDB/FerretDB/internal/util/testutil/testtb" ) @@ -61,6 +62,7 @@ type testCase struct { msgHeader *MsgHeader msgBody MsgBody command string // only for OpMsg + m string err string // unwrapped } @@ -87,7 +89,6 @@ func (tc *testCase) setExpectedB(tb testtb.TB) { func testMessages(t *testing.T, testCases []testCase) { for _, tc := range testCases { - tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -118,7 +119,7 @@ func testMessages(t *testing.T, testCases []testCase) { require.NotNil(t, msgHeader) require.NotNil(t, msgBody) assert.NotPanics(t, func() { _ = msgHeader.String() }) - assert.NotPanics(t, func() { _ = msgBody.String() }) + assert.Equal(t, testutil.Unindent(t, tc.m), msgBody.String()) require.NoError(t, msgBody.check())