From 6f34de511edc2f4f59c4326576f28e1bf2afd95c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Efe=20=C3=87etin?= Date: Tue, 11 Jan 2022 17:23:59 +0300 Subject: [PATCH] Update goccy/go-json to 0.9.1. (#1709) * Update goccy/go-json to 0.9.1. * fix lint error --- client_test.go | 27 + internal/go-json/cmd/generator/vm.go.tmpl | 27 +- internal/go-json/encode.go | 15 +- internal/go-json/encoder/code.go | 133 ++++- internal/go-json/encoder/compiler.go | 27 + internal/go-json/encoder/compiler_norace.go | 14 +- internal/go-json/encoder/compiler_race.go | 15 +- internal/go-json/encoder/decode_rune.go | 39 +- internal/go-json/encoder/encoder.go | 34 +- internal/go-json/encoder/opcode.go | 2 + internal/go-json/encoder/option.go | 2 + internal/go-json/encoder/query.go | 135 +++++ internal/go-json/encoder/string.go | 539 +++++++----------- internal/go-json/encoder/string_table.go | 415 ++++++++++++++ internal/go-json/encoder/vm/vm.go | 27 +- internal/go-json/encoder/vm_color/vm.go | 27 +- .../go-json/encoder/vm_color_indent/vm.go | 27 +- internal/go-json/encoder/vm_indent/vm.go | 27 +- internal/go-json/error.go | 2 +- internal/go-json/json.go | 5 + internal/go-json/option.go | 17 + internal/go-json/query.go | 47 ++ 22 files changed, 1161 insertions(+), 442 deletions(-) create mode 100644 internal/go-json/encoder/query.go create mode 100644 internal/go-json/encoder/string_table.go create mode 100644 internal/go-json/query.go diff --git a/client_test.go b/client_test.go index db88e8cbf6..730ba2c76a 100644 --- a/client_test.go +++ b/client_test.go @@ -18,6 +18,7 @@ import ( "github.com/gofiber/fiber/v2/internal/go-json" "github.com/gofiber/fiber/v2/internal/tlstest" + "github.com/gofiber/fiber/v2/internal/uuid" "github.com/gofiber/fiber/v2/utils" "github.com/valyala/fasthttp/fasthttputil" ) @@ -591,6 +592,32 @@ func Test_Client_Stdjson_Gojson(t *testing.T) { utils.AssertEqual(t, nil, err) utils.AssertEqual(t, expected, got) + + type config struct { + // debug enable a debug logging. + debug bool + // log used for logging on debug mode. + log func(...interface{}) + } + + type res struct { + config `json:"-"` + // ID of the ent. + ID uuid.UUID `json:"id,omitempty"` + } + + u := uuid.New() + test := res{ + ID: u, + } + + expected, err = stdjson.Marshal(test) + utils.AssertEqual(t, nil, err) + + got, err = json.Marshal(test) + utils.AssertEqual(t, nil, err) + + utils.AssertEqual(t, expected, got) } func Test_Client_Agent_Json(t *testing.T) { diff --git a/internal/go-json/cmd/generator/vm.go.tmpl b/internal/go-json/cmd/generator/vm.go.tmpl index 6c8292d01c..61a30cf655 100644 --- a/internal/go-json/cmd/generator/vm.go.tmpl +++ b/internal/go-json/cmd/generator/vm.go.tmpl @@ -199,7 +199,7 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b break } ctx.KeepRefs = append(ctx.KeepRefs, up) - ifaceCodeSet, err := encoder.CompileToGetCodeSet(uintptr(unsafe.Pointer(typ))) + ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ))) if err != nil { return nil, err } @@ -218,8 +218,7 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b oldOffset := ptrOffset ptrOffset += totalLength * uintptrSize oldBaseIndent := ctx.BaseIndent - indentDiffFromTop := c.Indent - 1 - ctx.BaseIndent += code.Indent - indentDiffFromTop + ctx.BaseIndent += code.Indent newLen := offsetNum + totalLength + nextTotalLength if curlen < newLen { @@ -403,11 +402,12 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b break } b = appendStructHead(ctx, b) - mapCtx := encoder.NewMapContext(mlen) + unorderedMap := (ctx.Option.Flag & encoder.UnorderedMapOption) != 0 + mapCtx := encoder.NewMapContext(mlen, unorderedMap) mapiterinit(code.Type, uptr, &mapCtx.Iter) store(ctxptr, code.Idx, uintptr(unsafe.Pointer(mapCtx))) ctx.KeepRefs = append(ctx.KeepRefs, unsafe.Pointer(mapCtx)) - if (ctx.Option.Flag & encoder.UnorderedMapOption) != 0 { + if unorderedMap { b = appendMapKeyIndent(ctx, code.Next, b) } else { mapCtx.Start = len(b) @@ -705,14 +705,15 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b if code.Flags&encoder.AnonymousHeadFlags == 0 { b = appendStructHead(ctx, b) } - u64 := ptrToUint64(p+uintptr(code.Offset), code.NumBitSize) + p += uintptr(code.Offset) + u64 := ptrToUint64(p, code.NumBitSize) v := u64 & ((1 << code.NumBitSize) - 1) if v == 0 { code = code.NextField } else { b = appendStructKey(ctx, code, b) b = append(b, '"') - b = appendInt(ctx, b, p+uintptr(code.Offset), code) + b = appendInt(ctx, b, p, code) b = append(b, '"') b = appendComma(ctx, b) code = code.Next @@ -2953,9 +2954,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b b = appendStructHead(ctx, b) } b = appendStructKey(ctx, code, b) + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadMarshalJSON { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } if p == 0 && (code.Flags&encoder.NilCheckFlags) != 0 { @@ -2994,9 +2996,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b if code.Flags&encoder.AnonymousHeadFlags == 0 { b = appendStructHead(ctx, b) } + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadOmitEmptyMarshalJSON { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } iface := ptrToInterface(code, p) @@ -3114,9 +3117,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b b = appendStructHead(ctx, b) } b = appendStructKey(ctx, code, b) + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadMarshalText { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } if p == 0 && (code.Flags&encoder.NilCheckFlags) != 0 { @@ -3155,9 +3159,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b if code.Flags&encoder.AnonymousHeadFlags == 0 { b = appendStructHead(ctx, b) } + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadOmitEmptyMarshalText { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } if p == 0 && (code.Flags&encoder.NilCheckFlags) != 0 { diff --git a/internal/go-json/encode.go b/internal/go-json/encode.go index 4f081ffcd6..cb0fc92f75 100644 --- a/internal/go-json/encode.go +++ b/internal/go-json/encode.go @@ -61,6 +61,7 @@ func (e *Encoder) encodeWithOption(ctx *encoder.RuntimeContext, v interface{}, o if e.enabledHTMLEscape { ctx.Option.Flag |= encoder.HTMLEscapeOption } + ctx.Option.Flag |= encoder.NormalizeUTF8Option for _, optFunc := range optFuncs { optFunc(ctx.Option) } @@ -111,7 +112,7 @@ func (e *Encoder) SetIndent(prefix, indent string) { func marshalContext(ctx context.Context, v interface{}, optFuncs ...EncodeOptionFunc) ([]byte, error) { rctx := encoder.TakeRuntimeContext() rctx.Option.Flag = 0 - rctx.Option.Flag = encoder.HTMLEscapeOption | encoder.ContextOption + rctx.Option.Flag = encoder.HTMLEscapeOption | encoder.NormalizeUTF8Option | encoder.ContextOption rctx.Option.Context = ctx for _, optFunc := range optFuncs { optFunc(rctx.Option) @@ -139,7 +140,7 @@ func marshal(v interface{}, optFuncs ...EncodeOptionFunc) ([]byte, error) { ctx := encoder.TakeRuntimeContext() ctx.Option.Flag = 0 - ctx.Option.Flag |= encoder.HTMLEscapeOption + ctx.Option.Flag |= (encoder.HTMLEscapeOption | encoder.NormalizeUTF8Option) for _, optFunc := range optFuncs { optFunc(ctx.Option) } @@ -166,7 +167,7 @@ func marshalNoEscape(v interface{}) ([]byte, error) { ctx := encoder.TakeRuntimeContext() ctx.Option.Flag = 0 - ctx.Option.Flag |= encoder.HTMLEscapeOption + ctx.Option.Flag |= (encoder.HTMLEscapeOption | encoder.NormalizeUTF8Option) buf, err := encodeNoEscape(ctx, v) if err != nil { @@ -190,7 +191,7 @@ func marshalIndent(v interface{}, prefix, indent string, optFuncs ...EncodeOptio ctx := encoder.TakeRuntimeContext() ctx.Option.Flag = 0 - ctx.Option.Flag |= (encoder.HTMLEscapeOption | encoder.IndentOption) + ctx.Option.Flag |= (encoder.HTMLEscapeOption | encoder.NormalizeUTF8Option | encoder.IndentOption) for _, optFunc := range optFuncs { optFunc(ctx.Option) } @@ -220,7 +221,7 @@ func encode(ctx *encoder.RuntimeContext, v interface{}) ([]byte, error) { typ := header.typ typeptr := uintptr(unsafe.Pointer(typ)) - codeSet, err := encoder.CompileToGetCodeSet(typeptr) + codeSet, err := encoder.CompileToGetCodeSet(ctx, typeptr) if err != nil { return nil, err } @@ -248,7 +249,7 @@ func encodeNoEscape(ctx *encoder.RuntimeContext, v interface{}) ([]byte, error) typ := header.typ typeptr := uintptr(unsafe.Pointer(typ)) - codeSet, err := encoder.CompileToGetCodeSet(typeptr) + codeSet, err := encoder.CompileToGetCodeSet(ctx, typeptr) if err != nil { return nil, err } @@ -275,7 +276,7 @@ func encodeIndent(ctx *encoder.RuntimeContext, v interface{}, prefix, indent str typ := header.typ typeptr := uintptr(unsafe.Pointer(typ)) - codeSet, err := encoder.CompileToGetCodeSet(typeptr) + codeSet, err := encoder.CompileToGetCodeSet(ctx, typeptr) if err != nil { return nil, err } diff --git a/internal/go-json/encoder/code.go b/internal/go-json/encoder/code.go index 779b7fbf7b..270c9ea427 100644 --- a/internal/go-json/encoder/code.go +++ b/internal/go-json/encoder/code.go @@ -10,6 +10,7 @@ import ( type Code interface { Kind() CodeKind ToOpcode(*compileContext) Opcodes + Filter(*FieldQuery) Code } type AnonymousCode interface { @@ -82,6 +83,10 @@ func (c *IntCode) ToOpcode(ctx *compileContext) Opcodes { return Opcodes{code} } +func (c *IntCode) Filter(_ *FieldQuery) Code { + return c +} + type UintCode struct { typ *runtime.Type bitSize uint8 @@ -108,6 +113,10 @@ func (c *UintCode) ToOpcode(ctx *compileContext) Opcodes { return Opcodes{code} } +func (c *UintCode) Filter(_ *FieldQuery) Code { + return c +} + type FloatCode struct { typ *runtime.Type bitSize uint8 @@ -140,6 +149,10 @@ func (c *FloatCode) ToOpcode(ctx *compileContext) Opcodes { return Opcodes{code} } +func (c *FloatCode) Filter(_ *FieldQuery) Code { + return c +} + type StringCode struct { typ *runtime.Type isPtr bool @@ -169,6 +182,10 @@ func (c *StringCode) ToOpcode(ctx *compileContext) Opcodes { return Opcodes{code} } +func (c *StringCode) Filter(_ *FieldQuery) Code { + return c +} + type BoolCode struct { typ *runtime.Type isPtr bool @@ -190,6 +207,10 @@ func (c *BoolCode) ToOpcode(ctx *compileContext) Opcodes { return Opcodes{code} } +func (c *BoolCode) Filter(_ *FieldQuery) Code { + return c +} + type BytesCode struct { typ *runtime.Type isPtr bool @@ -211,6 +232,10 @@ func (c *BytesCode) ToOpcode(ctx *compileContext) Opcodes { return Opcodes{code} } +func (c *BytesCode) Filter(_ *FieldQuery) Code { + return c +} + type SliceCode struct { typ *runtime.Type value Code @@ -245,6 +270,10 @@ func (c *SliceCode) ToOpcode(ctx *compileContext) Opcodes { return Opcodes{header}.Add(codes...).Add(elemCode).Add(end) } +func (c *SliceCode) Filter(_ *FieldQuery) Code { + return c +} + type ArrayCode struct { typ *runtime.Type value Code @@ -286,6 +315,10 @@ func (c *ArrayCode) ToOpcode(ctx *compileContext) Opcodes { return Opcodes{header}.Add(codes...).Add(elemCode).Add(end) } +func (c *ArrayCode) Filter(_ *FieldQuery) Code { + return c +} + type MapCode struct { typ *runtime.Type key Code @@ -332,6 +365,10 @@ func (c *MapCode) ToOpcode(ctx *compileContext) Opcodes { return Opcodes{header}.Add(keyCodes...).Add(value).Add(valueCodes...).Add(key).Add(end) } +func (c *MapCode) Filter(_ *FieldQuery) Code { + return c +} + type StructCode struct { typ *runtime.Type fields []*StructFieldCode @@ -520,6 +557,45 @@ func (c *StructCode) enableIndirect() { structCode.enableIndirect() } +func (c *StructCode) Filter(query *FieldQuery) Code { + fieldMap := map[string]*FieldQuery{} + for _, field := range query.Fields { + fieldMap[field.Name] = field + } + fields := make([]*StructFieldCode, 0, len(c.fields)) + for _, field := range c.fields { + query, exists := fieldMap[field.key] + if !exists { + continue + } + fieldCode := &StructFieldCode{ + typ: field.typ, + key: field.key, + tag: field.tag, + value: field.value, + offset: field.offset, + isAnonymous: field.isAnonymous, + isTaggedKey: field.isTaggedKey, + isNilableType: field.isNilableType, + isNilCheck: field.isNilCheck, + isAddrForMarshaler: field.isAddrForMarshaler, + isNextOpPtrType: field.isNextOpPtrType, + } + if len(query.Fields) > 0 { + fieldCode.value = fieldCode.value.Filter(query) + } + fields = append(fields, fieldCode) + } + return &StructCode{ + typ: c.typ, + fields: fields, + isPtr: c.isPtr, + disableIndirectConversion: c.disableIndirectConversion, + isIndirect: c.isIndirect, + isRecursive: c.isRecursive, + } +} + type StructFieldCode struct { typ *runtime.Type key string @@ -532,6 +608,7 @@ type StructFieldCode struct { isNilCheck bool isAddrForMarshaler bool isNextOpPtrType bool + isMarshalerContext bool } func (c *StructFieldCode) getStruct() *StructCode { @@ -574,8 +651,12 @@ func (c *StructFieldCode) headerOpcodes(ctx *compileContext, field *Opcode, valu value := valueCodes.First() op := optimizeStructHeader(value, c.tag) field.Op = op + if value.Flags&MarshalerContextFlags != 0 { + field.Flags |= MarshalerContextFlags + } field.NumBitSize = value.NumBitSize field.PtrNum = value.PtrNum + field.FieldQuery = value.FieldQuery fieldCodes := Opcodes{field} if op.IsMultipleOpHead() { field.Next = value @@ -590,8 +671,12 @@ func (c *StructFieldCode) fieldOpcodes(ctx *compileContext, field *Opcode, value value := valueCodes.First() op := optimizeStructField(value, c.tag) field.Op = op + if value.Flags&MarshalerContextFlags != 0 { + field.Flags |= MarshalerContextFlags + } field.NumBitSize = value.NumBitSize field.PtrNum = value.PtrNum + field.FieldQuery = value.FieldQuery fieldCodes := Opcodes{field} if op.IsMultipleOpField() { @@ -645,6 +730,9 @@ func (c *StructFieldCode) flags() OpFlags { if c.isAnonymous { flags |= AnonymousKeyFlags } + if c.isMarshalerContext { + flags |= MarshalerContextFlags + } return flags } @@ -725,8 +813,9 @@ func isEnableStructEndOptimization(value Code) bool { } type InterfaceCode struct { - typ *runtime.Type - isPtr bool + typ *runtime.Type + fieldQuery *FieldQuery + isPtr bool } func (c *InterfaceCode) Kind() CodeKind { @@ -741,6 +830,7 @@ func (c *InterfaceCode) ToOpcode(ctx *compileContext) Opcodes { default: code = newOpCode(ctx, c.typ, OpInterface) } + code.FieldQuery = c.fieldQuery if c.typ.NumMethod() > 0 { code.Flags |= NonEmptyInterfaceFlags } @@ -748,8 +838,17 @@ func (c *InterfaceCode) ToOpcode(ctx *compileContext) Opcodes { return Opcodes{code} } +func (c *InterfaceCode) Filter(query *FieldQuery) Code { + return &InterfaceCode{ + typ: c.typ, + fieldQuery: query, + isPtr: c.isPtr, + } +} + type MarshalJSONCode struct { typ *runtime.Type + fieldQuery *FieldQuery isAddrForMarshaler bool isNilableType bool isMarshalerContext bool @@ -761,6 +860,7 @@ func (c *MarshalJSONCode) Kind() CodeKind { func (c *MarshalJSONCode) ToOpcode(ctx *compileContext) Opcodes { code := newOpCode(ctx, c.typ, OpMarshalJSON) + code.FieldQuery = c.fieldQuery if c.isAddrForMarshaler { code.Flags |= AddrForMarshalerFlags } @@ -776,8 +876,19 @@ func (c *MarshalJSONCode) ToOpcode(ctx *compileContext) Opcodes { return Opcodes{code} } +func (c *MarshalJSONCode) Filter(query *FieldQuery) Code { + return &MarshalJSONCode{ + typ: c.typ, + fieldQuery: query, + isAddrForMarshaler: c.isAddrForMarshaler, + isNilableType: c.isNilableType, + isMarshalerContext: c.isMarshalerContext, + } +} + type MarshalTextCode struct { typ *runtime.Type + fieldQuery *FieldQuery isAddrForMarshaler bool isNilableType bool } @@ -788,6 +899,7 @@ func (c *MarshalTextCode) Kind() CodeKind { func (c *MarshalTextCode) ToOpcode(ctx *compileContext) Opcodes { code := newOpCode(ctx, c.typ, OpMarshalText) + code.FieldQuery = c.fieldQuery if c.isAddrForMarshaler { code.Flags |= AddrForMarshalerFlags } @@ -800,6 +912,15 @@ func (c *MarshalTextCode) ToOpcode(ctx *compileContext) Opcodes { return Opcodes{code} } +func (c *MarshalTextCode) Filter(query *FieldQuery) Code { + return &MarshalTextCode{ + typ: c.typ, + fieldQuery: query, + isAddrForMarshaler: c.isAddrForMarshaler, + isNilableType: c.isNilableType, + } +} + type PtrCode struct { typ *runtime.Type value Code @@ -830,6 +951,14 @@ func (c *PtrCode) ToAnonymousOpcode(ctx *compileContext) Opcodes { return codes } +func (c *PtrCode) Filter(query *FieldQuery) Code { + return &PtrCode{ + typ: c.typ, + value: c.value.Filter(query), + ptrNum: c.ptrNum, + } +} + func convertPtrOp(code *Opcode) OpType { ptrHeadOp := code.Op.HeadToPtrHead() if code.Op != ptrHeadOp { diff --git a/internal/go-json/encoder/compiler.go b/internal/go-json/encoder/compiler.go index 9ee137b4ff..92040bef85 100644 --- a/internal/go-json/encoder/compiler.go +++ b/internal/go-json/encoder/compiler.go @@ -63,6 +63,27 @@ func compileToGetCodeSetSlowPath(typeptr uintptr) (*OpcodeSet, error) { return codeSet, nil } +func getFilteredCodeSetIfNeeded(ctx *RuntimeContext, codeSet *OpcodeSet) (*OpcodeSet, error) { + if (ctx.Option.Flag & ContextOption) == 0 { + return codeSet, nil + } + query := FieldQueryFromContext(ctx.Option.Context) + if query == nil { + return codeSet, nil + } + ctx.Option.Flag |= FieldQueryOption + cacheCodeSet := codeSet.getQueryCache(query.Hash()) + if cacheCodeSet != nil { + return cacheCodeSet, nil + } + queryCodeSet, err := newCompiler().codeToOpcodeSet(codeSet.Type, codeSet.Code.Filter(query)) + if err != nil { + return nil, err + } + codeSet.setQueryCache(query.Hash(), queryCodeSet) + return queryCodeSet, nil +} + type Compiler struct { structTypeToCode map[uintptr]*StructCode } @@ -80,6 +101,10 @@ func (c *Compiler) compile(typeptr uintptr) (*OpcodeSet, error) { if err != nil { return nil, err } + return c.codeToOpcodeSet(typ, code) +} + +func (c *Compiler) codeToOpcodeSet(typ *runtime.Type, code Code) (*OpcodeSet, error) { noescapeKeyCode := c.codeToOpcode(&compileContext{ structTypeToCodes: map[uintptr]Opcodes{}, recursiveCodes: &Opcodes{}, @@ -107,6 +132,8 @@ func (c *Compiler) compile(typeptr uintptr) (*OpcodeSet, error) { InterfaceEscapeKeyCode: interfaceEscapeKeyCode, CodeLength: codeLength, EndCode: ToEndCode(interfaceNoescapeKeyCode), + Code: code, + QueryCache: map[string]*OpcodeSet{}, }, nil } diff --git a/internal/go-json/encoder/compiler_norace.go b/internal/go-json/encoder/compiler_norace.go index afc5b6623a..2576419aee 100644 --- a/internal/go-json/encoder/compiler_norace.go +++ b/internal/go-json/encoder/compiler_norace.go @@ -3,18 +3,26 @@ package encoder -func CompileToGetCodeSet(typeptr uintptr) (*OpcodeSet, error) { +func CompileToGetCodeSet(ctx *RuntimeContext, typeptr uintptr) (*OpcodeSet, error) { if typeptr > typeAddr.MaxTypeAddr { return compileToGetCodeSetSlowPath(typeptr) } index := (typeptr - typeAddr.BaseTypeAddr) >> typeAddr.AddrShift if codeSet := cachedOpcodeSets[index]; codeSet != nil { - return codeSet, nil + filtered, err := getFilteredCodeSetIfNeeded(ctx, codeSet) + if err != nil { + return nil, err + } + return filtered, nil } codeSet, err := newCompiler().compile(typeptr) if err != nil { return nil, err } + filtered, err := getFilteredCodeSetIfNeeded(ctx, codeSet) + if err != nil { + return nil, err + } cachedOpcodeSets[index] = codeSet - return codeSet, nil + return filtered, nil } diff --git a/internal/go-json/encoder/compiler_race.go b/internal/go-json/encoder/compiler_race.go index 846a898dda..c74451116d 100644 --- a/internal/go-json/encoder/compiler_race.go +++ b/internal/go-json/encoder/compiler_race.go @@ -9,15 +9,20 @@ import ( var setsMu sync.RWMutex -func CompileToGetCodeSet(typeptr uintptr) (*OpcodeSet, error) { +func CompileToGetCodeSet(ctx *RuntimeContext, typeptr uintptr) (*OpcodeSet, error) { if typeptr > typeAddr.MaxTypeAddr { return compileToGetCodeSetSlowPath(typeptr) } index := (typeptr - typeAddr.BaseTypeAddr) >> typeAddr.AddrShift setsMu.RLock() if codeSet := cachedOpcodeSets[index]; codeSet != nil { + filtered, err := getFilteredCodeSetIfNeeded(ctx, codeSet) + if err != nil { + setsMu.RUnlock() + return nil, err + } setsMu.RUnlock() - return codeSet, nil + return filtered, nil } setsMu.RUnlock() @@ -25,8 +30,12 @@ func CompileToGetCodeSet(typeptr uintptr) (*OpcodeSet, error) { if err != nil { return nil, err } + filtered, err := getFilteredCodeSetIfNeeded(ctx, codeSet) + if err != nil { + return nil, err + } setsMu.Lock() cachedOpcodeSets[index] = codeSet setsMu.Unlock() - return codeSet, nil + return filtered, nil } diff --git a/internal/go-json/encoder/decode_rune.go b/internal/go-json/encoder/decode_rune.go index 1087f0b616..35c959d481 100644 --- a/internal/go-json/encoder/decode_rune.go +++ b/internal/go-json/encoder/decode_rune.go @@ -44,13 +44,6 @@ var first = [256]uint8{ s5, s6, s6, s6, s7, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, // 0xF0-0xFF } -// acceptRange gives the range of valid values for the second byte in a UTF-8 -// sequence. -type acceptRange struct { - lo uint8 // lowest value for second byte. - hi uint8 // highest value for second byte. -} - const ( lineSep = byte(168) //'\u2028' paragraphSep = byte(169) //'\u2029' @@ -80,25 +73,31 @@ func decodeRuneInString(s string) (decodeRuneState, int) { return validUTF8State, 1 } sz := int(x & 7) - var accept acceptRange + if n < sz { + return runeErrorState, 1 + } + s1 := s[1] switch x >> 4 { case 0: - accept = acceptRange{locb, hicb} + if s1 < locb || hicb < s1 { + return runeErrorState, 1 + } case 1: - accept = acceptRange{0xA0, hicb} + if s1 < 0xA0 || hicb < s1 { + return runeErrorState, 1 + } case 2: - accept = acceptRange{locb, 0x9F} + if s1 < locb || 0x9F < s1 { + return runeErrorState, 1 + } case 3: - accept = acceptRange{0x90, hicb} + if s1 < 0x90 || hicb < s1 { + return runeErrorState, 1 + } case 4: - accept = acceptRange{locb, 0x8F} - } - if n < sz { - return runeErrorState, 1 - } - s1 := s[1] - if s1 < accept.lo || accept.hi < s1 { - return runeErrorState, 1 + if s1 < locb || 0x8F < s1 { + return runeErrorState, 1 + } } if sz <= 2 { return validUTF8State, 2 diff --git a/internal/go-json/encoder/encoder.go b/internal/go-json/encoder/encoder.go index f4d23ebb73..495971caba 100644 --- a/internal/go-json/encoder/encoder.go +++ b/internal/go-json/encoder/encoder.go @@ -101,6 +101,22 @@ type OpcodeSet struct { InterfaceEscapeKeyCode *Opcode CodeLength int EndCode *Opcode + Code Code + QueryCache map[string]*OpcodeSet + cacheMu sync.RWMutex +} + +func (s *OpcodeSet) getQueryCache(hash string) *OpcodeSet { + s.cacheMu.RLock() + codeSet := s.QueryCache[hash] + s.cacheMu.RUnlock() + return codeSet +} + +func (s *OpcodeSet) setQueryCache(hash string, codeSet *OpcodeSet) { + s.cacheMu.Lock() + s.QueryCache[hash] = codeSet + s.cacheMu.Unlock() } type CompiledCode struct { @@ -259,12 +275,14 @@ var mapContextPool = sync.Pool{ }, } -func NewMapContext(mapLen int) *MapContext { +func NewMapContext(mapLen int, unorderedMap bool) *MapContext { ctx := mapContextPool.Get().(*MapContext) - if len(ctx.Slice.Items) < mapLen { - ctx.Slice.Items = make([]MapItem, mapLen) - } else { - ctx.Slice.Items = ctx.Slice.Items[:mapLen] + if !unorderedMap { + if len(ctx.Slice.Items) < mapLen { + ctx.Slice.Items = make([]MapItem, mapLen) + } else { + ctx.Slice.Items = ctx.Slice.Items[:mapLen] + } } ctx.Buf = ctx.Buf[:0] ctx.Iter = mapIter{} @@ -395,7 +413,11 @@ func AppendMarshalJSON(ctx *RuntimeContext, code *Opcode, b []byte, v interface{ if !ok { return AppendNull(ctx, b), nil } - b, err := marshaler.MarshalJSON(ctx.Option.Context) + stdctx := ctx.Option.Context + if ctx.Option.Flag&FieldQueryOption != 0 { + stdctx = SetFieldQueryToContext(stdctx, code.FieldQuery) + } + b, err := marshaler.MarshalJSON(stdctx) if err != nil { return nil, &errors.MarshalerError{Type: reflect.TypeOf(v), Err: err} } diff --git a/internal/go-json/encoder/opcode.go b/internal/go-json/encoder/opcode.go index 8b8c8b0678..3d0707a392 100644 --- a/internal/go-json/encoder/opcode.go +++ b/internal/go-json/encoder/opcode.go @@ -39,6 +39,7 @@ type Opcode struct { Type *runtime.Type // go type Jmp *CompiledCode // for recursive call + FieldQuery *FieldQuery // field query for Interface / MarshalJSON / MarshalText ElemIdx uint32 // offset to access array/slice elem Length uint32 // offset to access slice length or array length Indent uint32 // indent number @@ -333,6 +334,7 @@ func copyOpcode(code *Opcode) *Opcode { Idx: c.Idx, Offset: c.Offset, Type: c.Type, + FieldQuery: c.FieldQuery, DisplayIdx: c.DisplayIdx, DisplayKey: c.DisplayKey, ElemIdx: c.ElemIdx, diff --git a/internal/go-json/encoder/option.go b/internal/go-json/encoder/option.go index f5f1f044e3..dcec8f20b1 100644 --- a/internal/go-json/encoder/option.go +++ b/internal/go-json/encoder/option.go @@ -11,6 +11,8 @@ const ( DebugOption ColorizeOption ContextOption + NormalizeUTF8Option + FieldQueryOption ) type Option struct { diff --git a/internal/go-json/encoder/query.go b/internal/go-json/encoder/query.go new file mode 100644 index 0000000000..1e1850cc15 --- /dev/null +++ b/internal/go-json/encoder/query.go @@ -0,0 +1,135 @@ +package encoder + +import ( + "context" + "fmt" + "reflect" +) + +var ( + Marshal func(interface{}) ([]byte, error) + Unmarshal func([]byte, interface{}) error +) + +type FieldQuery struct { + Name string + Fields []*FieldQuery + hash string +} + +func (q *FieldQuery) Hash() string { + if q.hash != "" { + return q.hash + } + b, _ := Marshal(q) + q.hash = string(b) + return q.hash +} + +func (q *FieldQuery) MarshalJSON() ([]byte, error) { + if q.Name != "" { + if len(q.Fields) > 0 { + return Marshal(map[string][]*FieldQuery{q.Name: q.Fields}) + } + return Marshal(q.Name) + } + return Marshal(q.Fields) +} + +func (q *FieldQuery) QueryString() (FieldQueryString, error) { + b, err := Marshal(q) + if err != nil { + return "", err + } + return FieldQueryString(b), nil +} + +type FieldQueryString string + +func (s FieldQueryString) Build() (*FieldQuery, error) { + var query interface{} + if err := Unmarshal([]byte(s), &query); err != nil { + return nil, err + } + return s.build(reflect.ValueOf(query)) +} + +func (s FieldQueryString) build(v reflect.Value) (*FieldQuery, error) { + switch v.Type().Kind() { + case reflect.String: + return s.buildString(v) + case reflect.Map: + return s.buildMap(v) + case reflect.Slice: + return s.buildSlice(v) + case reflect.Interface: + return s.build(reflect.ValueOf(v.Interface())) + } + return nil, fmt.Errorf("failed to build field query") +} + +func (s FieldQueryString) buildString(v reflect.Value) (*FieldQuery, error) { + b := []byte(v.String()) + switch b[0] { + case '[', '{': + var query interface{} + if err := Unmarshal(b, &query); err != nil { + return nil, err + } + if str, ok := query.(string); ok { + return &FieldQuery{Name: str}, nil + } + return s.build(reflect.ValueOf(query)) + } + return &FieldQuery{Name: string(b)}, nil +} + +func (s FieldQueryString) buildSlice(v reflect.Value) (*FieldQuery, error) { + fields := make([]*FieldQuery, 0, v.Len()) + for i := 0; i < v.Len(); i++ { + def, err := s.build(v.Index(i)) + if err != nil { + return nil, err + } + fields = append(fields, def) + } + return &FieldQuery{Fields: fields}, nil +} + +func (s FieldQueryString) buildMap(v reflect.Value) (*FieldQuery, error) { + keys := v.MapKeys() + if len(keys) != 1 { + return nil, fmt.Errorf("failed to build field query object") + } + key := keys[0] + if key.Type().Kind() != reflect.String { + return nil, fmt.Errorf("failed to build field query. invalid object key type") + } + name := key.String() + def, err := s.build(v.MapIndex(key)) + if err != nil { + return nil, err + } + return &FieldQuery{ + Name: name, + Fields: def.Fields, + }, nil +} + +type queryKey struct{} + +func FieldQueryFromContext(ctx context.Context) *FieldQuery { + query := ctx.Value(queryKey{}) + if query == nil { + return nil + } + q, ok := query.(*FieldQuery) + if !ok { + return nil + } + return q +} + +func SetFieldQueryToContext(ctx context.Context, query *FieldQuery) context.Context { + return context.WithValue(ctx, queryKey{}, query) +} diff --git a/internal/go-json/encoder/string.go b/internal/go-json/encoder/string.go index 236e2e9927..e4152b27c7 100644 --- a/internal/go-json/encoder/string.go +++ b/internal/go-json/encoder/string.go @@ -11,341 +11,6 @@ const ( msb = 0x8080808080808080 ) -var needEscapeWithHTML = [256]bool{ - '"': true, - '&': true, - '<': true, - '>': true, - '\\': true, - 0x00: true, - 0x01: true, - 0x02: true, - 0x03: true, - 0x04: true, - 0x05: true, - 0x06: true, - 0x07: true, - 0x08: true, - 0x09: true, - 0x0a: true, - 0x0b: true, - 0x0c: true, - 0x0d: true, - 0x0e: true, - 0x0f: true, - 0x10: true, - 0x11: true, - 0x12: true, - 0x13: true, - 0x14: true, - 0x15: true, - 0x16: true, - 0x17: true, - 0x18: true, - 0x19: true, - 0x1a: true, - 0x1b: true, - 0x1c: true, - 0x1d: true, - 0x1e: true, - 0x1f: true, - /* 0x20 - 0x7f */ - 0x80: true, - 0x81: true, - 0x82: true, - 0x83: true, - 0x84: true, - 0x85: true, - 0x86: true, - 0x87: true, - 0x88: true, - 0x89: true, - 0x8a: true, - 0x8b: true, - 0x8c: true, - 0x8d: true, - 0x8e: true, - 0x8f: true, - 0x90: true, - 0x91: true, - 0x92: true, - 0x93: true, - 0x94: true, - 0x95: true, - 0x96: true, - 0x97: true, - 0x98: true, - 0x99: true, - 0x9a: true, - 0x9b: true, - 0x9c: true, - 0x9d: true, - 0x9e: true, - 0x9f: true, - 0xa0: true, - 0xa1: true, - 0xa2: true, - 0xa3: true, - 0xa4: true, - 0xa5: true, - 0xa6: true, - 0xa7: true, - 0xa8: true, - 0xa9: true, - 0xaa: true, - 0xab: true, - 0xac: true, - 0xad: true, - 0xae: true, - 0xaf: true, - 0xb0: true, - 0xb1: true, - 0xb2: true, - 0xb3: true, - 0xb4: true, - 0xb5: true, - 0xb6: true, - 0xb7: true, - 0xb8: true, - 0xb9: true, - 0xba: true, - 0xbb: true, - 0xbc: true, - 0xbd: true, - 0xbe: true, - 0xbf: true, - 0xc0: true, - 0xc1: true, - 0xc2: true, - 0xc3: true, - 0xc4: true, - 0xc5: true, - 0xc6: true, - 0xc7: true, - 0xc8: true, - 0xc9: true, - 0xca: true, - 0xcb: true, - 0xcc: true, - 0xcd: true, - 0xce: true, - 0xcf: true, - 0xd0: true, - 0xd1: true, - 0xd2: true, - 0xd3: true, - 0xd4: true, - 0xd5: true, - 0xd6: true, - 0xd7: true, - 0xd8: true, - 0xd9: true, - 0xda: true, - 0xdb: true, - 0xdc: true, - 0xdd: true, - 0xde: true, - 0xdf: true, - 0xe0: true, - 0xe1: true, - 0xe2: true, - 0xe3: true, - 0xe4: true, - 0xe5: true, - 0xe6: true, - 0xe7: true, - 0xe8: true, - 0xe9: true, - 0xea: true, - 0xeb: true, - 0xec: true, - 0xed: true, - 0xee: true, - 0xef: true, - 0xf0: true, - 0xf1: true, - 0xf2: true, - 0xf3: true, - 0xf4: true, - 0xf5: true, - 0xf6: true, - 0xf7: true, - 0xf8: true, - 0xf9: true, - 0xfa: true, - 0xfb: true, - 0xfc: true, - 0xfd: true, - 0xfe: true, - 0xff: true, -} - -var needEscape = [256]bool{ - '"': true, - '\\': true, - 0x00: true, - 0x01: true, - 0x02: true, - 0x03: true, - 0x04: true, - 0x05: true, - 0x06: true, - 0x07: true, - 0x08: true, - 0x09: true, - 0x0a: true, - 0x0b: true, - 0x0c: true, - 0x0d: true, - 0x0e: true, - 0x0f: true, - 0x10: true, - 0x11: true, - 0x12: true, - 0x13: true, - 0x14: true, - 0x15: true, - 0x16: true, - 0x17: true, - 0x18: true, - 0x19: true, - 0x1a: true, - 0x1b: true, - 0x1c: true, - 0x1d: true, - 0x1e: true, - 0x1f: true, - /* 0x20 - 0x7f */ - 0x80: true, - 0x81: true, - 0x82: true, - 0x83: true, - 0x84: true, - 0x85: true, - 0x86: true, - 0x87: true, - 0x88: true, - 0x89: true, - 0x8a: true, - 0x8b: true, - 0x8c: true, - 0x8d: true, - 0x8e: true, - 0x8f: true, - 0x90: true, - 0x91: true, - 0x92: true, - 0x93: true, - 0x94: true, - 0x95: true, - 0x96: true, - 0x97: true, - 0x98: true, - 0x99: true, - 0x9a: true, - 0x9b: true, - 0x9c: true, - 0x9d: true, - 0x9e: true, - 0x9f: true, - 0xa0: true, - 0xa1: true, - 0xa2: true, - 0xa3: true, - 0xa4: true, - 0xa5: true, - 0xa6: true, - 0xa7: true, - 0xa8: true, - 0xa9: true, - 0xaa: true, - 0xab: true, - 0xac: true, - 0xad: true, - 0xae: true, - 0xaf: true, - 0xb0: true, - 0xb1: true, - 0xb2: true, - 0xb3: true, - 0xb4: true, - 0xb5: true, - 0xb6: true, - 0xb7: true, - 0xb8: true, - 0xb9: true, - 0xba: true, - 0xbb: true, - 0xbc: true, - 0xbd: true, - 0xbe: true, - 0xbf: true, - 0xc0: true, - 0xc1: true, - 0xc2: true, - 0xc3: true, - 0xc4: true, - 0xc5: true, - 0xc6: true, - 0xc7: true, - 0xc8: true, - 0xc9: true, - 0xca: true, - 0xcb: true, - 0xcc: true, - 0xcd: true, - 0xce: true, - 0xcf: true, - 0xd0: true, - 0xd1: true, - 0xd2: true, - 0xd3: true, - 0xd4: true, - 0xd5: true, - 0xd6: true, - 0xd7: true, - 0xd8: true, - 0xd9: true, - 0xda: true, - 0xdb: true, - 0xdc: true, - 0xdd: true, - 0xde: true, - 0xdf: true, - 0xe0: true, - 0xe1: true, - 0xe2: true, - 0xe3: true, - 0xe4: true, - 0xe5: true, - 0xe6: true, - 0xe7: true, - 0xe8: true, - 0xe9: true, - 0xea: true, - 0xeb: true, - 0xec: true, - 0xed: true, - 0xee: true, - 0xef: true, - 0xf0: true, - 0xf1: true, - 0xf2: true, - 0xf3: true, - 0xf4: true, - 0xf5: true, - 0xf6: true, - 0xf7: true, - 0xf8: true, - 0xf9: true, - 0xfa: true, - 0xfb: true, - 0xfc: true, - 0xfd: true, - 0xfe: true, - 0xff: true, -} - var hex = "0123456789abcdef" //nolint:govet @@ -358,9 +23,19 @@ func stringToUint64Slice(s string) []uint64 { } func AppendString(ctx *RuntimeContext, buf []byte, s string) []byte { - if ctx.Option.Flag&HTMLEscapeOption == 0 { - return appendString(buf, s) + if ctx.Option.Flag&HTMLEscapeOption != 0 { + if ctx.Option.Flag&NormalizeUTF8Option != 0 { + return appendNormalizedHTMLString(buf, s) + } + return appendHTMLString(buf, s) + } + if ctx.Option.Flag&NormalizeUTF8Option != 0 { + return appendNormalizedString(buf, s) } + return appendString(buf, s) +} + +func appendNormalizedHTMLString(buf []byte, s string) []byte { valLen := len(s) if valLen == 0 { return append(buf, `""`...) @@ -387,7 +62,7 @@ func AppendString(ctx *RuntimeContext, buf []byte, s string) []byte { } } for i := len(chunks) * 8; i < valLen; i++ { - if needEscapeWithHTML[s[i]] { + if needEscapeHTMLNormalizeUTF8[s[i]] { j = i goto ESCAPE_END } @@ -399,7 +74,7 @@ ESCAPE_END: for j < valLen { c := s[j] - if !needEscapeWithHTML[c] { + if !needEscapeHTMLNormalizeUTF8[c] { // fast path: most of the time, printable ascii characters are used j++ continue @@ -451,7 +126,6 @@ ESCAPE_END: j = j + 1 continue } - state, size := decodeRuneInString(s[j:]) switch state { case runeErrorState: @@ -486,7 +160,7 @@ ESCAPE_END: return append(append(buf, s[i:]...), '"') } -func appendString(buf []byte, s string) []byte { +func appendHTMLString(buf []byte, s string) []byte { valLen := len(s) if valLen == 0 { return append(buf, `""`...) @@ -503,26 +177,29 @@ func appendString(buf []byte, s string) []byte { // set (i.e. the byte was outside the ASCII range). mask := n | (n - (lsb * 0x20)) | ((n ^ (lsb * '"')) - lsb) | - ((n ^ (lsb * '\\')) - lsb) + ((n ^ (lsb * '\\')) - lsb) | + ((n ^ (lsb * '<')) - lsb) | + ((n ^ (lsb * '>')) - lsb) | + ((n ^ (lsb * '&')) - lsb) if (mask & msb) != 0 { j = bits.TrailingZeros64(mask&msb) / 8 goto ESCAPE_END } } - valLen := len(s) for i := len(chunks) * 8; i < valLen; i++ { - if needEscape[s[i]] { + if needEscapeHTML[s[i]] { j = i goto ESCAPE_END } } + // no found any escape characters. return append(append(buf, s...), '"') } ESCAPE_END: for j < valLen { c := s[j] - if !needEscape[c] { + if !needEscapeHTML[c] { // fast path: most of the time, printable ascii characters are used j++ continue @@ -565,6 +242,92 @@ ESCAPE_END: j = j + 1 continue + case 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0B, 0x0C, 0x0E, 0x0F, // 0x00-0x0F + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F: // 0x10-0x1F + buf = append(buf, s[i:j]...) + buf = append(buf, `\u00`...) + buf = append(buf, hex[c>>4], hex[c&0xF]) + i = j + 1 + j = j + 1 + continue + } + j++ + } + + return append(append(buf, s[i:]...), '"') +} + +func appendNormalizedString(buf []byte, s string) []byte { + valLen := len(s) + if valLen == 0 { + return append(buf, `""`...) + } + buf = append(buf, '"') + var ( + i, j int + ) + if valLen >= 8 { + chunks := stringToUint64Slice(s) + for _, n := range chunks { + // combine masks before checking for the MSB of each byte. We include + // `n` in the mask to check whether any of the *input* byte MSBs were + // set (i.e. the byte was outside the ASCII range). + mask := n | (n - (lsb * 0x20)) | + ((n ^ (lsb * '"')) - lsb) | + ((n ^ (lsb * '\\')) - lsb) + if (mask & msb) != 0 { + j = bits.TrailingZeros64(mask&msb) / 8 + goto ESCAPE_END + } + } + valLen := len(s) + for i := len(chunks) * 8; i < valLen; i++ { + if needEscapeNormalizeUTF8[s[i]] { + j = i + goto ESCAPE_END + } + } + return append(append(buf, s...), '"') + } +ESCAPE_END: + for j < valLen { + c := s[j] + + if !needEscapeNormalizeUTF8[c] { + // fast path: most of the time, printable ascii characters are used + j++ + continue + } + + switch c { + case '\\', '"': + buf = append(buf, s[i:j]...) + buf = append(buf, '\\', c) + i = j + 1 + j = j + 1 + continue + + case '\n': + buf = append(buf, s[i:j]...) + buf = append(buf, '\\', 'n') + i = j + 1 + j = j + 1 + continue + + case '\r': + buf = append(buf, s[i:j]...) + buf = append(buf, '\\', 'r') + i = j + 1 + j = j + 1 + continue + + case '\t': + buf = append(buf, s[i:j]...) + buf = append(buf, '\\', 't') + i = j + 1 + j = j + 1 + continue + case 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0B, 0x0C, 0x0E, 0x0F, // 0x00-0x0F 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F: // 0x10-0x1F buf = append(buf, s[i:j]...) @@ -608,3 +371,89 @@ ESCAPE_END: return append(append(buf, s[i:]...), '"') } + +func appendString(buf []byte, s string) []byte { + valLen := len(s) + if valLen == 0 { + return append(buf, `""`...) + } + buf = append(buf, '"') + var ( + i, j int + ) + if valLen >= 8 { + chunks := stringToUint64Slice(s) + for _, n := range chunks { + // combine masks before checking for the MSB of each byte. We include + // `n` in the mask to check whether any of the *input* byte MSBs were + // set (i.e. the byte was outside the ASCII range). + mask := n | (n - (lsb * 0x20)) | + ((n ^ (lsb * '"')) - lsb) | + ((n ^ (lsb * '\\')) - lsb) + if (mask & msb) != 0 { + j = bits.TrailingZeros64(mask&msb) / 8 + goto ESCAPE_END + } + } + valLen := len(s) + for i := len(chunks) * 8; i < valLen; i++ { + if needEscape[s[i]] { + j = i + goto ESCAPE_END + } + } + return append(append(buf, s...), '"') + } +ESCAPE_END: + for j < valLen { + c := s[j] + + if !needEscape[c] { + // fast path: most of the time, printable ascii characters are used + j++ + continue + } + + switch c { + case '\\', '"': + buf = append(buf, s[i:j]...) + buf = append(buf, '\\', c) + i = j + 1 + j = j + 1 + continue + + case '\n': + buf = append(buf, s[i:j]...) + buf = append(buf, '\\', 'n') + i = j + 1 + j = j + 1 + continue + + case '\r': + buf = append(buf, s[i:j]...) + buf = append(buf, '\\', 'r') + i = j + 1 + j = j + 1 + continue + + case '\t': + buf = append(buf, s[i:j]...) + buf = append(buf, '\\', 't') + i = j + 1 + j = j + 1 + continue + + case 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0B, 0x0C, 0x0E, 0x0F, // 0x00-0x0F + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F: // 0x10-0x1F + buf = append(buf, s[i:j]...) + buf = append(buf, `\u00`...) + buf = append(buf, hex[c>>4], hex[c&0xF]) + i = j + 1 + j = j + 1 + continue + } + j++ + } + + return append(append(buf, s[i:]...), '"') +} diff --git a/internal/go-json/encoder/string_table.go b/internal/go-json/encoder/string_table.go new file mode 100644 index 0000000000..ebe42c92df --- /dev/null +++ b/internal/go-json/encoder/string_table.go @@ -0,0 +1,415 @@ +package encoder + +var needEscapeHTMLNormalizeUTF8 = [256]bool{ + '"': true, + '&': true, + '<': true, + '>': true, + '\\': true, + 0x00: true, + 0x01: true, + 0x02: true, + 0x03: true, + 0x04: true, + 0x05: true, + 0x06: true, + 0x07: true, + 0x08: true, + 0x09: true, + 0x0a: true, + 0x0b: true, + 0x0c: true, + 0x0d: true, + 0x0e: true, + 0x0f: true, + 0x10: true, + 0x11: true, + 0x12: true, + 0x13: true, + 0x14: true, + 0x15: true, + 0x16: true, + 0x17: true, + 0x18: true, + 0x19: true, + 0x1a: true, + 0x1b: true, + 0x1c: true, + 0x1d: true, + 0x1e: true, + 0x1f: true, + /* 0x20 - 0x7f */ + 0x80: true, + 0x81: true, + 0x82: true, + 0x83: true, + 0x84: true, + 0x85: true, + 0x86: true, + 0x87: true, + 0x88: true, + 0x89: true, + 0x8a: true, + 0x8b: true, + 0x8c: true, + 0x8d: true, + 0x8e: true, + 0x8f: true, + 0x90: true, + 0x91: true, + 0x92: true, + 0x93: true, + 0x94: true, + 0x95: true, + 0x96: true, + 0x97: true, + 0x98: true, + 0x99: true, + 0x9a: true, + 0x9b: true, + 0x9c: true, + 0x9d: true, + 0x9e: true, + 0x9f: true, + 0xa0: true, + 0xa1: true, + 0xa2: true, + 0xa3: true, + 0xa4: true, + 0xa5: true, + 0xa6: true, + 0xa7: true, + 0xa8: true, + 0xa9: true, + 0xaa: true, + 0xab: true, + 0xac: true, + 0xad: true, + 0xae: true, + 0xaf: true, + 0xb0: true, + 0xb1: true, + 0xb2: true, + 0xb3: true, + 0xb4: true, + 0xb5: true, + 0xb6: true, + 0xb7: true, + 0xb8: true, + 0xb9: true, + 0xba: true, + 0xbb: true, + 0xbc: true, + 0xbd: true, + 0xbe: true, + 0xbf: true, + 0xc0: true, + 0xc1: true, + 0xc2: true, + 0xc3: true, + 0xc4: true, + 0xc5: true, + 0xc6: true, + 0xc7: true, + 0xc8: true, + 0xc9: true, + 0xca: true, + 0xcb: true, + 0xcc: true, + 0xcd: true, + 0xce: true, + 0xcf: true, + 0xd0: true, + 0xd1: true, + 0xd2: true, + 0xd3: true, + 0xd4: true, + 0xd5: true, + 0xd6: true, + 0xd7: true, + 0xd8: true, + 0xd9: true, + 0xda: true, + 0xdb: true, + 0xdc: true, + 0xdd: true, + 0xde: true, + 0xdf: true, + 0xe0: true, + 0xe1: true, + 0xe2: true, + 0xe3: true, + 0xe4: true, + 0xe5: true, + 0xe6: true, + 0xe7: true, + 0xe8: true, + 0xe9: true, + 0xea: true, + 0xeb: true, + 0xec: true, + 0xed: true, + 0xee: true, + 0xef: true, + 0xf0: true, + 0xf1: true, + 0xf2: true, + 0xf3: true, + 0xf4: true, + 0xf5: true, + 0xf6: true, + 0xf7: true, + 0xf8: true, + 0xf9: true, + 0xfa: true, + 0xfb: true, + 0xfc: true, + 0xfd: true, + 0xfe: true, + 0xff: true, +} + +var needEscapeNormalizeUTF8 = [256]bool{ + '"': true, + '\\': true, + 0x00: true, + 0x01: true, + 0x02: true, + 0x03: true, + 0x04: true, + 0x05: true, + 0x06: true, + 0x07: true, + 0x08: true, + 0x09: true, + 0x0a: true, + 0x0b: true, + 0x0c: true, + 0x0d: true, + 0x0e: true, + 0x0f: true, + 0x10: true, + 0x11: true, + 0x12: true, + 0x13: true, + 0x14: true, + 0x15: true, + 0x16: true, + 0x17: true, + 0x18: true, + 0x19: true, + 0x1a: true, + 0x1b: true, + 0x1c: true, + 0x1d: true, + 0x1e: true, + 0x1f: true, + /* 0x20 - 0x7f */ + 0x80: true, + 0x81: true, + 0x82: true, + 0x83: true, + 0x84: true, + 0x85: true, + 0x86: true, + 0x87: true, + 0x88: true, + 0x89: true, + 0x8a: true, + 0x8b: true, + 0x8c: true, + 0x8d: true, + 0x8e: true, + 0x8f: true, + 0x90: true, + 0x91: true, + 0x92: true, + 0x93: true, + 0x94: true, + 0x95: true, + 0x96: true, + 0x97: true, + 0x98: true, + 0x99: true, + 0x9a: true, + 0x9b: true, + 0x9c: true, + 0x9d: true, + 0x9e: true, + 0x9f: true, + 0xa0: true, + 0xa1: true, + 0xa2: true, + 0xa3: true, + 0xa4: true, + 0xa5: true, + 0xa6: true, + 0xa7: true, + 0xa8: true, + 0xa9: true, + 0xaa: true, + 0xab: true, + 0xac: true, + 0xad: true, + 0xae: true, + 0xaf: true, + 0xb0: true, + 0xb1: true, + 0xb2: true, + 0xb3: true, + 0xb4: true, + 0xb5: true, + 0xb6: true, + 0xb7: true, + 0xb8: true, + 0xb9: true, + 0xba: true, + 0xbb: true, + 0xbc: true, + 0xbd: true, + 0xbe: true, + 0xbf: true, + 0xc0: true, + 0xc1: true, + 0xc2: true, + 0xc3: true, + 0xc4: true, + 0xc5: true, + 0xc6: true, + 0xc7: true, + 0xc8: true, + 0xc9: true, + 0xca: true, + 0xcb: true, + 0xcc: true, + 0xcd: true, + 0xce: true, + 0xcf: true, + 0xd0: true, + 0xd1: true, + 0xd2: true, + 0xd3: true, + 0xd4: true, + 0xd5: true, + 0xd6: true, + 0xd7: true, + 0xd8: true, + 0xd9: true, + 0xda: true, + 0xdb: true, + 0xdc: true, + 0xdd: true, + 0xde: true, + 0xdf: true, + 0xe0: true, + 0xe1: true, + 0xe2: true, + 0xe3: true, + 0xe4: true, + 0xe5: true, + 0xe6: true, + 0xe7: true, + 0xe8: true, + 0xe9: true, + 0xea: true, + 0xeb: true, + 0xec: true, + 0xed: true, + 0xee: true, + 0xef: true, + 0xf0: true, + 0xf1: true, + 0xf2: true, + 0xf3: true, + 0xf4: true, + 0xf5: true, + 0xf6: true, + 0xf7: true, + 0xf8: true, + 0xf9: true, + 0xfa: true, + 0xfb: true, + 0xfc: true, + 0xfd: true, + 0xfe: true, + 0xff: true, +} + +var needEscapeHTML = [256]bool{ + '"': true, + '&': true, + '<': true, + '>': true, + '\\': true, + 0x00: true, + 0x01: true, + 0x02: true, + 0x03: true, + 0x04: true, + 0x05: true, + 0x06: true, + 0x07: true, + 0x08: true, + 0x09: true, + 0x0a: true, + 0x0b: true, + 0x0c: true, + 0x0d: true, + 0x0e: true, + 0x0f: true, + 0x10: true, + 0x11: true, + 0x12: true, + 0x13: true, + 0x14: true, + 0x15: true, + 0x16: true, + 0x17: true, + 0x18: true, + 0x19: true, + 0x1a: true, + 0x1b: true, + 0x1c: true, + 0x1d: true, + 0x1e: true, + 0x1f: true, + /* 0x20 - 0xff */ +} + +var needEscape = [256]bool{ + '"': true, + '\\': true, + 0x00: true, + 0x01: true, + 0x02: true, + 0x03: true, + 0x04: true, + 0x05: true, + 0x06: true, + 0x07: true, + 0x08: true, + 0x09: true, + 0x0a: true, + 0x0b: true, + 0x0c: true, + 0x0d: true, + 0x0e: true, + 0x0f: true, + 0x10: true, + 0x11: true, + 0x12: true, + 0x13: true, + 0x14: true, + 0x15: true, + 0x16: true, + 0x17: true, + 0x18: true, + 0x19: true, + 0x1a: true, + 0x1b: true, + 0x1c: true, + 0x1d: true, + 0x1e: true, + 0x1f: true, + /* 0x20 - 0xff */ +} diff --git a/internal/go-json/encoder/vm/vm.go b/internal/go-json/encoder/vm/vm.go index 6c8292d01c..61a30cf655 100644 --- a/internal/go-json/encoder/vm/vm.go +++ b/internal/go-json/encoder/vm/vm.go @@ -199,7 +199,7 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b break } ctx.KeepRefs = append(ctx.KeepRefs, up) - ifaceCodeSet, err := encoder.CompileToGetCodeSet(uintptr(unsafe.Pointer(typ))) + ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ))) if err != nil { return nil, err } @@ -218,8 +218,7 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b oldOffset := ptrOffset ptrOffset += totalLength * uintptrSize oldBaseIndent := ctx.BaseIndent - indentDiffFromTop := c.Indent - 1 - ctx.BaseIndent += code.Indent - indentDiffFromTop + ctx.BaseIndent += code.Indent newLen := offsetNum + totalLength + nextTotalLength if curlen < newLen { @@ -403,11 +402,12 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b break } b = appendStructHead(ctx, b) - mapCtx := encoder.NewMapContext(mlen) + unorderedMap := (ctx.Option.Flag & encoder.UnorderedMapOption) != 0 + mapCtx := encoder.NewMapContext(mlen, unorderedMap) mapiterinit(code.Type, uptr, &mapCtx.Iter) store(ctxptr, code.Idx, uintptr(unsafe.Pointer(mapCtx))) ctx.KeepRefs = append(ctx.KeepRefs, unsafe.Pointer(mapCtx)) - if (ctx.Option.Flag & encoder.UnorderedMapOption) != 0 { + if unorderedMap { b = appendMapKeyIndent(ctx, code.Next, b) } else { mapCtx.Start = len(b) @@ -705,14 +705,15 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b if code.Flags&encoder.AnonymousHeadFlags == 0 { b = appendStructHead(ctx, b) } - u64 := ptrToUint64(p+uintptr(code.Offset), code.NumBitSize) + p += uintptr(code.Offset) + u64 := ptrToUint64(p, code.NumBitSize) v := u64 & ((1 << code.NumBitSize) - 1) if v == 0 { code = code.NextField } else { b = appendStructKey(ctx, code, b) b = append(b, '"') - b = appendInt(ctx, b, p+uintptr(code.Offset), code) + b = appendInt(ctx, b, p, code) b = append(b, '"') b = appendComma(ctx, b) code = code.Next @@ -2953,9 +2954,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b b = appendStructHead(ctx, b) } b = appendStructKey(ctx, code, b) + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadMarshalJSON { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } if p == 0 && (code.Flags&encoder.NilCheckFlags) != 0 { @@ -2994,9 +2996,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b if code.Flags&encoder.AnonymousHeadFlags == 0 { b = appendStructHead(ctx, b) } + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadOmitEmptyMarshalJSON { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } iface := ptrToInterface(code, p) @@ -3114,9 +3117,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b b = appendStructHead(ctx, b) } b = appendStructKey(ctx, code, b) + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadMarshalText { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } if p == 0 && (code.Flags&encoder.NilCheckFlags) != 0 { @@ -3155,9 +3159,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b if code.Flags&encoder.AnonymousHeadFlags == 0 { b = appendStructHead(ctx, b) } + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadOmitEmptyMarshalText { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } if p == 0 && (code.Flags&encoder.NilCheckFlags) != 0 { diff --git a/internal/go-json/encoder/vm_color/vm.go b/internal/go-json/encoder/vm_color/vm.go index 41b326e601..fe28e064d1 100644 --- a/internal/go-json/encoder/vm_color/vm.go +++ b/internal/go-json/encoder/vm_color/vm.go @@ -199,7 +199,7 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b break } ctx.KeepRefs = append(ctx.KeepRefs, up) - ifaceCodeSet, err := encoder.CompileToGetCodeSet(uintptr(unsafe.Pointer(typ))) + ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ))) if err != nil { return nil, err } @@ -218,8 +218,7 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b oldOffset := ptrOffset ptrOffset += totalLength * uintptrSize oldBaseIndent := ctx.BaseIndent - indentDiffFromTop := c.Indent - 1 - ctx.BaseIndent += code.Indent - indentDiffFromTop + ctx.BaseIndent += code.Indent newLen := offsetNum + totalLength + nextTotalLength if curlen < newLen { @@ -403,11 +402,12 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b break } b = appendStructHead(ctx, b) - mapCtx := encoder.NewMapContext(mlen) + unorderedMap := (ctx.Option.Flag & encoder.UnorderedMapOption) != 0 + mapCtx := encoder.NewMapContext(mlen, unorderedMap) mapiterinit(code.Type, uptr, &mapCtx.Iter) store(ctxptr, code.Idx, uintptr(unsafe.Pointer(mapCtx))) ctx.KeepRefs = append(ctx.KeepRefs, unsafe.Pointer(mapCtx)) - if (ctx.Option.Flag & encoder.UnorderedMapOption) != 0 { + if unorderedMap { b = appendMapKeyIndent(ctx, code.Next, b) } else { mapCtx.Start = len(b) @@ -705,14 +705,15 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b if code.Flags&encoder.AnonymousHeadFlags == 0 { b = appendStructHead(ctx, b) } - u64 := ptrToUint64(p+uintptr(code.Offset), code.NumBitSize) + p += uintptr(code.Offset) + u64 := ptrToUint64(p, code.NumBitSize) v := u64 & ((1 << code.NumBitSize) - 1) if v == 0 { code = code.NextField } else { b = appendStructKey(ctx, code, b) b = append(b, '"') - b = appendInt(ctx, b, p+uintptr(code.Offset), code) + b = appendInt(ctx, b, p, code) b = append(b, '"') b = appendComma(ctx, b) code = code.Next @@ -2953,9 +2954,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b b = appendStructHead(ctx, b) } b = appendStructKey(ctx, code, b) + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadMarshalJSON { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } if p == 0 && (code.Flags&encoder.NilCheckFlags) != 0 { @@ -2994,9 +2996,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b if code.Flags&encoder.AnonymousHeadFlags == 0 { b = appendStructHead(ctx, b) } + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadOmitEmptyMarshalJSON { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } iface := ptrToInterface(code, p) @@ -3114,9 +3117,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b b = appendStructHead(ctx, b) } b = appendStructKey(ctx, code, b) + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadMarshalText { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } if p == 0 && (code.Flags&encoder.NilCheckFlags) != 0 { @@ -3155,9 +3159,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b if code.Flags&encoder.AnonymousHeadFlags == 0 { b = appendStructHead(ctx, b) } + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadOmitEmptyMarshalText { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } if p == 0 && (code.Flags&encoder.NilCheckFlags) != 0 { diff --git a/internal/go-json/encoder/vm_color_indent/vm.go b/internal/go-json/encoder/vm_color_indent/vm.go index f3f6d0b9c1..fe6e0a0013 100644 --- a/internal/go-json/encoder/vm_color_indent/vm.go +++ b/internal/go-json/encoder/vm_color_indent/vm.go @@ -199,7 +199,7 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b break } ctx.KeepRefs = append(ctx.KeepRefs, up) - ifaceCodeSet, err := encoder.CompileToGetCodeSet(uintptr(unsafe.Pointer(typ))) + ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ))) if err != nil { return nil, err } @@ -218,8 +218,7 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b oldOffset := ptrOffset ptrOffset += totalLength * uintptrSize oldBaseIndent := ctx.BaseIndent - indentDiffFromTop := c.Indent - 1 - ctx.BaseIndent += code.Indent - indentDiffFromTop + ctx.BaseIndent += code.Indent newLen := offsetNum + totalLength + nextTotalLength if curlen < newLen { @@ -403,11 +402,12 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b break } b = appendStructHead(ctx, b) - mapCtx := encoder.NewMapContext(mlen) + unorderedMap := (ctx.Option.Flag & encoder.UnorderedMapOption) != 0 + mapCtx := encoder.NewMapContext(mlen, unorderedMap) mapiterinit(code.Type, uptr, &mapCtx.Iter) store(ctxptr, code.Idx, uintptr(unsafe.Pointer(mapCtx))) ctx.KeepRefs = append(ctx.KeepRefs, unsafe.Pointer(mapCtx)) - if (ctx.Option.Flag & encoder.UnorderedMapOption) != 0 { + if unorderedMap { b = appendMapKeyIndent(ctx, code.Next, b) } else { mapCtx.Start = len(b) @@ -705,14 +705,15 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b if code.Flags&encoder.AnonymousHeadFlags == 0 { b = appendStructHead(ctx, b) } - u64 := ptrToUint64(p+uintptr(code.Offset), code.NumBitSize) + p += uintptr(code.Offset) + u64 := ptrToUint64(p, code.NumBitSize) v := u64 & ((1 << code.NumBitSize) - 1) if v == 0 { code = code.NextField } else { b = appendStructKey(ctx, code, b) b = append(b, '"') - b = appendInt(ctx, b, p+uintptr(code.Offset), code) + b = appendInt(ctx, b, p, code) b = append(b, '"') b = appendComma(ctx, b) code = code.Next @@ -2953,9 +2954,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b b = appendStructHead(ctx, b) } b = appendStructKey(ctx, code, b) + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadMarshalJSON { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } if p == 0 && (code.Flags&encoder.NilCheckFlags) != 0 { @@ -2994,9 +2996,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b if code.Flags&encoder.AnonymousHeadFlags == 0 { b = appendStructHead(ctx, b) } + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadOmitEmptyMarshalJSON { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } iface := ptrToInterface(code, p) @@ -3114,9 +3117,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b b = appendStructHead(ctx, b) } b = appendStructKey(ctx, code, b) + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadMarshalText { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } if p == 0 && (code.Flags&encoder.NilCheckFlags) != 0 { @@ -3155,9 +3159,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b if code.Flags&encoder.AnonymousHeadFlags == 0 { b = appendStructHead(ctx, b) } + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadOmitEmptyMarshalText { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } if p == 0 && (code.Flags&encoder.NilCheckFlags) != 0 { diff --git a/internal/go-json/encoder/vm_indent/vm.go b/internal/go-json/encoder/vm_indent/vm.go index 88d925c814..216ee4d131 100644 --- a/internal/go-json/encoder/vm_indent/vm.go +++ b/internal/go-json/encoder/vm_indent/vm.go @@ -199,7 +199,7 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b break } ctx.KeepRefs = append(ctx.KeepRefs, up) - ifaceCodeSet, err := encoder.CompileToGetCodeSet(uintptr(unsafe.Pointer(typ))) + ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ))) if err != nil { return nil, err } @@ -218,8 +218,7 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b oldOffset := ptrOffset ptrOffset += totalLength * uintptrSize oldBaseIndent := ctx.BaseIndent - indentDiffFromTop := c.Indent - 1 - ctx.BaseIndent += code.Indent - indentDiffFromTop + ctx.BaseIndent += code.Indent newLen := offsetNum + totalLength + nextTotalLength if curlen < newLen { @@ -403,11 +402,12 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b break } b = appendStructHead(ctx, b) - mapCtx := encoder.NewMapContext(mlen) + unorderedMap := (ctx.Option.Flag & encoder.UnorderedMapOption) != 0 + mapCtx := encoder.NewMapContext(mlen, unorderedMap) mapiterinit(code.Type, uptr, &mapCtx.Iter) store(ctxptr, code.Idx, uintptr(unsafe.Pointer(mapCtx))) ctx.KeepRefs = append(ctx.KeepRefs, unsafe.Pointer(mapCtx)) - if (ctx.Option.Flag & encoder.UnorderedMapOption) != 0 { + if unorderedMap { b = appendMapKeyIndent(ctx, code.Next, b) } else { mapCtx.Start = len(b) @@ -705,14 +705,15 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b if code.Flags&encoder.AnonymousHeadFlags == 0 { b = appendStructHead(ctx, b) } - u64 := ptrToUint64(p+uintptr(code.Offset), code.NumBitSize) + p += uintptr(code.Offset) + u64 := ptrToUint64(p, code.NumBitSize) v := u64 & ((1 << code.NumBitSize) - 1) if v == 0 { code = code.NextField } else { b = appendStructKey(ctx, code, b) b = append(b, '"') - b = appendInt(ctx, b, p+uintptr(code.Offset), code) + b = appendInt(ctx, b, p, code) b = append(b, '"') b = appendComma(ctx, b) code = code.Next @@ -2953,9 +2954,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b b = appendStructHead(ctx, b) } b = appendStructKey(ctx, code, b) + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadMarshalJSON { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } if p == 0 && (code.Flags&encoder.NilCheckFlags) != 0 { @@ -2994,9 +2996,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b if code.Flags&encoder.AnonymousHeadFlags == 0 { b = appendStructHead(ctx, b) } + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadOmitEmptyMarshalJSON { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } iface := ptrToInterface(code, p) @@ -3114,9 +3117,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b b = appendStructHead(ctx, b) } b = appendStructKey(ctx, code, b) + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadMarshalText { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } if p == 0 && (code.Flags&encoder.NilCheckFlags) != 0 { @@ -3155,9 +3159,10 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b if code.Flags&encoder.AnonymousHeadFlags == 0 { b = appendStructHead(ctx, b) } + p += uintptr(code.Offset) if (code.Flags & encoder.IsNilableTypeFlags) != 0 { if (code.Flags&encoder.IndirectFlags) != 0 || code.Op == encoder.OpStructPtrHeadOmitEmptyMarshalText { - p = ptrToPtr(p + uintptr(code.Offset)) + p = ptrToPtr(p) } } if p == 0 && (code.Flags&encoder.NilCheckFlags) != 0 { diff --git a/internal/go-json/error.go b/internal/go-json/error.go index 906b3ffb88..853aa805d3 100644 --- a/internal/go-json/error.go +++ b/internal/go-json/error.go @@ -26,7 +26,7 @@ type SyntaxError = errors.SyntaxError // led to an unexported (and therefore unwritable) struct field. // // Deprecated: No longer used; kept for compatibility. -//lint:ignore SA1019 we love invalid regular expressions. +//lint:ignore SA1019 we love invalid regular expressions! type UnmarshalFieldError = errors.UnmarshalFieldError //nolint:staticcheck // An UnmarshalTypeError describes a JSON value that was diff --git a/internal/go-json/json.go b/internal/go-json/json.go index 63217f4de9..9ff4e07d17 100644 --- a/internal/go-json/json.go +++ b/internal/go-json/json.go @@ -364,3 +364,8 @@ func Valid(data []byte) bool { } return decoder.InputOffset() >= int64(len(data)) } + +func init() { + encoder.Marshal = Marshal + encoder.Unmarshal = Unmarshal +} diff --git a/internal/go-json/option.go b/internal/go-json/option.go index dba8333c72..fd1870758e 100644 --- a/internal/go-json/option.go +++ b/internal/go-json/option.go @@ -15,6 +15,23 @@ func UnorderedMap() EncodeOptionFunc { } } +// DisableHTMLEscape disables escaping of HTML characters ( '&', '<', '>' ) when encoding string. +func DisableHTMLEscape() EncodeOptionFunc { + return func(opt *EncodeOption) { + opt.Flag &= ^encoder.HTMLEscapeOption + } +} + +// DisableNormalizeUTF8 +// By default, when encoding string, UTF8 characters in the range of 0x80 - 0xFF are processed by applying \ufffd for invalid code and escaping for \u2028 and \u2029. +// This option disables this behaviour. You can expect faster speeds by applying this option, but be careful. +// encoding/json implements here: https://github.com/golang/go/blob/6178d25fc0b28724b1b5aec2b1b74fc06d9294c7/src/encoding/json/encode.go#L1067-L1093. +func DisableNormalizeUTF8() EncodeOptionFunc { + return func(opt *EncodeOption) { + opt.Flag &= ^encoder.NormalizeUTF8Option + } +} + // Debug outputs debug information when panic occurs during encoding. func Debug() EncodeOptionFunc { return func(opt *EncodeOption) { diff --git a/internal/go-json/query.go b/internal/go-json/query.go new file mode 100644 index 0000000000..a1c74c897c --- /dev/null +++ b/internal/go-json/query.go @@ -0,0 +1,47 @@ +package json + +import ( + "github.com/gofiber/fiber/v2/internal/go-json/encoder" +) + +type ( + // FieldQuery you can dynamically filter the fields in the structure by creating a FieldQuery, + // adding it to context.Context using SetFieldQueryToContext and then passing it to MarshalContext. + // This is a type-safe operation, so it is faster than filtering using map[string]interface{}. + FieldQuery = encoder.FieldQuery + FieldQueryString = encoder.FieldQueryString +) + +var ( + // FieldQueryFromContext get current FieldQuery from context.Context. + FieldQueryFromContext = encoder.FieldQueryFromContext + // SetFieldQueryToContext set current FieldQuery to context.Context. + SetFieldQueryToContext = encoder.SetFieldQueryToContext +) + +// BuildFieldQuery builds FieldQuery by fieldName or sub field query. +// First, specify the field name that you want to keep in structure type. +// If the field you want to keep is a structure type, by creating a sub field query using BuildSubFieldQuery, +// you can select the fields you want to keep in the structure. +// This description can be written recursively. +func BuildFieldQuery(fields ...FieldQueryString) (*FieldQuery, error) { + query, err := Marshal(fields) + if err != nil { + return nil, err + } + return FieldQueryString(query).Build() +} + +// BuildSubFieldQuery builds sub field query. +func BuildSubFieldQuery(name string) *SubFieldQuery { + return &SubFieldQuery{name: name} +} + +type SubFieldQuery struct { + name string +} + +func (q *SubFieldQuery) Fields(fields ...FieldQueryString) FieldQueryString { + query, _ := Marshal(map[string][]FieldQueryString{q.name: fields}) + return FieldQueryString(query) +}