diff --git a/mw_transform_test.go b/mw_transform_test.go index cde43aed837..9ec95a963b0 100644 --- a/mw_transform_test.go +++ b/mw_transform_test.go @@ -2,6 +2,7 @@ package main import ( "io/ioutil" + "strings" "testing" "text/template" @@ -31,3 +32,16 @@ func TestTransformNonAscii(t *testing.T) { t.Fatalf("wanted body %q, got %q", want, got) } } + +func TestTransformXMLCrash(t *testing.T) { + // mxj.NewMapXmlReader used to take forever and crash the + // process by eating up all the memory. + in := strings.NewReader("") + r := testReq(t, "GET", "/", in) + tmeta := &TransformSpec{} + tmeta.TemplateData.Input = apidef.RequestXML + tmeta.Template = template.Must(template.New("blob").Parse("")) + if err := transformBody(r, tmeta, false); err == nil { + t.Fatalf("wanted error, got nil") + } +} diff --git a/vendor/github.com/clbanning/mxj/doc.go b/vendor/github.com/clbanning/mxj/doc.go index 325c982ce67..a0779fb7ac8 100644 --- a/vendor/github.com/clbanning/mxj/doc.go +++ b/vendor/github.com/clbanning/mxj/doc.go @@ -10,7 +10,14 @@ mxj supplants the legacy x2j and j2x packages. If you want the old syntax, use m Note: this library was designed for processing ad hoc anonymous messages. Bulk processing large data sets may be much more efficiently performed using the encoding/xml or encoding/json packages from Go's standard library directly. -Note: +Related Packages: + checkxml: github.com/clbanning/checkxml provides functions for validating XML data. + +Notes: + 2017.02.22: LeafNode paths can use ".N" syntax rather than "[N]" for list member indexing. + 2017.02.21: github.com/clbanning/checkxml provides functions for validating XML data. + 2017.02.10: SetFieldSeparator changes field separator for args in UpdateValuesForPath, ValuesFor... methods. + 2017.02.06: Support XMPP stream processing - HandleXMPPStreamTag(). 2016.11.07: Preserve name space prefix syntax in XmlSeq parser - NewMapXmlSeq(), etc. 2016.06.25: Support overriding default XML attribute prefix, "-", in Map keys - SetAttrPrefix(). 2016.05.26: Support customization of xml.Decoder by exposing CustomDecoder variable. @@ -82,7 +89,8 @@ XML PARSING CONVENTIONS Using NewMapXml() - Attributes are parsed to `map[string]interface{}` values by prefixing a hyphen, `-`, - to the attribute label. (Unless overridden by `PrependAttrWithHyphen(false)`.) + to the attribute label. (Unless overridden by `PrependAttrWithHyphen(false)` or + `SetAttrPrefix()`.) - If the element is a simple element and has attributes, the element value is given the key `#text` for its `map[string]interface{}` representation. (See the 'atomFeedString.xml' test data, below.) @@ -99,8 +107,8 @@ XML PARSING CONVENTIONS keys "#comment", "#directive", and "#procinst", respectively. (See documentation for more specifics.) - Name space syntax is preserved: - - something parses to map["ns:key"]interface{}("something") - - xmlns:ns="http://myns.com/ns" parses to map["xmlns:ns"]interface{}("http://myns.com/ns") + - something parses to map["ns:key"]interface{}{"something"} + - xmlns:ns="http://myns.com/ns" parses to map["xmlns:ns"]interface{}{"http://myns.com/ns"} Both diff --git a/vendor/github.com/clbanning/mxj/files.go b/vendor/github.com/clbanning/mxj/files.go index 58971187402..27e06e1e801 100644 --- a/vendor/github.com/clbanning/mxj/files.go +++ b/vendor/github.com/clbanning/mxj/files.go @@ -84,12 +84,6 @@ func NewMapsFromJsonFileRaw(name string) ([]MapRaw, error) { // NewMapsFromXmlFile - creates an array from a file of XML values. func NewMapsFromXmlFile(name string) (Maps, error) { - x := XmlWriterBufSize - XmlWriterBufSize = 0 - defer func() { - XmlWriterBufSize = x - }() - fi, err := os.Stat(name) if err != nil { return nil, err @@ -124,12 +118,6 @@ func NewMapsFromXmlFile(name string) (Maps, error) { // NOTE: the slice with the raw XML is clean with no extra capacity - unlike NewMapXmlReaderRaw(). // It is slow at parsing a file from disk and is intended for relatively small utility files. func NewMapsFromXmlFileRaw(name string) ([]MapRaw, error) { - x := XmlWriterBufSize - XmlWriterBufSize = 0 - defer func() { - XmlWriterBufSize = x - }() - fi, err := os.Stat(name) if err != nil { return nil, err diff --git a/vendor/github.com/clbanning/mxj/keyvalues.go b/vendor/github.com/clbanning/mxj/keyvalues.go index 7feb766b44c..7443b4133ad 100644 --- a/vendor/github.com/clbanning/mxj/keyvalues.go +++ b/vendor/github.com/clbanning/mxj/keyvalues.go @@ -42,6 +42,7 @@ func SetArraySize(size int) int { // - The subkey can be wildcarded - "key:*" - to require that it's there with some value. // - If a subkey is preceeded with the '!' character, the key:value[:type] entry is treated as an // exclusion critera - e.g., "!author:William T. Gaddis". +// - If val contains ":" symbol, use SetFieldSeparator to a unused symbol, perhaps "|". func (mv Map) ValuesForKey(key string, subkeys ...string) ([]interface{}, error) { m := map[string]interface{}(mv) var subKeyMap map[string]interface{} @@ -151,6 +152,7 @@ func hasKey(iv interface{}, key string, ret *[]interface{}, cnt *int, subkeys ma // - The subkey can be wildcarded - "key:*" - to require that it's there with some value. // - If a subkey is preceeded with the '!' character, the key:value[:type] entry is treated as an // exclusion critera - e.g., "!author:William T. Gaddis". +// - If val contains ":" symbol, use SetFieldSeparator to a unused symbol, perhaps "|". func (mv Map) ValuesForPath(path string, subkeys ...string) ([]interface{}, error) { // If there are no array indexes in path, use legacy ValuesForPath() logic. if strings.Index(path, "[") < 0 { @@ -499,12 +501,12 @@ func getSubKeyMap(kv ...string) (map[string]interface{}, error) { } m := make(map[string]interface{}, 0) for _, v := range kv { - vv := strings.Split(v, ":") + vv := strings.Split(v, fieldSep) switch len(vv) { case 2: m[vv[0]] = interface{}(vv[1]) case 3: - switch vv[3] { + switch vv[2] { case "string", "char", "text": m[vv[0]] = interface{}(vv[1]) case "bool", "boolean": @@ -593,13 +595,14 @@ func hasKeyPath(crumbs string, iv interface{}, key string, basket map[string]boo case map[string]interface{}: vv := iv.(map[string]interface{}) if _, ok := vv[key]; ok { + // create a new breadcrumb, intialized with the one we have + var nbc string if crumbs == "" { - crumbs = key + nbc = key } else { - crumbs += "." + key + nbc = crumbs + "." + key } - // *basket = append(*basket, crumb) - basket[crumbs] = true + basket[nbc] = true } // walk on down the path, key could occur again at deeper node for k, v := range vv { diff --git a/vendor/github.com/clbanning/mxj/leafnode.go b/vendor/github.com/clbanning/mxj/leafnode.go index 3e8b18da2eb..cf413ebdd4f 100644 --- a/vendor/github.com/clbanning/mxj/leafnode.go +++ b/vendor/github.com/clbanning/mxj/leafnode.go @@ -24,9 +24,13 @@ type LeafNode struct { // LeafNodes - returns an array of all LeafNode values for the Map. // The option no_attr argument suppresses attribute values (keys with prepended hyphen, '-') // as well as the "#text" key for the associated simple element value. -// NOTE: if PrependAttrWithHypen(false), then #test is stripping "#text" key -// will result in attributes having .attr-name as terminal node in 'path' while -// the path for the element value, itself, will be the base path w/o "#text". +// +// PrependAttrWithHypen(false) will result in attributes having .attr-name as +// terminal node in 'path' while the path for the element value, itself, will be +// the base path w/o "#text". +// +// LeafUseDotNotation(true) causes list members to be identified using ".N" syntax +// rather than "[N]" syntax. func (mv Map) LeafNodes(no_attr ...bool) []LeafNode { var a bool if len(no_attr) == 1 { @@ -57,7 +61,11 @@ func getLeafNodes(path, node string, mv interface{}, l *[]LeafNode, noattr bool) } case []interface{}: for i, v := range mv.([]interface{}) { - getLeafNodes(path, "["+strconv.Itoa(i)+"]", v, l, noattr) + if useDotNotation { + getLeafNodes(path, strconv.Itoa(i), v, l, noattr) + } else { + getLeafNodes(path, "["+strconv.Itoa(i)+"]", v, l, noattr) + } } default: // can't walk any further, so create leaf @@ -85,3 +93,20 @@ func (mv Map) LeafValues(no_attr ...bool) []interface{} { } return vv } + +// ====================== utilities ====================== + +// https://groups.google.com/forum/#!topic/golang-nuts/pj0C5IrZk4I +var useDotNotation bool + +// LeafUseDotNotation sets a flag that list members in LeafNode paths +// should be identified using ".N" syntax rather than the default "[N]" +// syntax. Calling LeafUseDotNotation with no arguments toggles the +// flag on/off; otherwise, the argument sets the flag value 'true'/'false'. +func LeafUseDotNotation(b ...bool) { + if len(b) == 0 { + useDotNotation = !useDotNotation + return + } + useDotNotation = b[0] +} diff --git a/vendor/github.com/clbanning/mxj/newmap.go b/vendor/github.com/clbanning/mxj/newmap.go index fc1320c002b..4a9b2e8d9c8 100644 --- a/vendor/github.com/clbanning/mxj/newmap.go +++ b/vendor/github.com/clbanning/mxj/newmap.go @@ -27,7 +27,7 @@ import ( // should be the value for 'newKey' in the returned Map. // - 'oldKey' supports dot-notation as described for (Map)ValuesForPath() // - 'newKey' supports dot-notation but with no wildcards, '*', or indexed arrays -// - "oldKey" is shorthand for for the keypair value "oldKey:oldKey" +// - "oldKey" is shorthand for the keypair value "oldKey:oldKey" // - "oldKey:" and ":newKey" are invalid keypair values // - if 'oldKey' does not exist in the current Map, it is not written to the new Map. // "null" is not supported unless it is the current Map. diff --git a/vendor/github.com/clbanning/mxj/readme.md b/vendor/github.com/clbanning/mxj/readme.md index bd87973eb48..5abcd799ffe 100644 --- a/vendor/github.com/clbanning/mxj/readme.md +++ b/vendor/github.com/clbanning/mxj/readme.md @@ -3,6 +3,10 @@ Decode/encode XML to/from map[string]interface{} (or JSON) values, and extract/m mxj supplants the legacy x2j and j2x packages. If you want the old syntax, use mxj/x2j and mxj/j2x packages. +

Related Packages

+ +https://github.com/clbanning/checkxml provides functions for validating XML data. +

Refactor Decoder - 2015.11.15

For over a year I've wanted to refactor the XML-to-map[string]interface{} decoder to make it more performant. I recently took the time to do that, since we were using github.com/clbanning/mxj in a production system that could be deployed on a Raspberry Pi. Now the decoder is comparable to the stdlib JSON-to-map[string]interface{} decoder in terms of its additional processing overhead relative to decoding to a structure value. As shown by: @@ -16,6 +20,10 @@ For over a year I've wanted to refactor the XML-to-map[string]interface{} decode BenchmarkNewStructJsonBooks-4 100000 15309 ns/op

Notices

+ + 2017.02.22: LeafNode paths can use ".N" syntax rather than "[N]" for list member indexing. + 2017.02.10: SetFieldSeparator changes field separator for args in UpdateValuesForPath, ValuesFor... methods. + 2017.02.06: Support XMPP stream processing - HandleXMPPStreamTag(). 2016.11.07: Preserve name space prefix syntax in XmlSeq parser - NewMapXmlSeq(), etc. 2016.06.25: Support overriding default XML attribute prefix, "-", in Map keys - SetAttrPrefix(). 2016.05.26: Support customization of xml.Decoder by exposing CustomDecoder variable. @@ -92,7 +100,7 @@ newJson, err := newMap.Json() // ditto

Usage

-The package is fairly well self-documented with examples. (http://godoc.org/github.com/clbanning/mxj) +The package is fairly well [self-documented with examples](http://godoc.org/github.com/clbanning/mxj). Also, the subdirectory "examples" contains a wide range of examples, several taken from golang-nuts discussions. @@ -101,7 +109,8 @@ Also, the subdirectory "examples" contains a wide range of examples, several tak Using NewMapXml() - Attributes are parsed to `map[string]interface{}` values by prefixing a hyphen, `-`, - to the attribute label. (Unless overridden by `PrependAttrWithHyphen(false)`.) + to the attribute label. (Unless overridden by `PrependAttrWithHyphen(false)` or + `SetAttrPrefix()`.) - If the element is a simple element and has attributes, the element value is given the key `#text` for its `map[string]interface{}` representation. (See the 'atomFeedString.xml' test data, below.) @@ -118,8 +127,8 @@ Using NewMapXmlSeq() keys "#comment", "#directive", and "#procinst", respectively. (See documentation for more specifics.) - Name space syntax is preserved: - - `something` parses to `map["ns:key"]interface{}("something")` - - `xmlns:ns="http://myns.com/ns"` parses to `map["xmlns:ns"]interface{}("http://myns.com/ns")` + - `something` parses to `map["ns:key"]interface{}{"something"}` + - `xmlns:ns="http://myns.com/ns"` parses to `map["xmlns:ns"]interface{}{"http://myns.com/ns"}` Both diff --git a/vendor/github.com/clbanning/mxj/rename.go b/vendor/github.com/clbanning/mxj/rename.go index 511d7b1711e..e95a9639af7 100644 --- a/vendor/github.com/clbanning/mxj/rename.go +++ b/vendor/github.com/clbanning/mxj/rename.go @@ -9,10 +9,10 @@ import ( // It works only for nested maps. It doesn't work for cases when it buried in a list. func (mv Map) RenameKey(path string, newName string) error { if !mv.Exists(path) { - return errors.New("RenameKey: the path not found: " + path) + return errors.New("RenameKey: path not found: " + path) } if mv.Exists(parentPath(path) + "." + newName) { - return errors.New("RenameKey: the key already exists: " + newName) + return errors.New("RenameKey: key already exists: " + newName) } m := map[string]interface{}(mv) @@ -50,5 +50,5 @@ func prevValueByPath(m interface{}, path string) (map[string]interface{}, error) } } } - return nil, errors.New("prevValueByPath: didn't find the path – " + path) + return nil, errors.New("prevValueByPath: didn't find path – " + path) } diff --git a/vendor/github.com/clbanning/mxj/setfieldsep.go b/vendor/github.com/clbanning/mxj/setfieldsep.go new file mode 100644 index 00000000000..b70715ebc65 --- /dev/null +++ b/vendor/github.com/clbanning/mxj/setfieldsep.go @@ -0,0 +1,20 @@ +package mxj + +// Per: https://github.com/clbanning/mxj/issues/37#issuecomment-278651862 +var fieldSep string = ":" + +// SetFieldSeparator changes the default field separator, ":", for the +// newVal argument in mv.UpdateValuesForPath and the optional 'subkey' arguments +// in mv.ValuesForKey and mv.ValuesForPath. +// +// E.g., if the newVal value is "http://blah/blah", setting the field separator +// to "|" will allow the newVal specification, "|http://blah/blah" to parse +// properly. If called with no argument or an empty string value, the field +// separator is set to the default, ":". +func SetFieldSeparator(s ...string) { + if len(s) == 0 || s[0] == "" { + fieldSep = ":" // the default + return + } + fieldSep = s[0] +} diff --git a/vendor/github.com/clbanning/mxj/struct.go b/vendor/github.com/clbanning/mxj/struct.go index b9fb7160dda..9be636cdcab 100644 --- a/vendor/github.com/clbanning/mxj/struct.go +++ b/vendor/github.com/clbanning/mxj/struct.go @@ -1,4 +1,4 @@ -// Copyright 2012-2014 Charles Banning. All rights reserved. +// Copyright 2012-2017 Charles Banning. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file @@ -7,19 +7,32 @@ package mxj import ( "encoding/json" "errors" - "github.com/fatih/structs" "reflect" + + // "github.com/fatih/structs" ) -// Create a new Map value from a structure. Error returned if argument is not a structure -// or if there is a json.Marshal or json.Unmarshal error. -// Only public structure fields are decoded in the Map value. Also, json.Marshal structure encoding rules -// are followed for decoding the structure fields. +// Create a new Map value from a structure. Error returned if argument is not a structure. +// Only public structure fields are decoded in the Map value. See github.com/fatih/structs#Map +// for handling of "structs" tags. + +// DEPRECATED - import github.com/fatih/structs and cast result of structs.Map to mxj.Map. +// import "github.com/fatih/structs" +// ... +// sm, err := structs.Map() +// if err != nil { +// // handle error +// } +// m := mxj.Map(sm) +// Alernatively uncomment the old source and import in struct.go. func NewMapStruct(structVal interface{}) (Map, error) { - if !structs.IsStruct(structVal) { - return nil, errors.New("NewMapStruct() error: argument is not type Struct") - } - return structs.Map(structVal), nil + return nil, errors.New("deprecated - see package documentation") + /* + if !structs.IsStruct(structVal) { + return nil, errors.New("NewMapStruct() error: argument is not type Struct") + } + return structs.Map(structVal), nil + */ } // Marshal a map[string]interface{} into a structure referenced by 'structPtr'. Error returned diff --git a/vendor/github.com/clbanning/mxj/updatevalues.go b/vendor/github.com/clbanning/mxj/updatevalues.go index fed54bc48c5..46779f4f063 100644 --- a/vendor/github.com/clbanning/mxj/updatevalues.go +++ b/vendor/github.com/clbanning/mxj/updatevalues.go @@ -1,8 +1,9 @@ -// Copyright 2012-2014 Charles Banning. All rights reserved. +// Copyright 2012-2014, 2017 Charles Banning. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file // updatevalues.go - modify a value based on path and possibly sub-keys +// TODO(clb): handle simple elements with attributes and NewMapXmlSeq Map values. package mxj @@ -23,6 +24,12 @@ import ( // The subkey can be wildcarded - "key:*" - to require that it's there with some value. // If a subkey is preceeded with the '!' character, the key:value[:type] entry is treated as an // exclusion critera - e.g., "!author:William T. Gaddis". +// +// NOTES: +// 1. Simple elements with attributes need a path terminated as ".#text" to modify the actual value. +// 2. Values in Maps created using NewMapXmlSeq are map[string]interface{} values with a "#text" key. +// 3. If values in 'newVal' or 'subkeys' args contain ":", use SetFieldSeparator to an unused symbol, +// perhaps "|". func (mv Map) UpdateValuesForPath(newVal interface{}, path string, subkeys ...string) (int, error) { m := map[string]interface{}(mv) @@ -51,7 +58,7 @@ func (mv Map) UpdateValuesForPath(newVal interface{}, path string, subkeys ...st for key, val = range newVal.(map[string]interface{}) { } case string: // split it as a key:value pair - ss := strings.Split(newVal.(string), ":") + ss := strings.Split(newVal.(string), fieldSep) n := len(ss) if n < 2 || n > 3 { return 0, fmt.Errorf("unknown newVal spec - %+v", newVal) @@ -160,14 +167,14 @@ func updateValue(key string, value interface{}, m interface{}, keys0 string, sub if key == keys0 { switch endVal.(type) { case map[string]interface{}: - if ok := hasSubKeys(m, subkeys); ok { + if hasSubKeys(m, subkeys) { (m.(map[string]interface{}))[keys0] = value (*cnt)++ } case []interface{}: // without subkeys can't select list member to modify // so key:value spec is it ... - if len(subkeys) == 0 { + if hasSubKeys(m, subkeys) { (m.(map[string]interface{}))[keys0] = value (*cnt)++ break @@ -176,7 +183,7 @@ func updateValue(key string, value interface{}, m interface{}, keys0 string, sub var valmodified bool for _, v := range endVal.([]interface{}) { // check entry subkeys - if ok := hasSubKeys(v, subkeys); ok { + if hasSubKeys(v, subkeys) { // replace v with value nv = append(nv, value) valmodified = true @@ -189,7 +196,7 @@ func updateValue(key string, value interface{}, m interface{}, keys0 string, sub (m.(map[string]interface{}))[keys0] = interface{}(nv) } default: // anything else is a strict replacement - if len(subkeys) == 0 { + if hasSubKeys(m, subkeys) { (m.(map[string]interface{}))[keys0] = value (*cnt)++ } @@ -202,7 +209,7 @@ func updateValue(key string, value interface{}, m interface{}, keys0 string, sub // if endVal is a list then 'key' must be in a list member w/ subkeys switch endVal.(type) { case map[string]interface{}: - if ok := hasSubKeys(endVal, subkeys); !ok { + if !hasSubKeys(endVal, subkeys) { return } if _, ok := (endVal.(map[string]interface{}))[key]; ok { diff --git a/vendor/github.com/clbanning/mxj/xml.go b/vendor/github.com/clbanning/mxj/xml.go index 2f21887798e..3f03dd6243d 100644 --- a/vendor/github.com/clbanning/mxj/xml.go +++ b/vendor/github.com/clbanning/mxj/xml.go @@ -24,7 +24,7 @@ import ( // ------------------- NewMapXml & NewMapXmlReader ... ------------------------- // If XmlCharsetReader != nil, it will be used to decode the XML, if required. -// Note: if CustomDeocder != nil, then XmlCharsetReader is ignored; +// Note: if CustomDecoder != nil, then XmlCharsetReader is ignored; // set the CustomDecoder attribute instead. // import ( // charset "code.google.com/p/go-charset/charset" @@ -54,6 +54,7 @@ var XmlCharsetReader func(charset string, input io.Reader) (io.Reader, error) // 1. The 'xmlVal' will be parsed looking for an xml.StartElement, so BOM and other // extraneous xml.CharData will be ignored unless io.EOF is reached first. // 2. If CoerceKeysToLower() has been called, then all key values will be lower case. +// 3. If CoerceKeysToSnakeCase() has been called, then all key values will be converted to snake case. func NewMapXml(xmlVal []byte, cast ...bool) (Map, error) { var r bool if len(cast) == 1 { @@ -67,6 +68,7 @@ func NewMapXml(xmlVal []byte, cast ...bool) (Map, error) { // 1. The 'xmlReader' will be parsed looking for an xml.StartElement, so BOM and other // extraneous xml.CharData will be ignored unless io.EOF is reached first. // 2. If CoerceKeysToLower() has been called, then all key values will be lower case. +// 3. If CoerceKeysToSnakeCase() has been called, then all key values will be converted to snake case. func NewMapXmlReader(xmlReader io.Reader, cast ...bool) (Map, error) { var r bool if len(cast) == 1 { @@ -84,12 +86,6 @@ func NewMapXmlReader(xmlReader io.Reader, cast ...bool) (Map, error) { return xmlReaderToMap(xmlReader, r) } -// XmlWriterBufSize - set the size of io.Writer for the TeeReader used by NewMapXmlReaderRaw() -// and HandleXmlReaderRaw(). This reduces repeated memory allocations and copy() calls in most cases. -// NOTE: the 'xmlVal' will be parsed looking for an xml.StartElement, so BOM and other -// extraneous xml.CharData will be ignored unless io.EOF is reached first. -var XmlWriterBufSize int = 512 - // Get next XML doc from an io.Reader as a Map value. Returns Map value and slice with the raw XML. // NOTES: // 1. Due to the implementation of xml.Decoder, the raw XML off the reader is buffered to []byte @@ -101,22 +97,21 @@ var XmlWriterBufSize int = 512 // 3. The 'xmlReader' will be parsed looking for an xml.StartElement, so BOM and other // extraneous xml.CharData will be ignored unless io.EOF is reached first. // 4. If CoerceKeysToLower() has been called, then all key values will be lower case. +// 5. If CoerceKeysToSnakeCase() has been called, then all key values will be converted to snake case. func NewMapXmlReaderRaw(xmlReader io.Reader, cast ...bool) (Map, []byte, error) { var r bool if len(cast) == 1 { r = cast[0] } // create TeeReader so we can retrieve raw XML - buf := make([]byte, XmlWriterBufSize) + buf := make([]byte, 0) wb := bytes.NewBuffer(buf) trdr := myTeeReader(xmlReader, wb) // see code at EOF - // build the node tree m, err := xmlReaderToMap(trdr, r) // retrieve the raw XML that was decoded - b := make([]byte, wb.Len()) - _, _ = wb.Read(b) + b := wb.Bytes() if err != nil { return nil, b, err @@ -225,14 +220,10 @@ var lowerCase bool // NOTE: only recognized by NewMapXml, NewMapXmlReader, and NewMapXmlReaderRaw functions as well as // the associated HandleXmlReader and HandleXmlReaderRaw. func CoerceKeysToLower(b ...bool) { - if len(b) == 1 { + if len(b) == 0 { + lowerCase = !lowerCase + } else if len(b) == 1 { lowerCase = b[0] - return - } - if !lowerCase { - lowerCase = true - } else { - lowerCase = false } } @@ -251,6 +242,42 @@ func SetAttrPrefix(s string) { lenAttrPrefix = len(attrPrefix) } +// 18jan17: Allows user to specify if the map keys should be in snake case instead +// of the default hyphenated notation. +var snakeCaseKeys bool + +// CoerceKeysToSnakeCase changes the default, false, to the specified value, b. +// Note: the attribute prefix will be a hyphen, '-', or what ever string value has +// been specified using SetAttrPrefix. +func CoerceKeysToSnakeCase(b ...bool) { + if len(b) == 0 { + snakeCaseKeys = !snakeCaseKeys + } else if len(b) == 1 { + snakeCaseKeys = b[0] + } +} + +// 05feb17: support processing XMPP streams (issue #36) +var handleXMPPStreamTag bool + +// HandleXMPPStreamTag causes decoder to parse XMPP elements. +// If called with no argument, XMPP stream element handling is toggled on/off. +// (See xmppStream_test.go for example.) +// If called with NewMapXml, NewMapXmlReader, New MapXmlReaderRaw the "stream" +// element will be returned as: +// map["stream"]interface{}{map[-]interface{}}. +// If called with NewMapSeq, NewMapSeqReader, NewMapSeqReaderRaw the "stream" +// element will be returned as: +// map["stream:stream"]interface{}{map["#attr"]interface{}{map[string]interface{}}} +// where the "#attr" values have "#text" and "#seq" keys. (See NewMapXmlSeq.) +func HandleXMPPStreamTag(b ...bool) { + if len(b) == 0 { + handleXMPPStreamTag = !handleXMPPStreamTag + } else if len(b) == 1 { + handleXMPPStreamTag = b[0] + } +} + // xmlToMapParser (2015.11.12) - load a 'clean' XML doc into a map[string]interface{} directly. // A refactoring of xmlToTreeParser(), markDuplicate() and treeToMap() - here, all-in-one. // We've removed the intermediate *node tree with the allocation and subsequent rescanning. @@ -258,6 +285,9 @@ func xmlToMapParser(skey string, a []xml.Attr, p *xml.Decoder, r bool) (map[stri if lowerCase { skey = strings.ToLower(skey) } + if snakeCaseKeys { + skey = strings.Replace(skey, "-", "_", -1) + } // NOTE: all attributes and sub-elements parsed into 'na', 'na' is returned as value for 'skey' in 'n'. // Unless 'skey' is a simple element w/o attributes, in which case the xml.CharData value is the value. @@ -273,6 +303,9 @@ func xmlToMapParser(skey string, a []xml.Attr, p *xml.Decoder, r bool) (map[stri na = make(map[string]interface{}) // old n.nodes if len(a) > 0 { for _, v := range a { + if snakeCaseKeys { + v.Name.Local = strings.Replace(v.Name.Local, "-", "_", -1) + } var key string key = attrPrefix + v.Name.Local if lowerCase { @@ -282,6 +315,12 @@ func xmlToMapParser(skey string, a []xml.Attr, p *xml.Decoder, r bool) (map[stri } } } + // Return XMPP message. + if handleXMPPStreamTag && skey == "stream" { + n[skey] = na + return n, nil + } + for { t, err := p.Token() if err != nil { @@ -316,7 +355,9 @@ func xmlToMapParser(skey string, a []xml.Attr, p *xml.Decoder, r bool) (map[stri // We need to see if nn_key already exists - means we're parsing a list. // This may require converting na[nn_key] value into []interface{} type. // First, extract the key:val for the map - it's a singleton. - // Note: if CoerceKeysToLower() called, then key will be lower case. + // Note: + // * if CoerceKeysToLower() called, then key will be lower case. + // * if CoerceKeysToSnakeCase() called, then key will be converted to snake case. var key string var val interface{} for key, val = range nn { @@ -690,15 +731,16 @@ func myByteReader(r io.Reader) io.Reader { return &byteReader{r, b} } -// need for io.Reader - but we don't use it ... +// Need for io.Reader interface ... +// Needed if reading a malformed http.Request.Body - issue #38. func (b *byteReader) Read(p []byte) (int, error) { - return 0, nil + return b.r.Read(p) } func (b *byteReader) ReadByte() (byte, error) { _, err := b.r.Read(b.b) if len(b.b) > 0 { - return b.b[0], nil + return b.b[0], err } var c byte return c, err @@ -782,7 +824,7 @@ func mapToXmlIndent(doIndent bool, s *string, key string, value interface{}, pp var n int var ss string for k, v := range vv { - if k[:lenAttrPrefix] == attrPrefix { + if lenAttrPrefix > 0 && lenAttrPrefix < len(k) && k[:lenAttrPrefix] == attrPrefix { switch v.(type) { case string: if xmlEscapeChars { @@ -833,7 +875,14 @@ func mapToXmlIndent(doIndent bool, s *string, key string, value interface{}, pp elen = 1 isSimple = true break + } else if ok { + // Handle edge case where simple element with attributes + // is unmarshal'd using NewMapXml() where attribute prefix + // has been set to "". + // TODO(clb): should probably scan all keys for invalid chars. + return fmt.Errorf("invalid attribute key label: #text - due to attributes not being prefixed") } + // close tag with possible attributes *s += ">" if doIndent { @@ -845,7 +894,7 @@ func mapToXmlIndent(doIndent bool, s *string, key string, value interface{}, pp elemlist := make([][2]interface{}, len(vv)) n = 0 for k, v := range vv { - if k[:lenAttrPrefix] == attrPrefix { + if lenAttrPrefix > 0 && lenAttrPrefix < len(k) && k[:lenAttrPrefix] == attrPrefix { continue } elemlist[n][0] = k @@ -864,7 +913,9 @@ func mapToXmlIndent(doIndent bool, s *string, key string, value interface{}, pp } } i++ - mapToXmlIndent(doIndent, s, v[0].(string), v[1], p) + if err := mapToXmlIndent(doIndent, s, v[0].(string), v[1], p); err != nil { + return err + } switch v[1].(type) { case []interface{}: // handled in []interface{} case default: @@ -892,7 +943,37 @@ func mapToXmlIndent(doIndent bool, s *string, key string, value interface{}, pp if doIndent { p.Indent() } - mapToXmlIndent(doIndent, s, key, v, p) + if err := mapToXmlIndent(doIndent, s, key, v, p); err != nil { + return err + } + if doIndent { + p.Outdent() + } + } + return nil + case []string: + // This was added by https://github.com/slotix ... not a type that + // would be encountered if mv generated from NewMapXml, NewMapJson. + // Could be encountered in AnyXml(), so we'll let it stay, though + // it should be merged with case []interface{}, above. + //quick fix for []string type + //[]string should be treated exaclty as []interface{} + if len(value.([]string)) == 0 { + if doIndent { + *s += p.padding + p.indent + } + *s += "<" + key + elen = 0 + endTag = true + break + } + for _, v := range value.([]string) { + if doIndent { + p.Indent() + } + if err := mapToXmlIndent(doIndent, s, key, v, p); err != nil { + return err + } if doIndent { p.Outdent() } @@ -900,7 +981,11 @@ func mapToXmlIndent(doIndent bool, s *string, key string, value interface{}, pp return nil case nil: // terminate the tag + if doIndent { + *s += p.padding + } *s += "<" + key + endTag, isSimple = true, true break default: // handle anything - even goofy stuff elen = 0 diff --git a/vendor/github.com/clbanning/mxj/xmlseq.go b/vendor/github.com/clbanning/mxj/xmlseq.go index 985a037f9bc..40c9757e6a4 100644 --- a/vendor/github.com/clbanning/mxj/xmlseq.go +++ b/vendor/github.com/clbanning/mxj/xmlseq.go @@ -65,6 +65,7 @@ var NO_ROOT = NoRoot // maintain backwards compatibility // extraneous xml.CharData will be ignored unless io.EOF is reached first. // 2. CoerceKeysToLower() is NOT recognized, since the intent here is to eventually call m.XmlSeq() to // re-encode the message in its original structure. +// 3. If CoerceKeysToSnakeCase() has been called, then all key values will be converted to snake case. // // NAME SPACES: // 1. Keys in the Map value that are parsed from a : tag preserve the @@ -86,6 +87,7 @@ func NewMapXmlSeq(xmlVal []byte, cast ...bool) (Map, error) { // extraneous xml.CharData will be ignored unless io.EOF is reached first. // 2. CoerceKeysToLower() is NOT recognized, since the intent here is to eventually call m.XmlSeq() to // re-encode the message in its original structure. +// 3. If CoerceKeysToSnakeCase() has been called, then all key values will be converted to snake case. func NewMapXmlSeqReader(xmlReader io.Reader, cast ...bool) (Map, error) { var r bool if len(cast) == 1 { @@ -117,23 +119,21 @@ func NewMapXmlSeqReader(xmlReader io.Reader, cast ...bool) (Map, error) { // extraneous xml.CharData will be ignored unless io.EOF is reached first. // 4. CoerceKeysToLower() is NOT recognized, since the intent here is to eventually call m.XmlSeq() to // re-encode the message in its original structure. +// 5. If CoerceKeysToSnakeCase() has been called, then all key values will be converted to snake case. func NewMapXmlSeqReaderRaw(xmlReader io.Reader, cast ...bool) (Map, []byte, error) { var r bool if len(cast) == 1 { r = cast[0] } // create TeeReader so we can retrieve raw XML - buf := make([]byte, XmlWriterBufSize) + buf := make([]byte, 0) wb := bytes.NewBuffer(buf) trdr := myTeeReader(xmlReader, wb) - // build the node tree m, err := xmlSeqReaderToMap(trdr, r) // retrieve the raw XML that was decoded - b := make([]byte, wb.Len()) - _, _ = wb.Read(b) - b = bytes.TrimSpace(b) + b := wb.Bytes() // err may be NoRoot return m, b, err @@ -168,6 +168,10 @@ func xmlSeqToMap(doc []byte, r bool) (map[string]interface{}, error) { // xmlSeqToMapParser - load a 'clean' XML doc into a map[string]interface{} directly. // Add #seq tag value for each element decoded - to be used for Encoding later. func xmlSeqToMapParser(skey string, a []xml.Attr, p *xml.Decoder, r bool) (map[string]interface{}, error) { + if snakeCaseKeys { + skey = strings.Replace(skey, "-", "_", -1) + } + // NOTE: all attributes and sub-elements parsed into 'na', 'na' is returned as value for 'skey' in 'n'. var n, na map[string]interface{} var seq int // for including seq num when decoding @@ -186,6 +190,9 @@ func xmlSeqToMapParser(skey string, a []xml.Attr, p *xml.Decoder, r bool) (map[s // where interface{} is map[string]interface{}{"#text":, "#seq":} aa := make(map[string]interface{}, len(a)) for i, v := range a { + if snakeCaseKeys { + v.Name.Local = strings.Replace(v.Name.Local, "-", "_", -1) + } if len(v.Name.Space) > 0 { aa[v.Name.Space+`:`+v.Name.Local] = map[string]interface{}{"#text": cast(v.Value, r), "#seq": i} } else { @@ -195,6 +202,13 @@ func xmlSeqToMapParser(skey string, a []xml.Attr, p *xml.Decoder, r bool) (map[s na["#attr"] = aa } } + + // Return XMPP message. + if handleXMPPStreamTag && skey == "stream:stream" { + n[skey] = na + return n, nil + } + for { t, err := p.RawToken() if err != nil { @@ -280,6 +294,9 @@ func xmlSeqToMapParser(skey string, a []xml.Attr, p *xml.Decoder, r bool) (map[s case xml.EndElement: if skey != "" { tt := t.(xml.EndElement) + if snakeCaseKeys { + tt.Name.Local = strings.Replace(tt.Name.Local, "-", "_", -1) + } var name string if len(tt.Name.Space) > 0 { name = tt.Name.Space + `:` + tt.Name.Local @@ -637,9 +654,7 @@ func mapToXmlSeqIndent(doIndent bool, s *string, key string, value interface{}, } // something more complex p.mapDepth++ - // PrintElemListSeq(elemListSeq(kv)) sort.Sort(elemListSeq(kv)) - // PrintElemListSeq(elemListSeq(kv)) i := 0 for _, v := range kv { switch v.v.(type) { @@ -650,7 +665,9 @@ func mapToXmlSeqIndent(doIndent bool, s *string, key string, value interface{}, } } i++ - mapToXmlSeqIndent(doIndent, s, v.k, v.v, p) + if err := mapToXmlSeqIndent(doIndent, s, v.k, v.v, p); err != nil { + return err + } switch v.v.(type) { case []interface{}: // handled in []interface{} case default: @@ -668,7 +685,9 @@ func mapToXmlSeqIndent(doIndent bool, s *string, key string, value interface{}, if doIndent { p.Indent() } - mapToXmlSeqIndent(doIndent, s, key, v, p) + if err := mapToXmlSeqIndent(doIndent, s, key, v, p); err != nil { + return err + } if doIndent { p.Outdent() } @@ -676,7 +695,11 @@ func mapToXmlSeqIndent(doIndent bool, s *string, key string, value interface{}, return nil case nil: // terminate the tag + if doIndent { + *s += p.padding + } *s += "<" + key + endTag, isSimple = true, true break default: // handle anything - even goofy stuff elen = 0 @@ -796,12 +819,6 @@ func (e elemListSeq) Less(i, j int) bool { return true } -func PrintElemListSeq(e elemListSeq) { - for n, v := range e { - fmt.Printf("%d: %v\n", n, v) - } -} - // =============== https://groups.google.com/forum/#!topic/golang-nuts/lHPOHD-8qio // BeautifyXml (re)formats an XML doc similar to Map.XmlIndent(). diff --git a/vendor/vendor.json b/vendor/vendor.json index 61d0cb9baba..3271d675ef9 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -85,10 +85,10 @@ "revisionTime": "2017-01-23T21:22:43Z" }, { - "checksumSHA1": "ZpbyHYDGQyxkwm325SX9yLTYIj8=", + "checksumSHA1": "Y9anndxw8wlCmbcT1Wx05q57y7U=", "path": "github.com/clbanning/mxj", - "revision": "be8d58af431713d76a316ce978f9c0909593d005", - "revisionTime": "2016-11-09T14:28:45Z" + "revision": "1cda74d18ebbb04f525b355309c16d594e1f8f77", + "revisionTime": "2017-06-20T21:51:35Z" }, { "checksumSHA1": "2Fy1Y6Z3lRRX1891WF/+HT4XS2I=",