From cf8cfbd8656fec53814d88ad14fb00b3513b575e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20Bj=C3=B6rklund?= Date: Wed, 1 Mar 2023 03:19:06 +0100 Subject: [PATCH] Allow build without json and xml support (#1556) * Refactor ordered_map into separate files Separate json and xml, from the regular yaml. Makes it possible to compile, without those... * Refactor encoder and decoder creation Use more consistent parameters vs globals Return errors instead of calling panic() * Allow build without json and xml support --- cmd/evaluate_all_command.go | 5 +- cmd/evalute_sequence_command.go | 5 +- cmd/utils.go | 46 ++++-- pkg/yqlib/decoder_json.go | 2 + pkg/yqlib/decoder_xml.go | 2 + pkg/yqlib/encoder.go | 163 +-------------------- pkg/yqlib/encoder_json.go | 17 +-- pkg/yqlib/encoder_test.go | 2 + pkg/yqlib/encoder_xml.go | 2 + pkg/yqlib/json_test.go | 2 + pkg/yqlib/no_json.go | 11 ++ pkg/yqlib/no_xml.go | 11 ++ pkg/yqlib/operator_encoder_decoder.go | 26 +++- pkg/yqlib/operator_encoder_decoder_test.go | 45 +++--- pkg/yqlib/operator_load.go | 3 + pkg/yqlib/operator_load_test.go | 7 +- pkg/yqlib/operators_test.go | 18 +++ pkg/yqlib/ordered_map.go | 14 ++ pkg/yqlib/ordered_map_json.go | 83 +++++++++++ pkg/yqlib/ordered_map_yaml.go | 79 ++++++++++ pkg/yqlib/printer_test.go | 6 +- pkg/yqlib/xml_test.go | 2 + 22 files changed, 338 insertions(+), 213 deletions(-) create mode 100644 pkg/yqlib/no_json.go create mode 100644 pkg/yqlib/no_xml.go create mode 100644 pkg/yqlib/ordered_map.go create mode 100644 pkg/yqlib/ordered_map_json.go create mode 100644 pkg/yqlib/ordered_map_yaml.go diff --git a/cmd/evaluate_all_command.go b/cmd/evaluate_all_command.go index 63b6887107..fff64fa53f 100644 --- a/cmd/evaluate_all_command.go +++ b/cmd/evaluate_all_command.go @@ -84,7 +84,10 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) { if err != nil { return err } - encoder := configureEncoder(format) + encoder, err := configureEncoder() + if err != nil { + return err + } printer := yqlib.NewPrinter(encoder, printerWriter) diff --git a/cmd/evalute_sequence_command.go b/cmd/evalute_sequence_command.go index 53108165a5..dab803c3c3 100644 --- a/cmd/evalute_sequence_command.go +++ b/cmd/evalute_sequence_command.go @@ -93,7 +93,10 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) { if err != nil { return err } - encoder := configureEncoder(format) + encoder, err := configureEncoder() + if err != nil { + return err + } printer := yqlib.NewPrinter(encoder, printerWriter) diff --git a/cmd/utils.go b/cmd/utils.go index 8dad7c3079..671dd86b66 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -61,7 +61,15 @@ func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) { if err != nil { return nil, err } - switch yqlibInputFormat { + yqlibDecoder, err := createDecoder(yqlibInputFormat, evaluateTogether) + if yqlibDecoder == nil { + return nil, fmt.Errorf("no support for %s input format", inputFormat) + } + return yqlibDecoder, err +} + +func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decoder, error) { + switch format { case yqlib.XMLInputFormat: return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil case yqlib.PropertiesInputFormat: @@ -72,10 +80,12 @@ func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) { return yqlib.NewCSVObjectDecoder(','), nil case yqlib.TSVObjectInputFormat: return yqlib.NewCSVObjectDecoder('\t'), nil + case yqlib.YamlInputFormat: + prefs := yqlib.ConfiguredYamlPreferences + prefs.EvaluateTogether = evaluateTogether + return yqlib.NewYamlDecoder(prefs), nil } - prefs := yqlib.ConfiguredYamlPreferences - prefs.EvaluateTogether = evaluateTogether - return yqlib.NewYamlDecoder(prefs), nil + return nil, fmt.Errorf("invalid decoder: %v", format) } func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) { @@ -95,22 +105,34 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq return printerWriter, nil } -func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder { +func configureEncoder() (yqlib.Encoder, error) { + yqlibOutputFormat, err := yqlib.OutputFormatFromString(outputFormat) + if err != nil { + return nil, err + } + yqlibEncoder, err := createEncoder(yqlibOutputFormat) + if yqlibEncoder == nil { + return nil, fmt.Errorf("no support for %s output format", outputFormat) + } + return yqlibEncoder, err +} + +func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) { switch format { case yqlib.JSONOutputFormat: - return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar) + return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar), nil case yqlib.PropsOutputFormat: - return yqlib.NewPropertiesEncoder(unwrapScalar) + return yqlib.NewPropertiesEncoder(unwrapScalar), nil case yqlib.CSVOutputFormat: - return yqlib.NewCsvEncoder(',') + return yqlib.NewCsvEncoder(','), nil case yqlib.TSVOutputFormat: - return yqlib.NewCsvEncoder('\t') + return yqlib.NewCsvEncoder('\t'), nil case yqlib.YamlOutputFormat: - return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences) + return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences), nil case yqlib.XMLOutputFormat: - return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences) + return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences), nil } - panic("invalid encoder") + return nil, fmt.Errorf("invalid encoder: %v", format) } // this is a hack to enable backwards compatibility with githubactions (which pipe /dev/null into everything) diff --git a/pkg/yqlib/decoder_json.go b/pkg/yqlib/decoder_json.go index a8e1e6052e..35da646775 100644 --- a/pkg/yqlib/decoder_json.go +++ b/pkg/yqlib/decoder_json.go @@ -1,3 +1,5 @@ +//go:build !yq_nojson + package yqlib import ( diff --git a/pkg/yqlib/decoder_xml.go b/pkg/yqlib/decoder_xml.go index ac70e403d3..bfb18e125a 100644 --- a/pkg/yqlib/decoder_xml.go +++ b/pkg/yqlib/decoder_xml.go @@ -1,3 +1,5 @@ +//go:build !yq_noxml + package yqlib import ( diff --git a/pkg/yqlib/encoder.go b/pkg/yqlib/encoder.go index e9fae6afee..e0ecaccf94 100644 --- a/pkg/yqlib/encoder.go +++ b/pkg/yqlib/encoder.go @@ -1,10 +1,6 @@ package yqlib import ( - "bytes" - "encoding/json" - "errors" - "fmt" "io" yaml "gopkg.in/yaml.v3" @@ -17,162 +13,17 @@ type Encoder interface { CanHandleAliases() bool } -// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the -// order of keys and values in a map or an object. -type orderedMap struct { - // if this is an object, kv != nil. If this is not an object, kv == nil. - kv []orderedMapKV - altVal interface{} -} - -type orderedMapKV struct { - K string - V orderedMap -} - -func (o *orderedMap) UnmarshalJSON(data []byte) error { - switch data[0] { - case '{': - // initialise so that even if the object is empty it is not nil - o.kv = []orderedMapKV{} - - // create decoder - dec := json.NewDecoder(bytes.NewReader(data)) - _, err := dec.Token() // open object - if err != nil { - return err - } - - // cycle through k/v - var tok json.Token - for tok, err = dec.Token(); err == nil; tok, err = dec.Token() { - // we can expect two types: string or Delim. Delim automatically means - // that it is the closing bracket of the object, whereas string means - // that there is another key. - if _, ok := tok.(json.Delim); ok { - break - } - kv := orderedMapKV{ - K: tok.(string), - } - if err := dec.Decode(&kv.V); err != nil { - return err - } - o.kv = append(o.kv, kv) - } - // unexpected error - if err != nil && !errors.Is(err, io.EOF) { - return err - } - return nil - case '[': - var res []*orderedMap - if err := json.Unmarshal(data, &res); err != nil { - return err - } - o.altVal = res - o.kv = nil - return nil - } - - return json.Unmarshal(data, &o.altVal) -} +func mapKeysToStrings(node *yaml.Node) { -func (o orderedMap) MarshalJSON() ([]byte, error) { - buf := new(bytes.Buffer) - enc := json.NewEncoder(buf) - enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, > - if o.kv == nil { - if err := enc.Encode(o.altVal); err != nil { - return nil, err - } - return buf.Bytes(), nil - } - buf.WriteByte('{') - for idx, el := range o.kv { - if err := enc.Encode(el.K); err != nil { - return nil, err - } - buf.WriteByte(':') - if err := enc.Encode(el.V); err != nil { - return nil, err - } - if idx != len(o.kv)-1 { - buf.WriteByte(',') - } - } - buf.WriteByte('}') - return buf.Bytes(), nil -} - -func (o *orderedMap) UnmarshalYAML(node *yaml.Node) error { - switch node.Kind { - case yaml.DocumentNode: - if len(node.Content) == 0 { - return nil - } - return o.UnmarshalYAML(node.Content[0]) - case yaml.AliasNode: - return o.UnmarshalYAML(node.Alias) - case yaml.ScalarNode: - return node.Decode(&o.altVal) - case yaml.MappingNode: - // set kv to non-nil - o.kv = []orderedMapKV{} - for i := 0; i < len(node.Content); i += 2 { - var key string - var val orderedMap - if err := node.Content[i].Decode(&key); err != nil { - return err + if node.Kind == yaml.MappingNode { + for index, child := range node.Content { + if index%2 == 0 { // its a map key + child.Tag = "!!str" } - if err := node.Content[i+1].Decode(&val); err != nil { - return err - } - o.kv = append(o.kv, orderedMapKV{ - K: key, - V: val, - }) - } - return nil - case yaml.SequenceNode: - // note that this has to be a pointer, so that nulls can be represented. - var res []*orderedMap - if err := node.Decode(&res); err != nil { - return err } - o.altVal = res - o.kv = nil - return nil - case 0: - // null - o.kv = nil - o.altVal = nil - return nil - default: - return fmt.Errorf("orderedMap: invalid yaml node") } -} -func (o *orderedMap) MarshalYAML() (interface{}, error) { - // fast path: kv is nil, use altVal - if o.kv == nil { - return o.altVal, nil - } - content := make([]*yaml.Node, 0, len(o.kv)*2) - for _, val := range o.kv { - n := new(yaml.Node) - if err := n.Encode(val.V); err != nil { - return nil, err - } - content = append(content, &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: val.K, - }, n) + for _, child := range node.Content { + mapKeysToStrings(child) } - return &yaml.Node{ - Kind: yaml.MappingNode, - Tag: "!!map", - Content: content, - }, nil } diff --git a/pkg/yqlib/encoder_json.go b/pkg/yqlib/encoder_json.go index 9f5b71ac97..53c6bd061a 100644 --- a/pkg/yqlib/encoder_json.go +++ b/pkg/yqlib/encoder_json.go @@ -1,3 +1,5 @@ +//go:build !yq_nojson + package yqlib import ( @@ -14,21 +16,6 @@ type jsonEncoder struct { UnwrapScalar bool } -func mapKeysToStrings(node *yaml.Node) { - - if node.Kind == yaml.MappingNode { - for index, child := range node.Content { - if index%2 == 0 { // its a map key - child.Tag = "!!str" - } - } - } - - for _, child := range node.Content { - mapKeysToStrings(child) - } -} - func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder { var indentString = "" diff --git a/pkg/yqlib/encoder_test.go b/pkg/yqlib/encoder_test.go index a17c51a904..cb38456263 100644 --- a/pkg/yqlib/encoder_test.go +++ b/pkg/yqlib/encoder_test.go @@ -1,3 +1,5 @@ +//go:build !yq_nojson + package yqlib import ( diff --git a/pkg/yqlib/encoder_xml.go b/pkg/yqlib/encoder_xml.go index 1d2e77983f..d60ede8fc2 100644 --- a/pkg/yqlib/encoder_xml.go +++ b/pkg/yqlib/encoder_xml.go @@ -1,3 +1,5 @@ +//go:build !yq_noxml + package yqlib import ( diff --git a/pkg/yqlib/json_test.go b/pkg/yqlib/json_test.go index ad1b6bb810..b2eddb5805 100644 --- a/pkg/yqlib/json_test.go +++ b/pkg/yqlib/json_test.go @@ -1,3 +1,5 @@ +//go:build !yq_nojson + package yqlib import ( diff --git a/pkg/yqlib/no_json.go b/pkg/yqlib/no_json.go new file mode 100644 index 0000000000..ae9d531adf --- /dev/null +++ b/pkg/yqlib/no_json.go @@ -0,0 +1,11 @@ +//go:build yq_nojson + +package yqlib + +func NewJSONDecoder() Decoder { + return nil +} + +func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder { + return nil +} diff --git a/pkg/yqlib/no_xml.go b/pkg/yqlib/no_xml.go new file mode 100644 index 0000000000..d3f96bb6c2 --- /dev/null +++ b/pkg/yqlib/no_xml.go @@ -0,0 +1,11 @@ +//go:build yq_noxml + +package yqlib + +func NewXMLDecoder(prefs XmlPreferences) Decoder { + return nil +} + +func NewXMLEncoder(indent int, prefs XmlPreferences) Encoder { + return nil +} diff --git a/pkg/yqlib/operator_encoder_decoder.go b/pkg/yqlib/operator_encoder_decoder.go index 255b0664f5..1c9582d1ba 100644 --- a/pkg/yqlib/operator_encoder_decoder.go +++ b/pkg/yqlib/operator_encoder_decoder.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "container/list" + "errors" "regexp" "strings" @@ -39,6 +40,9 @@ func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string, log.Debug("printing with indent: %v", prefs.indent) encoder := configureEncoder(prefs.format, prefs.indent) + if encoder == nil { + return "", errors.New("no support for output format") + } printer := NewPrinter(encoder, NewSinglePrinterWriter(bufio.NewWriter(&output))) err := printer.PrintResults(candidate.AsList()) @@ -98,13 +102,11 @@ type decoderPreferences struct { format InputFormat } -/* takes a string and decodes it back into an object */ -func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - preferences := expressionNode.Operation.Preferences.(decoderPreferences) - +func createDecoder(format InputFormat) Decoder { var decoder Decoder - switch preferences.format { + switch format { + case JsonInputFormat: + decoder = NewJSONDecoder() case YamlInputFormat: decoder = NewYamlDecoder(ConfiguredYamlPreferences) case XMLInputFormat: @@ -120,6 +122,18 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre case UriInputFormat: decoder = NewUriDecoder() } + return decoder +} + +/* takes a string and decodes it back into an object */ +func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + + preferences := expressionNode.Operation.Preferences.(decoderPreferences) + + decoder := createDecoder(preferences.format) + if decoder == nil { + return Context{}, errors.New("no support for input format") + } var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { diff --git a/pkg/yqlib/operator_encoder_decoder_test.go b/pkg/yqlib/operator_encoder_decoder_test.go index 13448adf6b..00441f73de 100644 --- a/pkg/yqlib/operator_encoder_decoder_test.go +++ b/pkg/yqlib/operator_encoder_decoder_test.go @@ -8,15 +8,17 @@ var prefix = "D0, P[], (doc)::a:\n cool:\n bob: dylan\n" var encoderDecoderOperatorScenarios = []expressionScenario{ { - description: "Encode value as json string", - document: `{a: {cool: "thing"}}`, - expression: `.b = (.a | to_json)`, + requiresFormat: "json", + description: "Encode value as json string", + document: `{a: {cool: "thing"}}`, + expression: `.b = (.a | to_json)`, expected: []string{ `D0, P[], (doc)::{a: {cool: "thing"}, b: "{\n \"cool\": \"thing\"\n}\n"} `, }, }, { + requiresFormat: "json", description: "Encode value as json string, on one line", subdescription: "Pass in a 0 indent to print json on a single line.", document: `{a: {cool: "thing"}}`, @@ -27,6 +29,7 @@ var encoderDecoderOperatorScenarios = []expressionScenario{ }, }, { + requiresFormat: "json", description: "Encode value as json string, on one line shorthand", subdescription: "Pass in a 0 indent to print json on a single line.", document: `{a: {cool: "thing"}}`, @@ -37,6 +40,7 @@ var encoderDecoderOperatorScenarios = []expressionScenario{ }, }, { + requiresFormat: "json", description: "Decode a json encoded string", subdescription: "Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling.", document: `a: '{"cool":"thing"}'`, @@ -193,33 +197,37 @@ var encoderDecoderOperatorScenarios = []expressionScenario{ }, }, { - description: "Encode value as xml string", - document: `{a: {cool: {foo: "bar", +@id: hi}}}`, - expression: `.a | to_xml`, + requiresFormat: "xml", + description: "Encode value as xml string", + document: `{a: {cool: {foo: "bar", +@id: hi}}}`, + expression: `.a | to_xml`, expected: []string{ "D0, P[a], (!!str)::\n bar\n\n\n", }, }, { - description: "Encode value as xml string on a single line", - document: `{a: {cool: {foo: "bar", +@id: hi}}}`, - expression: `.a | @xml`, + requiresFormat: "xml", + description: "Encode value as xml string on a single line", + document: `{a: {cool: {foo: "bar", +@id: hi}}}`, + expression: `.a | @xml`, expected: []string{ "D0, P[a], (!!str)::bar\n\n", }, }, { - description: "Encode value as xml string with custom indentation", - document: `{a: {cool: {foo: "bar", +@id: hi}}}`, - expression: `{"cat": .a | to_xml(1)}`, + requiresFormat: "xml", + description: "Encode value as xml string with custom indentation", + document: `{a: {cool: {foo: "bar", +@id: hi}}}`, + expression: `{"cat": .a | to_xml(1)}`, expected: []string{ "D0, P[], (!!map)::cat: |\n \n bar\n \n", }, }, { - description: "Decode a xml encoded string", - document: `a: "bar"`, - expression: `.b = (.a | from_xml)`, + requiresFormat: "xml", + description: "Decode a xml encoded string", + document: `a: "bar"`, + expression: `.b = (.a | from_xml)`, expected: []string{ "D0, P[], (doc)::a: \"bar\"\nb:\n foo: bar\n", }, @@ -303,9 +311,10 @@ var encoderDecoderOperatorScenarios = []expressionScenario{ }, }, { - description: "empty xml decode", - skipDoc: true, - expression: `"" | @xmld`, + requiresFormat: "xml", + description: "empty xml decode", + skipDoc: true, + expression: `"" | @xmld`, expected: []string{ "D0, P[], (!!null)::\n", }, diff --git a/pkg/yqlib/operator_load.go b/pkg/yqlib/operator_load.go index 7d083d5b8f..678854f666 100644 --- a/pkg/yqlib/operator_load.go +++ b/pkg/yqlib/operator_load.go @@ -34,6 +34,9 @@ func loadString(filename string) (*CandidateNode, error) { } func loadYaml(filename string, decoder Decoder) (*CandidateNode, error) { + if decoder == nil { + return nil, fmt.Errorf("could not load %s", filename) + } file, err := os.Open(filename) // #nosec if err != nil { diff --git a/pkg/yqlib/operator_load_test.go b/pkg/yqlib/operator_load_test.go index b27e8f703b..5bb30321fb 100644 --- a/pkg/yqlib/operator_load_test.go +++ b/pkg/yqlib/operator_load_test.go @@ -74,9 +74,10 @@ var loadScenarios = []expressionScenario{ }, }, { - description: "Load from XML", - document: "cool: things", - expression: `.more_stuff = load_xml("../../examples/small.xml")`, + requiresFormat: "xml", + description: "Load from XML", + document: "cool: things", + expression: `.more_stuff = load_xml("../../examples/small.xml")`, expected: []string{ "D0, P[], (doc)::cool: things\nmore_stuff:\n this: is some xml\n", }, diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 7975789525..c040cbedbc 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -28,6 +28,7 @@ type expressionScenario struct { skipDoc bool expectedError string dontFormatInputForDoc bool // dont format input doc for documentation generation + requiresFormat string } func TestMain(m *testing.M) { @@ -103,6 +104,23 @@ func testScenario(t *testing.T, s *expressionScenario) { return } + if s.requiresFormat != "" { + format := s.requiresFormat + inputFormat, err := InputFormatFromString(format) + if err != nil { + t.Error(err) + } + if decoder := createDecoder(inputFormat); decoder == nil { + t.Skipf("no support for %s input format", format) + } + outputFormat, err := OutputFormatFromString(format) + if err != nil { + t.Error(err) + } + if encoder := configureEncoder(outputFormat, 4); encoder == nil { + t.Skipf("no support for %s output format", format) + } + } if err != nil { t.Error(fmt.Errorf("%w: %v: %v", err, s.description, s.expression)) return diff --git a/pkg/yqlib/ordered_map.go b/pkg/yqlib/ordered_map.go new file mode 100644 index 0000000000..a783423b3a --- /dev/null +++ b/pkg/yqlib/ordered_map.go @@ -0,0 +1,14 @@ +package yqlib + +// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the +// order of keys and values in a map or an object. +type orderedMap struct { + // if this is an object, kv != nil. If this is not an object, kv == nil. + kv []orderedMapKV + altVal interface{} +} + +type orderedMapKV struct { + K string + V orderedMap +} diff --git a/pkg/yqlib/ordered_map_json.go b/pkg/yqlib/ordered_map_json.go new file mode 100644 index 0000000000..94a1d780f6 --- /dev/null +++ b/pkg/yqlib/ordered_map_json.go @@ -0,0 +1,83 @@ +package yqlib + +import ( + "bytes" + "encoding/json" + "errors" + "io" +) + +func (o *orderedMap) UnmarshalJSON(data []byte) error { + switch data[0] { + case '{': + // initialise so that even if the object is empty it is not nil + o.kv = []orderedMapKV{} + + // create decoder + dec := json.NewDecoder(bytes.NewReader(data)) + _, err := dec.Token() // open object + if err != nil { + return err + } + + // cycle through k/v + var tok json.Token + for tok, err = dec.Token(); err == nil; tok, err = dec.Token() { + // we can expect two types: string or Delim. Delim automatically means + // that it is the closing bracket of the object, whereas string means + // that there is another key. + if _, ok := tok.(json.Delim); ok { + break + } + kv := orderedMapKV{ + K: tok.(string), + } + if err := dec.Decode(&kv.V); err != nil { + return err + } + o.kv = append(o.kv, kv) + } + // unexpected error + if err != nil && !errors.Is(err, io.EOF) { + return err + } + return nil + case '[': + var res []*orderedMap + if err := json.Unmarshal(data, &res); err != nil { + return err + } + o.altVal = res + o.kv = nil + return nil + } + + return json.Unmarshal(data, &o.altVal) +} + +func (o orderedMap) MarshalJSON() ([]byte, error) { + buf := new(bytes.Buffer) + enc := json.NewEncoder(buf) + enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, > + if o.kv == nil { + if err := enc.Encode(o.altVal); err != nil { + return nil, err + } + return buf.Bytes(), nil + } + buf.WriteByte('{') + for idx, el := range o.kv { + if err := enc.Encode(el.K); err != nil { + return nil, err + } + buf.WriteByte(':') + if err := enc.Encode(el.V); err != nil { + return nil, err + } + if idx != len(o.kv)-1 { + buf.WriteByte(',') + } + } + buf.WriteByte('}') + return buf.Bytes(), nil +} diff --git a/pkg/yqlib/ordered_map_yaml.go b/pkg/yqlib/ordered_map_yaml.go new file mode 100644 index 0000000000..b3c494422e --- /dev/null +++ b/pkg/yqlib/ordered_map_yaml.go @@ -0,0 +1,79 @@ +package yqlib + +import ( + "fmt" + + yaml "gopkg.in/yaml.v3" +) + +func (o *orderedMap) UnmarshalYAML(node *yaml.Node) error { + switch node.Kind { + case yaml.DocumentNode: + if len(node.Content) == 0 { + return nil + } + return o.UnmarshalYAML(node.Content[0]) + case yaml.AliasNode: + return o.UnmarshalYAML(node.Alias) + case yaml.ScalarNode: + return node.Decode(&o.altVal) + case yaml.MappingNode: + // set kv to non-nil + o.kv = []orderedMapKV{} + for i := 0; i < len(node.Content); i += 2 { + var key string + var val orderedMap + if err := node.Content[i].Decode(&key); err != nil { + return err + } + if err := node.Content[i+1].Decode(&val); err != nil { + return err + } + o.kv = append(o.kv, orderedMapKV{ + K: key, + V: val, + }) + } + return nil + case yaml.SequenceNode: + // note that this has to be a pointer, so that nulls can be represented. + var res []*orderedMap + if err := node.Decode(&res); err != nil { + return err + } + o.altVal = res + o.kv = nil + return nil + case 0: + // null + o.kv = nil + o.altVal = nil + return nil + default: + return fmt.Errorf("orderedMap: invalid yaml node") + } +} + +func (o *orderedMap) MarshalYAML() (interface{}, error) { + // fast path: kv is nil, use altVal + if o.kv == nil { + return o.altVal, nil + } + content := make([]*yaml.Node, 0, len(o.kv)*2) + for _, val := range o.kv { + n := new(yaml.Node) + if err := n.Encode(val.V); err != nil { + return nil, err + } + content = append(content, &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: val.K, + }, n) + } + return &yaml.Node{ + Kind: yaml.MappingNode, + Tag: "!!map", + Content: content, + }, nil +} diff --git a/pkg/yqlib/printer_test.go b/pkg/yqlib/printer_test.go index cc7b3dbe05..f3be640119 100644 --- a/pkg/yqlib/printer_test.go +++ b/pkg/yqlib/printer_test.go @@ -314,7 +314,11 @@ func TestPrinterMultipleDocsJson(t *testing.T) { var writer = bufio.NewWriter(&output) // note printDocSeparators is true, it should still not print document separators // when outputing JSON. - printer := NewPrinter(NewJSONEncoder(0, false, false), NewSinglePrinterWriter(writer)) + encoder := NewJSONEncoder(0, false, false) + if encoder == nil { + t.Skipf("no support for %s output format", "json") + } + printer := NewPrinter(encoder, NewSinglePrinterWriter(writer)) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { diff --git a/pkg/yqlib/xml_test.go b/pkg/yqlib/xml_test.go index 2e9badf47b..ec9f5b6783 100644 --- a/pkg/yqlib/xml_test.go +++ b/pkg/yqlib/xml_test.go @@ -1,3 +1,5 @@ +//go:build !yq_noxml + package yqlib import (