From 6d7d76a3f100e54f9ab69d2d83eb54978325a688 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Mon, 23 Jan 2023 11:37:18 +1100 Subject: [PATCH] Adds @uri/@urid #1529 --- pkg/yqlib/decoder.go | 1 + pkg/yqlib/decoder_uri.go | 60 +++++++++++++++++++ pkg/yqlib/doc/operators/encode-decode.md | 29 +++++++++ .../doc/operators/headers/encode-decode.md | 1 + pkg/yqlib/encoder_uri.go | 37 ++++++++++++ pkg/yqlib/lexer_participle.go | 3 + pkg/yqlib/operator_encoder_decoder.go | 4 ++ pkg/yqlib/operator_encoder_decoder_test.go | 16 +++++ pkg/yqlib/printer.go | 1 + 9 files changed, 152 insertions(+) create mode 100644 pkg/yqlib/decoder_uri.go create mode 100644 pkg/yqlib/encoder_uri.go diff --git a/pkg/yqlib/decoder.go b/pkg/yqlib/decoder.go index 904b65c3ae..ff7ab2db9d 100644 --- a/pkg/yqlib/decoder.go +++ b/pkg/yqlib/decoder.go @@ -15,6 +15,7 @@ const ( JsonInputFormat CSVObjectInputFormat TSVObjectInputFormat + UriInputFormat ) type Decoder interface { diff --git a/pkg/yqlib/decoder_uri.go b/pkg/yqlib/decoder_uri.go new file mode 100644 index 0000000000..eabf26beb5 --- /dev/null +++ b/pkg/yqlib/decoder_uri.go @@ -0,0 +1,60 @@ +package yqlib + +import ( + "bytes" + "io" + "net/url" + + yaml "gopkg.in/yaml.v3" +) + +type uriDecoder struct { + reader io.Reader + finished bool + readAnything bool +} + +func NewUriDecoder() Decoder { + return &uriDecoder{finished: false} +} + +func (dec *uriDecoder) Init(reader io.Reader) error { + dec.reader = reader + dec.readAnything = false + dec.finished = false + return nil +} + +func (dec *uriDecoder) Decode() (*CandidateNode, error) { + if dec.finished { + return nil, io.EOF + } + + buf := new(bytes.Buffer) + + if _, err := buf.ReadFrom(dec.reader); err != nil { + return nil, err + } + if buf.Len() == 0 { + dec.finished = true + + // if we've read _only_ an empty string, lets return that + // otherwise if we've already read some bytes, and now we get + // an empty string, then we are done. + if dec.readAnything { + return nil, io.EOF + } + } + newValue, err := url.QueryUnescape(buf.String()) + if err != nil { + return nil, err + } + dec.readAnything = true + return &CandidateNode{ + Node: &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: newValue, + }, + }, nil +} diff --git a/pkg/yqlib/doc/operators/encode-decode.md b/pkg/yqlib/doc/operators/encode-decode.md index f9e5097df1..f6be772cda 100644 --- a/pkg/yqlib/doc/operators/encode-decode.md +++ b/pkg/yqlib/doc/operators/encode-decode.md @@ -16,6 +16,7 @@ These operators are useful to process yaml documents that have stringified embed | TSV | from_tsv/@tsvd | to_tsv/@tsv | | XML | from_xml/@xmld | to_xml(i)/@xml | | Base64 | @base64d | @base64 | +| URI | @urid | @uri | See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats. @@ -435,6 +436,34 @@ will output YTogYXBwbGUK ``` +## Encode a string to uri +Given a sample.yml file of: +```yaml +coolData: this has & special () characters * +``` +then +```bash +yq '.coolData | @uri' sample.yml +``` +will output +```yaml +this+has+%26+special+%28%29+characters+%2A +``` + +## Decode a URI to a string +Given a sample.yml file of: +```yaml +this+has+%26+special+%28%29+characters+%2A +``` +then +```bash +yq '@urid' sample.yml +``` +will output +```yaml +this has & special () characters * +``` + ## Decode a base64 encoded string Decoded data is assumed to be a string. diff --git a/pkg/yqlib/doc/operators/headers/encode-decode.md b/pkg/yqlib/doc/operators/headers/encode-decode.md index f83b45df1c..31c1c10bf0 100644 --- a/pkg/yqlib/doc/operators/headers/encode-decode.md +++ b/pkg/yqlib/doc/operators/headers/encode-decode.md @@ -16,6 +16,7 @@ These operators are useful to process yaml documents that have stringified embed | TSV | from_tsv/@tsvd | to_tsv/@tsv | | XML | from_xml/@xmld | to_xml(i)/@xml | | Base64 | @base64d | @base64 | +| URI | @urid | @uri | See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats. diff --git a/pkg/yqlib/encoder_uri.go b/pkg/yqlib/encoder_uri.go new file mode 100644 index 0000000000..e4141cb878 --- /dev/null +++ b/pkg/yqlib/encoder_uri.go @@ -0,0 +1,37 @@ +package yqlib + +import ( + "fmt" + "io" + "net/url" + + yaml "gopkg.in/yaml.v3" +) + +type uriEncoder struct { +} + +func NewUriEncoder() Encoder { + return &uriEncoder{} +} + +func (e *uriEncoder) CanHandleAliases() bool { + return false +} + +func (e *uriEncoder) PrintDocumentSeparator(writer io.Writer) error { + return nil +} + +func (e *uriEncoder) PrintLeadingContent(writer io.Writer, content string) error { + return nil +} + +func (e *uriEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error { + node := unwrapDoc(originalNode) + if guessTagFromCustomType(node) != "!!str" { + return fmt.Errorf("cannot encode %v as url, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag) + } + _, err := writer.Write([]byte(url.QueryEscape(originalNode.Value))) + return err +} diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index 14ea14e8f0..547c181023 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -78,6 +78,9 @@ var participleYqRules = []*participleYqRule{ {"Base64d", `@base64d`, decodeOp(Base64InputFormat), 0}, {"Base64", `@base64`, encodeWithIndent(Base64OutputFormat, 0), 0}, + {"Urld", `@urid`, decodeOp(UriInputFormat), 0}, + {"Url", `@uri`, encodeWithIndent(UriOutputFormat, 0), 0}, + {"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(ConfiguredXMLPreferences), false), 0}, {"LoadBase64", `load_?base64`, loadOp(NewBase64Decoder(), false), 0}, diff --git a/pkg/yqlib/operator_encoder_decoder.go b/pkg/yqlib/operator_encoder_decoder.go index 8f41389b6d..2fa0cb05b7 100644 --- a/pkg/yqlib/operator_encoder_decoder.go +++ b/pkg/yqlib/operator_encoder_decoder.go @@ -26,6 +26,8 @@ func configureEncoder(format PrinterOutputFormat, indent int) Encoder { return NewXMLEncoder(indent, ConfiguredXMLPreferences) case Base64OutputFormat: return NewBase64Encoder() + case UriOutputFormat: + return NewUriEncoder() } panic("invalid encoder") } @@ -113,6 +115,8 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre decoder = NewCSVObjectDecoder(',') case TSVObjectInputFormat: decoder = NewCSVObjectDecoder('\t') + case UriInputFormat: + decoder = NewUriDecoder() } var results = list.New() diff --git a/pkg/yqlib/operator_encoder_decoder_test.go b/pkg/yqlib/operator_encoder_decoder_test.go index e3961fbe68..ee54868ef1 100644 --- a/pkg/yqlib/operator_encoder_decoder_test.go +++ b/pkg/yqlib/operator_encoder_decoder_test.go @@ -241,6 +241,22 @@ var encoderDecoderOperatorScenarios = []expressionScenario{ "D0, P[], (!!str)::YTogYXBwbGUK\n", }, }, + { + description: "Encode a string to uri", + document: "coolData: this has & special () characters *", + expression: ".coolData | @uri", + expected: []string{ + "D0, P[coolData], (!!str)::this+has+%26+special+%28%29+characters+%2A\n", + }, + }, + { + description: "Decode a URI to a string", + document: "this+has+%26+special+%28%29+characters+%2A", + expression: "@urid", + expected: []string{ + "D0, P[], (!!str)::this has & special () characters *\n", + }, + }, { description: "Decode a base64 encoded string", subdescription: "Decoded data is assumed to be a string.", diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index aef082d9ee..14f96cc3bc 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -27,6 +27,7 @@ const ( TSVOutputFormat XMLOutputFormat Base64OutputFormat + UriOutputFormat ) func OutputFormatFromString(format string) (PrinterOutputFormat, error) {