Detect input format based on file name extension (#1582)

* detect inputFormat from filename * refactor and extract func InputFormatFromFilename * detect inputFormat only when file is provided * add test for automatic input format detection
mikefarah · Mar 8, 2023 · d30941b · d30941b
1 parent fed96f6
commit d30941b
Show file tree

Hide file tree

Showing 7 changed files with 260 additions and 7 deletions.
diff --git a/acceptance_tests/inputs-format-auto.sh b/acceptance_tests/inputs-format-auto.sh
@@ -0,0 +1,221 @@
+#!/bin/bash
+
+setUp() {
+  rm test*.yml 2>/dev/null || true
+  rm test*.json 2>/dev/null || true
+  rm test*.properties 2>/dev/null || true
+  rm test*.csv 2>/dev/null || true
+  rm test*.tsv 2>/dev/null || true
+  rm test*.xml 2>/dev/null || true
+}
+
+testInputJson() {
+  cat >test.json <<EOL
+{ "mike" : { "things": "cool" } }
+EOL
+
+  read -r -d '' expected << EOM
+mike:
+  things: cool
+EOM
+
+  X=$(./yq test.json)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.json)
+  assertEquals "$expected" "$X"
+}
+
+testInputProperties() {
+  cat >test.properties <<EOL
+mike.things = hello
+EOL
+
+  read -r -d '' expected << EOM
+mike:
+  things: hello
+EOM
+
+  X=$(./yq e test.properties)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.properties)
+  assertEquals "$expected" "$X"
+}
+
+testInputPropertiesGitHubAction() {
+  cat >test.properties <<EOL
+mike.things = hello
+EOL
+
+  read -r -d '' expected << EOM
+mike:
+  things: hello
+EOM
+
+  X=$(cat /dev/null | ./yq e test.properties)
+  assertEquals "$expected" "$X"
+
+  X=$(cat /dev/null | ./yq ea test.properties)
+  assertEquals "$expected" "$X"
+}
+
+testInputCSV() {
+  cat >test.csv <<EOL
+fruit,yumLevel
+apple,5
+banana,4
+EOL
+
+  read -r -d '' expected << EOM
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOM
+
+  X=$(./yq e test.csv)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.csv)
+  assertEquals "$expected" "$X"
+}
+
+testInputCSVUTF8() {
+  read -r -d '' expected << EOM
+- id: 1
+  first: john
+  last: smith
+- id: 1
+  first: jane
+  last: smith
+EOM
+
+  X=$(./yq utf8.csv)
+  assertEquals "$expected" "$X"
+}
+
+testInputTSV() {
+  cat >test.tsv <<EOL
+fruit	yumLevel
+apple	5
+banana	4
+EOL
+
+  read -r -d '' expected << EOM
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOM
+
+  X=$(./yq e test.tsv)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.tsv)
+  assertEquals "$expected" "$X"
+}
+
+
+
+
+testInputXml() {
+  cat >test.xml <<EOL
+<cat legs="4">BiBi</cat>
+EOL
+
+  read -r -d '' expected << EOM
+cat:
+  +content: BiBi
+  +@legs: "4"
+EOM
+
+  X=$(./yq e test.xml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.xml)
+  assertEquals "$expected" "$X"
+}
+
+testInputXmlNamespaces() {
+  cat >test.xml <<EOL
+<?xml version="1.0"?>
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
+</map>
+EOL
+
+  read -r -d '' expected << EOM
++p_xml: version="1.0"
+map:
+  +@xmlns: some-namespace
+  +@xmlns:xsi: some-instance
+  +@xsi:schemaLocation: some-url
+EOM
+
+  X=$(./yq e test.xml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.xml)
+  assertEquals "$expected" "$X"
+}
+
+testInputXmlRoundtrip() {
+  cat >test.xml <<EOL
+<?xml version="1.0"?>
+<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
+EOL
+
+  read -r -d '' expected << EOM
+<?xml version="1.0"?>
+<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
+EOM
+
+  X=$(./yq -o=xml test.xml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=xml test.xml)
+  assertEquals "$expected" "$X"
+}
+
+
+testInputXmlStrict() {
+  cat >test.xml <<EOL
+<?xml version="1.0"?>
+<!DOCTYPE root [
+<!ENTITY writer "Catherine.">
+<!ENTITY copyright "(r) Great">
+]>
+<root>
+    <item>&writer;&copyright;</item>
+</root>
+EOL
+
+  X=$(./yq --xml-strict-mode test.xml -o=xml 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
+
+  X=$(./yq ea --xml-strict-mode test.xml -o=xml 2>&1)
+  assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
+}
+
+testInputXmlGithubAction() {
+  cat >test.xml <<EOL
+<cat legs="4">BiBi</cat>
+EOL
+
+  read -r -d '' expected << EOM
+cat:
+  +content: BiBi
+  +@legs: "4"
+EOM
+
+  X=$(cat /dev/null | ./yq e test.xml)
+  assertEquals "$expected" "$X"
+
+  X=$(cat /dev/null | ./yq ea test.xml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2
diff --git a/cmd/constant.go b/cmd/constant.go
@@ -7,7 +7,8 @@ var unwrapScalar = false
 var writeInplace = false
 var outputToJSON = false
 var outputFormat = "yaml"
-var inputFormat = "yaml"
+var inputFormatDefault = "yaml"
+var inputFormat = ""
 
 var exitStatus = false
 var forceColor = false

diff --git a/cmd/evaluate_all_command.go b/cmd/evaluate_all_command.go
@@ -75,7 +75,12 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 		return err
 	}
 
-	decoder, err := configureDecoder(true)
+	inputFilename := ""
+	if len(args) > 0 {
+		inputFilename = args[0]
+	}
+
+	decoder, err := configureDecoder(true, inputFilename)
 	if err != nil {
 		return err
 	}

diff --git a/cmd/evalute_sequence_command.go b/cmd/evalute_sequence_command.go
@@ -100,7 +100,12 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 
 	printer := yqlib.NewPrinter(encoder, printerWriter)
 
-	decoder, err := configureDecoder(false)
+	inputFilename := ""
+	if len(args) > 0 {
+		inputFilename = args[0]
+	}
+
+	decoder, err := configureDecoder(false, inputFilename)
 	if err != nil {
 		return err
 	}

diff --git a/cmd/root.go b/cmd/root.go
@@ -85,7 +85,7 @@ yq -P sample.json
 	}
 
 	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p|xml|x] output format type.")
-	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "yaml", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
+	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
 
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")

diff --git a/cmd/utils.go b/cmd/utils.go
@@ -56,7 +56,12 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 	return expression, args, nil
 }
 
-func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
+func configureDecoder(evaluateTogether bool, inputFilename string) (yqlib.Decoder, error) {
+	if inputFormat == "" {
+		inputFormat = yqlib.InputFormatFromFilename(inputFilename, inputFormatDefault)
+	} else {
+		yqlib.GetLogger().Debugf("user specified inputFormat '%s'", inputFormat)
+	}
 	yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat)
 	if err != nil {
 		return nil, err

diff --git a/pkg/yqlib/decoder.go b/pkg/yqlib/decoder.go
@@ -3,6 +3,7 @@ package yqlib
 import (
 	"fmt"
 	"io"
+	"strings"
 )
 
 type InputFormat uint
@@ -25,11 +26,11 @@ type Decoder interface {
 
 func InputFormatFromString(format string) (InputFormat, error) {
 	switch format {
-	case "yaml", "y":
+	case "yaml", "yml", "y":
 		return YamlInputFormat, nil
 	case "xml", "x":
 		return XMLInputFormat, nil
-	case "props", "p":
+	case "properties", "props", "p":
 		return PropertiesInputFormat, nil
 	case "json", "ndjson", "j":
 		return JsonInputFormat, nil
@@ -41,3 +42,18 @@ func InputFormatFromString(format string) (InputFormat, error) {
 		return 0, fmt.Errorf("unknown format '%v' please use [yaml|xml|props]", format)
 	}
 }
+
+func InputFormatFromFilename(filename string, defaultFormat string) string {
+	if filename != "" {
+		GetLogger().Debugf("checking filename '%s' for inputFormat", filename)
+		nPos := strings.LastIndex(filename, ".")
+		if nPos > -1 {
+			inputFormat := filename[nPos+1:]
+			GetLogger().Debugf("detected inputFormat '%s'", inputFormat)
+			return inputFormat
+		}
+	}
+
+	GetLogger().Debugf("using default inputFormat '%s'", defaultFormat)
+	return defaultFormat
+}