iter.PeekWalk(walkFn)
- metadata, err := metadecoders.UnmarshalToMap(pf.frontMatterSource, pf.frontMatterFormat)
+ metadata, err := metadecoders.Default.UnmarshalToMap(pf.frontMatterSource, pf.frontMatterFormat)
if err != nil {
return pf, err
}
b, err := afero.ReadFile(fs, path)
- tomlMeta, err := metadecoders.UnmarshalToMap(b, metadecoders.TOML)
+ tomlMeta, err := metadecoders.Default.UnmarshalToMap(b, metadecoders.TOML)
if err != nil {
continue
return nil
}
- c, err := metadecoders.UnmarshalToMap(b, metadecoders.YAML)
+ c, err := metadecoders.Default.UnmarshalToMap(b, metadecoders.YAML)
if err != nil {
return nil
}
func readConfig(format metadecoders.Format, data []byte) (map[string]interface{}, error) {
- m, err := metadecoders.UnmarshalToMap(data, format)
+ m, err := metadecoders.Default.UnmarshalToMap(data, format)
if err != nil {
return nil, err
}
}
func loadConfigFromFile(fs afero.Fs, filename string) (map[string]interface{}, error) {
- m, err := metadecoders.UnmarshalFileToMap(fs, filename)
+ m, err := metadecoders.Default.UnmarshalFileToMap(fs, filename)
if err != nil {
return nil, err
}
github.com/magefile/mage v1.4.0/go.mod h1:IUDi13rsHje59lecXokTfGX0QIzO45uVPlXnJYsXepA=
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6 h1:LZhVjIISSbj8qLf2qDPP0D8z0uvOWAW5C85ly5mJW6c=
github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6/go.mod h1:oTeZL2KHA7CUX6X+fovmK9OvIOFuqu0TwdQrZjLTh88=
github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs=
github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
name := helpers.Filename(filepath.Base(path))
- item, err := metadecoders.UnmarshalFileToMap(sourceFs, path)
+ item, err := metadecoders.Default.UnmarshalFileToMap(sourceFs, path)
if err != nil {
return l.wrapFileError(err, path)
}
result.Write(it.Val)
case it.IsFrontMatter():
f := metadecoders.FormatFromFrontMatterType(it.Type)
- m, err := metadecoders.UnmarshalToMap(it.Val, f)
+ m, err := metadecoders.Default.UnmarshalToMap(it.Val, f)
if err != nil {
if fe, ok := err.(herrors.FileError); ok {
return herrors.ToFileErrorWithOffset(fe, iter.LineNumber()-1)
{"unmarshal", func() bool { return true }, func(b *sitesBuilder) {
b.WithTemplates("home.html", `
{{ $toml := "slogan = \"Hugo Rocks!\"" | resources.FromString "slogan.toml" | transform.Unmarshal }}
+{{ $csv1 := "\"Hugo Rocks\",\"Hugo is Fast!\"" | resources.FromString "slogans.csv" | transform.Unmarshal }}
+{{ $csv2 := "a;b;c" | resources.FromString "abc.csv" | transform.Unmarshal (dict "csvComma" ";") }}
+
Slogan: {{ $toml.slogan }}
+CSV1: {{ $csv1 }} {{ len (index $csv1 0) }}
+CSV2: {{ $csv2 }}
`)
}, func(b *sitesBuilder) {
- b.AssertFileContent("public/index.html", `Slogan: Hugo Rocks!`)
+ b.AssertFileContent("public/index.html",
+ `Slogan: Hugo Rocks!`,
+ `[[Hugo Rocks Hugo is Fast!]] 2`,
+ )
}},
{"template", func() bool { return true }, func(b *sitesBuilder) {}, func(b *sitesBuilder) {
content := helpers.ReaderToBytes(file)
format := metadecoders.FormatFromString(f.Extension())
- return metadecoders.Unmarshal(content, format)
+ return metadecoders.Default.Unmarshal(content, format)
}
func (s *Site) readDataFromSourceFS() error {
package metadecoders
import (
+ "bytes"
+ "encoding/csv"
"encoding/json"
"fmt"
yaml "gopkg.in/yaml.v2"
)
+// Decoder provides some configuration options for the decoders.
+type Decoder struct {
+ // Comma is the field delimiter used in the CSV decoder. It defaults to ','.
+ Comma rune
+
+ // Comment, if not 0, is the comment character ued in the CSV decoder. Lines beginning with the
+ // Comment character without preceding whitespace are ignored.
+ Comment rune
+}
+
+// Default is a Decoder in its default configuration.
+var Default = Decoder{
+ Comma: ',',
+}
+
// UnmarshalToMap will unmarshall data in format f into a new map. This is
// what's needed for Hugo's front matter decoding.
-func UnmarshalToMap(data []byte, f Format) (map[string]interface{}, error) {
+func (d Decoder) UnmarshalToMap(data []byte, f Format) (map[string]interface{}, error) {
m := make(map[string]interface{})
if data == nil {
return m, nil
}
- err := unmarshal(data, f, &m)
+ err := d.unmarshal(data, f, &m)
return m, err
}
// UnmarshalFileToMap is the same as UnmarshalToMap, but reads the data from
// the given filename.
-func UnmarshalFileToMap(fs afero.Fs, filename string) (map[string]interface{}, error) {
+func (d Decoder) UnmarshalFileToMap(fs afero.Fs, filename string) (map[string]interface{}, error) {
format := FormatFromString(filename)
if format == "" {
return nil, errors.Errorf("%q is not a valid configuration format", filename)
if err != nil {
return nil, err
}
- return UnmarshalToMap(data, format)
+ return d.UnmarshalToMap(data, format)
}
// Unmarshal will unmarshall data in format f into an interface{}.
// This is what's needed for Hugo's /data handling.
-func Unmarshal(data []byte, f Format) (interface{}, error) {
+func (d Decoder) Unmarshal(data []byte, f Format) (interface{}, error) {
if data == nil {
- return make(map[string]interface{}), nil
+ switch f {
+ case CSV:
+ return make([][]string, 0), nil
+ default:
+ return make(map[string]interface{}), nil
+ }
+
}
var v interface{}
- err := unmarshal(data, f, &v)
+ err := d.unmarshal(data, f, &v)
return v, err
}
// unmarshal unmarshals data in format f into v.
-func unmarshal(data []byte, f Format, v interface{}) error {
+func (d Decoder) unmarshal(data []byte, f Format, v interface{}) error {
var err error
*v.(*interface{}) = mm
}
}
+ case CSV:
+ return d.unmarshalCSV(data, v)
+
default:
return errors.Errorf("unmarshal of format %q is not supported", f)
}
}
+func (d Decoder) unmarshalCSV(data []byte, v interface{}) error {
+ r := csv.NewReader(bytes.NewReader(data))
+ r.Comma = d.Comma
+ r.Comment = d.Comment
+
+ records, err := r.ReadAll()
+ if err != nil {
+ return err
+ }
+
+ switch v.(type) {
+ case *interface{}:
+ *v.(*interface{}) = records
+ default:
+ return errors.Errorf("CSV cannot be unmarshaled into %T", v)
+
+ }
+
+ return nil
+
+}
+
func toFileError(f Format, err error) error {
return herrors.ToFileError(string(f), err)
}
expect := map[string]interface{}{"a": "b"}
+ d := Default
+
for i, test := range []struct {
data string
format Format
{`#+a: b`, ORG, expect},
// errors
{`a = b`, TOML, false},
+ {`a,b,c`, CSV, false}, // Use Unmarshal for CSV
} {
msg := fmt.Sprintf("%d: %s", i, test.format)
- m, err := UnmarshalToMap([]byte(test.data), test.format)
+ m, err := d.UnmarshalToMap([]byte(test.data), test.format)
if b, ok := test.expect.(bool); ok && !b {
assert.Error(err, msg)
} else {
expect := map[string]interface{}{"a": "b"}
+ d := Default
+
for i, test := range []struct {
data string
format Format
{`#+a: b`, ORG, expect},
{`a = "b"`, TOML, expect},
{`a: "b"`, YAML, expect},
+ {`a,b,c`, CSV, [][]string{[]string{"a", "b", "c"}}},
{"a: Easy!\nb:\n c: 2\n d: [3, 4]", YAML, map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}},
// errors
{`a = "`, TOML, false},
} {
msg := fmt.Sprintf("%d: %s", i, test.format)
- m, err := Unmarshal([]byte(test.data), test.format)
+ m, err := d.Unmarshal([]byte(test.data), test.format)
if b, ok := test.expect.(bool); ok && !b {
assert.Error(err, msg)
} else {
JSON Format = "json"
TOML Format = "toml"
YAML Format = "yaml"
+ CSV Format = "csv"
)
// FormatFromString turns formatStr, typically a file extension without any ".",
return TOML
case "org":
return ORG
+ case "csv":
+ return CSV
}
return ""
// FormatFromContentString tries to detect the format (JSON, YAML or TOML)
// in the given string.
// It return an empty string if no format could be detected.
-func FormatFromContentString(data string) Format {
+func (d Decoder) FormatFromContentString(data string) Format {
+ csvIdx := strings.IndexRune(data, d.Comma)
jsonIdx := strings.Index(data, "{")
yamlIdx := strings.Index(data, ":")
tomlIdx := strings.Index(data, "=")
+ if isLowerIndexThan(csvIdx, jsonIdx, yamlIdx, tomlIdx) {
+ return CSV
+ }
+
if isLowerIndexThan(jsonIdx, yamlIdx, tomlIdx) {
return JSON
}
{`foo: "bar"`, YAML},
{`foo:"bar"`, YAML},
{`{ "foo": "bar"`, JSON},
+ {`a,b,c"`, CSV},
{`asdfasdf`, Format("")},
{``, Format("")},
} {
errMsg := fmt.Sprintf("[%d] %s", i, test.data)
- result := FormatFromContentString(test.data)
+ result := Default.FormatFromContentString(test.data)
assert.Equal(test.expect, result, errMsg)
}
return "", err
}
- fromFormat := metadecoders.FormatFromContentString(from)
+ fromFormat := metadecoders.Default.FormatFromContentString(from)
if fromFormat == "" {
return "", errors.New("failed to detect format from content")
}
- meta, err := metadecoders.UnmarshalToMap([]byte(from), fromFormat)
+ meta, err := metadecoders.Default.UnmarshalToMap([]byte(from), fromFormat)
var result bytes.Buffer
if err := parser.InterfaceToConfig(meta, mark, &result); err != nil {
import (
"io/ioutil"
+ "strings"
"github.com/gohugoio/hugo/common/hugio"
+ "github.com/mitchellh/mapstructure"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/parser/metadecoders"
)
// Unmarshal unmarshals the data given, which can be either a string
-// or a Resource. Supported formats are JSON, TOML and YAML.
-func (ns *Namespace) Unmarshal(data interface{}) (interface{}, error) {
+// or a Resource. Supported formats are JSON, TOML, YAML, and CSV.
+// You can optional provide an Options object as the first argument.
+func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) {
+ if len(args) < 1 || len(args) > 2 {
+ return nil, errors.New("unmarshal takes 1 or 2 arguments")
+ }
+
+ var data interface{}
+ var decoder = metadecoders.Default
+
+ if len(args) == 1 {
+ data = args[0]
+ } else {
+ m, ok := args[0].(map[string]interface{})
+ if !ok {
+ return nil, errors.New("first argument must be a map")
+ }
+
+ var err error
+
+ data = args[1]
+ decoder, err = decodeDecoder(m)
+ if err != nil {
+ return nil, errors.WithMessage(err, "failed to decode options")
+ }
+
+ }
// All the relevant Resource types implements ReadSeekCloserResource,
// which should be the most effective way to get the content.
return nil, err
}
- return metadecoders.Unmarshal(b, f)
+ return decoder.Unmarshal(b, f)
})
}
key := helpers.MD5String(dataStr)
return ns.cache.GetOrCreate(key, func() (interface{}, error) {
- f := metadecoders.FormatFromContentString(dataStr)
+ f := decoder.FormatFromContentString(dataStr)
if f == "" {
return nil, errors.New("unknown format")
}
- return metadecoders.Unmarshal([]byte(dataStr), f)
+ return decoder.Unmarshal([]byte(dataStr), f)
})
}
+
+func decodeDecoder(m map[string]interface{}) (metadecoders.Decoder, error) {
+ opts := metadecoders.Default
+
+ if m == nil {
+ return opts, nil
+ }
+
+ // mapstructure does not support string to rune conversion, so do that manually.
+ // See https://github.com/mitchellh/mapstructure/issues/151
+ for k, v := range m {
+ if strings.EqualFold(k, "Comma") {
+ r, err := stringToRune(v)
+ if err != nil {
+ return opts, err
+ }
+ opts.Comma = r
+ delete(m, k)
+
+ } else if strings.EqualFold(k, "Comment") {
+ r, err := stringToRune(v)
+ if err != nil {
+ return opts, err
+ }
+ opts.Comment = r
+ delete(m, k)
+ }
+ }
+
+ err := mapstructure.WeakDecode(m, &opts)
+
+ return opts, err
+}
+
+func stringToRune(v interface{}) (rune, error) {
+ s, err := cast.ToStringE(v)
+ if err != nil {
+ return 0, err
+ }
+
+ if len(s) == 0 {
+ return 0, nil
+ }
+
+ var r rune
+
+ for i, rr := range s {
+ if i == 0 {
+ r = rr
+ } else {
+ return 0, errors.Errorf("invalid character: %q", v)
+ }
+ }
+
+ return r, nil
+}
}
for i, test := range []struct {
- data interface{}
- expect interface{}
+ data interface{}
+ options interface{}
+ expect interface{}
}{
- {`{ "slogan": "Hugo Rocks!" }`, func(m map[string]interface{}) {
+ {`{ "slogan": "Hugo Rocks!" }`, nil, func(m map[string]interface{}) {
assertSlogan(m)
}},
- {`slogan: "Hugo Rocks!"`, func(m map[string]interface{}) {
+ {`slogan: "Hugo Rocks!"`, nil, func(m map[string]interface{}) {
assertSlogan(m)
}},
- {`slogan = "Hugo Rocks!"`, func(m map[string]interface{}) {
+ {`slogan = "Hugo Rocks!"`, nil, func(m map[string]interface{}) {
assertSlogan(m)
}},
- {testContentResource{content: `slogan: "Hugo Rocks!"`, mime: media.YAMLType}, func(m map[string]interface{}) {
+ {testContentResource{content: `slogan: "Hugo Rocks!"`, mime: media.YAMLType}, nil, func(m map[string]interface{}) {
assertSlogan(m)
}},
- {testContentResource{content: `{ "slogan": "Hugo Rocks!" }`, mime: media.JSONType}, func(m map[string]interface{}) {
+ {testContentResource{content: `{ "slogan": "Hugo Rocks!" }`, mime: media.JSONType}, nil, func(m map[string]interface{}) {
assertSlogan(m)
}},
- {testContentResource{content: `slogan = "Hugo Rocks!"`, mime: media.TOMLType}, func(m map[string]interface{}) {
+ {testContentResource{content: `slogan = "Hugo Rocks!"`, mime: media.TOMLType}, nil, func(m map[string]interface{}) {
assertSlogan(m)
}},
+ {testContentResource{content: `1997,Ford,E350,"ac, abs, moon",3000.00
+1999,Chevy,"Venture ""Extended Edition""","",4900.00`, mime: media.CSVType}, nil, func(r [][]string) {
+ assert.Equal(2, len(r))
+ first := r[0]
+ assert.Equal(5, len(first))
+ assert.Equal("Ford", first[1])
+ }},
+ {testContentResource{content: `a;b;c`, mime: media.CSVType}, map[string]interface{}{"comma": ";"}, func(r [][]string) {
+ assert.Equal(r, [][]string{[]string{"a", "b", "c"}})
+
+ }},
+ {"a,b,c", nil, func(r [][]string) {
+ assert.Equal(r, [][]string{[]string{"a", "b", "c"}})
+
+ }},
+ {"a;b;c", map[string]interface{}{"comma": ";"}, func(r [][]string) {
+ assert.Equal(r, [][]string{[]string{"a", "b", "c"}})
+
+ }},
+ {testContentResource{content: `
+% This is a comment
+a;b;c`, mime: media.CSVType}, map[string]interface{}{"CommA": ";", "Comment": "%"}, func(r [][]string) {
+ assert.Equal(r, [][]string{[]string{"a", "b", "c"}})
+
+ }},
// errors
- {"thisisnotavaliddataformat", false},
- {testContentResource{content: `invalid&toml"`, mime: media.TOMLType}, false},
- {testContentResource{content: `unsupported: MIME"`, mime: media.CalendarType}, false},
- {"thisisnotavaliddataformat", false},
- {`{ notjson }`, false},
- {tstNoStringer{}, false},
+ {"thisisnotavaliddataformat", nil, false},
+ {testContentResource{content: `invalid&toml"`, mime: media.TOMLType}, nil, false},
+ {testContentResource{content: `unsupported: MIME"`, mime: media.CalendarType}, nil, false},
+ {"thisisnotavaliddataformat", nil, false},
+ {`{ notjson }`, nil, false},
+ {tstNoStringer{}, nil, false},
} {
errMsg := fmt.Sprintf("[%d]", i)
- result, err := ns.Unmarshal(test.data)
+ ns.cache.Clear()
+
+ var args []interface{}
+
+ if test.options != nil {
+ args = []interface{}{test.options, test.data}
+ } else {
+ args = []interface{}{test.data}
+ }
+
+ result, err := ns.Unmarshal(args...)
if b, ok := test.expect.(bool); ok && !b {
assert.Error(err, errMsg)
m, ok := result.(map[string]interface{})
assert.True(ok, errMsg)
fn(m)
+ } else if fn, ok := test.expect.(func(r [][]string)); ok {
+ assert.NoError(err, errMsg)
+ r, ok := result.([][]string)
+ assert.True(ok, errMsg)
+ fn(r)
} else {
assert.NoError(err, errMsg)
assert.Equal(test.expect, result, errMsg)
projects / brevno-suite / hugo / commitdiff