DataDir: Minor polish and add missing tests

Also, now logs an ERROR on duplicate keys, instead of returning an error and make sure sub-folders take presedence in data dir.

diff --git a/hugolib/site.go b/hugolib/site.go
index d72b29795bdb4797636136a07882a2b66411b999..a91eefcbc20231f84df6051b9b0dbb7639d74c09 100644 (file)
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -114,7 +114,7 @@ type SiteInfo struct {
        BuildDrafts         bool
        canonifyUrls        bool
        paginationPageCount uint64
-       Data            *map[string]interface{}
+       Data                *map[string]interface{}
 }
 
 // SiteSocial is a place to put social details on a site level. These are the
@@ -269,12 +269,12 @@ func (s *Site) addTemplate(name, data string) error {
 
 func (s *Site) loadData(fs source.Input) (err error) {
        s.Data = make(map[string]interface{})
+       var current map[string]interface{}
 
        for _, r := range fs.Files() {
                // Crawl in data tree to insert data
-               var current map[string]interface{}
                current = s.Data
-               for _, key := range strings.Split(r.Dir(), string(os.PathSeparator)) {
+               for _, key := range strings.Split(r.Dir(), helpers.FilePathSeparator) {
                        if key != "" {
                                if _, ok := current[key]; !ok {
                                        current[key] = make(map[string]interface{})
@@ -283,8 +283,7 @@ func (s *Site) loadData(fs source.Input) (err error) {
                        }
                }
 
-               // Read data file
-               data, err := readFile(r)
+               data, err := readData(r)
                if err != nil {
                        return err
                }
@@ -295,10 +294,10 @@ func (s *Site) loadData(fs source.Input) (err error) {
 
                        for key, value := range current[r.BaseFileName()].(map[string]interface{}) {
                                if _, override := data[key]; override {
-                                       return errors.New("Data in " + r.Path() + " is overrided in subfolder.")
-                               } else {
-                                       data[key] = value
+                                       // filepath.Walk walks the files in lexical order, '/' comes before '.'
+                                       jww.ERROR.Printf("Data for key '%s' in path '%s' is overridden in subfolder", key, r.Path())
                                }
+                               data[key] = value
                        }
                }
 
@@ -309,7 +308,7 @@ func (s *Site) loadData(fs source.Input) (err error) {
        return
 }
 
-func readFile(f *source.File) (interface{}, error) {
+func readData(f *source.File) (interface{}, error) {
        switch f.Extension() {
        case "yaml", "yml":
                return parser.HandleYamlMetaData(f.Bytes())
@@ -318,7 +317,7 @@ func readFile(f *source.File) (interface{}, error) {
        case "toml":
                return parser.HandleTomlMetaData(f.Bytes())
        default:
-               return nil, errors.New("Not supported for data: " + f.Extension())
+               return nil, fmt.Errorf("Data not supported for extension '%s'", f.Extension())
        }
 }
 
@@ -328,11 +327,13 @@ func (s *Site) Process() (err error) {
        }
        s.prepTemplates()
        s.Tmpl.PrintErrors()
+       s.timerStep("initialize & template prep")
+
        if err = s.loadData(&source.Filesystem{Base: s.absDataDir()}); err != nil {
                return
        }
        s.timerStep("load data")
-       s.timerStep("initialize & template prep")
+
        if err = s.CreatePages(); err != nil {
                return
        }

diff --git a/hugolib/site_test.go b/hugolib/site_test.go
index 145b4755858a3656d4e81d30bf3bcbc0ccb1c8a3..04af0e61e3c90d240be066bdd56cd25e9ffc738a 100644 (file)
--- a/hugolib/site_test.go
+++ b/hugolib/site_test.go
@@ -3,9 +3,9 @@ package hugolib
 import (
        "bytes"
        "fmt"
+       "github.com/spf13/hugo/parser"
        "html/template"
        "io"
-       "os"
        "path/filepath"
        "strings"
        "testing"
@@ -17,6 +17,7 @@ import (
        "github.com/spf13/hugo/target"
        "github.com/spf13/hugo/tpl"
        "github.com/spf13/viper"
+       "reflect"
 )
 
 const (
@@ -747,17 +748,68 @@ func TestWeightedTaxonomies(t *testing.T) {
        }
 }
 
-func TestDataDir(t *testing.T) {
+func TestDataDirJson(t *testing.T) {
        sources := []source.ByteSource{
-               {filepath.FromSlash("test" + string(os.PathSeparator) + "foo.yaml"), []byte("bar: foofoo")},
-               {filepath.FromSlash("test.yaml"), []byte("hello:\n- world: foo")},
+               {filepath.FromSlash("test/foo.json"), []byte(`{ "bar": "foofoo"  }`)},
+               {filepath.FromSlash("test.json"), []byte(`{ "hello": [ { "world": "foo" } ] }`)},
        }
 
+       expected, err := parser.HandleJsonMetaData([]byte(`{ "test": { "hello": [{ "world": "foo"  }] , "foo": { "bar":"foofoo" } } }`))
+
+       if err != nil {
+               t.Fatalf("Error %s", err)
+       }
+
+       doTestDataDir(t, expected, sources)
+}
+
+func TestDataDirToml(t *testing.T) {
+       sources := []source.ByteSource{
+               {filepath.FromSlash("test/kung.toml"), []byte("[foo]\nbar = 1")},
+       }
+
+       expected, err := parser.HandleTomlMetaData([]byte("[test]\n[test.kung]\n[test.kung.foo]\nbar = 1"))
+
+       if err != nil {
+               t.Fatalf("Error %s", err)
+       }
+
+       doTestDataDir(t, expected, sources)
+}
+
+func TestDataDirYamlWithOverridenValue(t *testing.T) {
+       sources := []source.ByteSource{
+               // filepath.Walk walks the files in lexical order, '/' comes before '.'. Simulate this:
+               {filepath.FromSlash("a.yaml"), []byte("a: 1")},
+               {filepath.FromSlash("test/v1.yaml"), []byte("v1-2: 2")},
+               {filepath.FromSlash("test/v2.yaml"), []byte("v2:\n- 2\n- 3")},
+               {filepath.FromSlash("test.yaml"), []byte("v1: 1")},
+       }
+
+       expected := map[string]interface{}{"a": map[string]interface{}{"a": 1},
+               "test": map[string]interface{}{"v1": map[string]interface{}{"v1-2": 2}, "v2": map[string]interface{}{"v2": []interface{}{2, 3}}}}
+
+       doTestDataDir(t, expected, sources)
+}
+
+func TestDataDirUnknownFormat(t *testing.T) {
+       sources := []source.ByteSource{
+               {filepath.FromSlash("test.roml"), []byte("boo")},
+       }
        s := &Site{}
-       s.loadData(&source.InMemorySource{ByteSource: sources})
+       err := s.loadData(&source.InMemorySource{ByteSource: sources})
+       if err == nil {
+               t.Fatalf("Should return an error")
+       }
+}
 
-       expected := "map[test:map[hello:[map[world:foo]] foo:map[bar:foofoo]]]"
-       if fmt.Sprint(s.Data) != expected {
-               t.Errorf("Expected structure '%s', got '%s'", expected, s.Data)
+func doTestDataDir(t *testing.T, expected interface{}, sources []source.ByteSource) {
+       s := &Site{}
+       err := s.loadData(&source.InMemorySource{ByteSource: sources})
+       if err != nil {
+               t.Fatalf("Error loading data: %s", err)
+       }
+       if !reflect.DeepEqual(expected, s.Data) {
+               t.Errorf("Expected structure\n%#v got\n%#v", expected, s.Data)
        }
 }