Introduce source.Filesystem

This provides an abstraction over how files are processed by Hugo.  This
allows for alternatives like CMS systems or Dropbox, etc.

diff --git a/hugolib/content_directory_test.go b/hugolib/content_directory_test.go
deleted file mode 100644 (file)
index ab072e5..0000000
--- a/hugolib/content_directory_test.go
+++ /dev/null
@@ -1,25 +0,0 @@
-package hugolib
-
-import (
-       "testing"
-)
-
-func TestIgnoreDotFiles(t *testing.T) {
-       tests := []struct {
-               path   string
-               ignore bool
-       }{
-               {"barfoo.md", false},
-               {"foobar/barfoo.md", false},
-               {"foobar/.barfoo.md", true},
-               {".barfoo.md", true},
-               {".md", true},
-               {"", true},
-       }
-
-       for _, test := range tests {
-               if ignored := ignoreDotFile(test.path); test.ignore != ignored {
-                       t.Errorf("File not ignored.  Expected: %t, got: %t", test.ignore, ignored)
-               }
-       }
-}

diff --git a/hugolib/planner.go b/hugolib/planner.go
index 31d9e61667635df4fbd319a2d3b617ffa8ac809e..9abd92ff8df5db06f1272cb8b31a2b0285ee0288 100644 (file)
--- a/hugolib/planner.go
+++ b/hugolib/planner.go
@@ -6,19 +6,19 @@ import (
 )
 
 func (s *Site) ShowPlan(out io.Writer) (err error) {
-       if len(s.Files) <= 0 {
+       if s.Source == nil || len(s.Source.Files()) <= 0 {
                fmt.Fprintf(out, "No source files provided.\n")
        }
 
-       for _, file := range s.Files {
-               fmt.Fprintf(out, "%s\n", file)
+       for _, p := range s.Pages {
+               fmt.Fprintf(out, "%s\n", p.FileName)
                fmt.Fprintf(out, " canonical => ")
                if s.Target == nil {
                        fmt.Fprintf(out, "%s\n", "!no target specified!")
                        continue
                }
 
-               trns, err := s.Target.Translate(file)
+               trns, err := s.Target.Translate(p.OutFile)
                if err != nil {
                        return err
                }

diff --git a/hugolib/site.go b/hugolib/site.go
index 0eac054b0aeb337699dedd5d3730e840fd3360c5..9b466590162d9c64d917a98977549cc1ee4ca281 100644 (file)
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -17,6 +17,7 @@ import (
        "bitbucket.org/pkg/inflect"
        "bytes"
        "fmt"
+       "github.com/spf13/hugo/source"
        "github.com/spf13/hugo/target"
        helpers "github.com/spf13/hugo/template"
        "github.com/spf13/hugo/template/bundle"
@@ -69,7 +70,7 @@ type Site struct {
        Pages      Pages
        Tmpl       bundle.Template
        Indexes    IndexList
-       Files      []string
+       Source     source.Input
        Sections   Index
        Info       SiteInfo
        Shortcodes map[string]ShortcodeFunc
@@ -186,26 +187,11 @@ func (s *Site) initialize() {
 
        staticDir := s.Config.GetAbsPath(s.Config.StaticDir + "/")
 
-       walker := func(path string, fi os.FileInfo, err error) error {
-               if err != nil {
-                       return nil
-               }
-
-               if fi.IsDir() {
-                       if path == staticDir {
-                               return filepath.SkipDir
-                       }
-                       return nil
-               } else {
-                       if ignoreDotFile(path) {
-                               return nil
-                       }
-                       s.Files = append(s.Files, path)
-                       return nil
-               }
+       s.Source = &source.Filesystem{
+               AvoidPaths: []string{staticDir},
+               Base:       s.absContentDir(),
        }
 
-       filepath.Walk(s.absContentDir(), walker)
        s.Info = SiteInfo{
                BaseUrl: template.URL(s.Config.BaseUrl),
                Title:   s.Config.Title,
@@ -228,10 +214,6 @@ func exists(path string) (bool, error) {
        return false, err
 }
 
-func ignoreDotFile(path string) bool {
-       return filepath.Base(path)[0] == '.'
-}
-
 func (s *Site) absLayoutDir() string {
        return s.Config.GetAbsPath(s.Config.LayoutDir)
 }
@@ -275,12 +257,8 @@ func (s *Site) AbsUrlify() {
 }
 
 func (s *Site) CreatePages() (err error) {
-       for _, fileName := range s.Files {
-               f, err := os.Open(fileName)     
-               if err != nil {
-                       return err
-               }
-               page, err := ReadFrom(f, fileName)
+       for _, file := range s.Source.Files() {
+               page, err := ReadFrom(file.Contents, file.Name)
                if err != nil {
                        return err
                }

diff --git a/hugolib/site_show_plan_test.go b/hugolib/site_show_plan_test.go
index 97ab46b7e061d47fc9eeec6da800996d2d87a369..8332a6f94e73302a73bf98ca92298e90c499810f 100644 (file)
--- a/hugolib/site_show_plan_test.go
+++ b/hugolib/site_show_plan_test.go
@@ -2,57 +2,80 @@ package hugolib
 
 import (
        "bytes"
-       "testing"
+       "github.com/spf13/hugo/source"
        "github.com/spf13/hugo/target"
+       "testing"
 )
 
-func checkShowPlanExpected(t *testing.T, expected, got string) {
-       if got != expected {
-               t.Errorf("ShowPlan expected:\n%q\ngot\n%q", expected, got)
+var fakeSource = []struct {
+       name    string
+       content []byte
+}{
+       {"foo/bar/file.md", []byte(SIMPLE_PAGE)},
+}
+
+type inMemorySource struct {
+}
+
+func (i inMemorySource) Files() (files []*source.File) {
+       files = make([]*source.File, len(fakeSource))
+       for i, fake := range fakeSource {
+               files[i] = &source.File{
+                       Name:     fake.name,
+                       Contents: bytes.NewReader(fake.content),
+               }
        }
+       return
 }
 
-func TestDegenerateNoFiles(t *testing.T) {
-       s := new(Site)
+func checkShowPlanExpected(t *testing.T, s *Site, expected string) {
        out := new(bytes.Buffer)
        if err := s.ShowPlan(out); err != nil {
-               t.Errorf("ShowPlan unexpectedly returned an error: %s", err)
+               t.Fatalf("ShowPlan unexpectedly returned an error: %s", err)
        }
-       expected := "No source files provided.\n"
        got := out.String()
-       checkShowPlanExpected(t, expected, got)
+       if got != expected {
+               t.Errorf("ShowPlan expected:\n%q\ngot\n%q", expected, got)
+       }
 }
 
-func TestDegenerateNoTarget(t *testing.T) {
-       s := new(Site)
-       s.Files = append(s.Files, "foo/bar/file.md")
-       out := new(bytes.Buffer)
-       if err := s.ShowPlan(out); err != nil {
-               t.Errorf("ShowPlan unexpectedly returned an error: %s", err)
-       }
+func TestDegenerateNoFiles(t *testing.T) {
+       checkShowPlanExpected(t, new(Site), "No source files provided.\n")
+}
 
+func TestDegenerateNoTarget(t *testing.T) {
+       s := &Site{Source: new(inMemorySource)}
+       must(s.CreatePages())
        expected := "foo/bar/file.md\n canonical => !no target specified!\n"
-       checkShowPlanExpected(t, expected, out.String())
+       checkShowPlanExpected(t, s, expected)
 }
 
 func TestFileTarget(t *testing.T) {
-       s := &Site{Target: new(target.Filesystem)}
-       s.Files = append(s.Files, "foo/bar/file.md")
-       out := new(bytes.Buffer)
-       s.ShowPlan(out)
-
-       expected := "foo/bar/file.md\n canonical => foo/bar/file/index.html\n"
-       checkShowPlanExpected(t, expected, out.String())
+       s := &Site{
+               Source: new(inMemorySource),
+               Target: new(target.Filesystem),
+       }
+       must(s.CreatePages())
+       checkShowPlanExpected(t, s, "foo/bar/file.md\n canonical => foo/bar/file/index.html\n")
 }
 
 func TestFileTargetUgly(t *testing.T) {
-       s := &Site{Target: &target.Filesystem{UglyUrls: true}}
-       s.Files = append(s.Files, "foo/bar/file.md")
-       out := new(bytes.Buffer)
-       s.ShowPlan(out)
-
+       s := &Site{
+               Target: &target.Filesystem{UglyUrls: true},
+               Source: new(inMemorySource),
+       }
+       s.CreatePages()
        expected := "foo/bar/file.md\n canonical => foo/bar/file.html\n"
-       checkShowPlanExpected(t, expected, out.String())
+       checkShowPlanExpected(t, s, expected)
 }
 
+func TestFileTargetPublishDir(t *testing.T) {
+       s := &Site{
+               Target: &target.Filesystem{PublishDir: "../public"},
+               Source: new(inMemorySource),
+       }
 
+       must(s.CreatePages())
+       expected := "foo/bar/file.md\n canonical => ../public/foo/bar/file/index.html\n"
+       checkShowPlanExpected(t, s, expected)
+}

diff --git a/hugolib/site_url_test.go b/hugolib/site_url_test.go
index f182503c0cb8c3d649c801c23751913e2cc87f77..3353e4a84abee22a421c7e73be034c00be4d39ab 100644 (file)
--- a/hugolib/site_url_test.go
+++ b/hugolib/site_url_test.go
@@ -49,8 +49,8 @@ func TestPageCount(t *testing.T) {
        s := &Site{Target: target}
        s.prepTemplates()
        must(s.addTemplate("indexes/blue.html", INDEX_TEMPLATE))
-       s.Files = append(s.Files, "blue/doc1.md")
-       s.Files = append(s.Files, "blue/doc2.md")
+       //s.Files = append(s.Files, "blue/doc1.md")
+       //s.Files = append(s.Files, "blue/doc2.md")
        s.Pages = append(s.Pages, mustReturn(ReadFrom(strings.NewReader(SLUG_DOC_1), filepath.FromSlash("content/blue/doc1.md"))))
        s.Pages = append(s.Pages, mustReturn(ReadFrom(strings.NewReader(SLUG_DOC_2), filepath.FromSlash("content/blue/doc2.md"))))
 

diff --git a/source/content_directory_test.go b/source/content_directory_test.go
new file mode 100644 (file)
index 0000000..7f06160
--- /dev/null
+++ b/source/content_directory_test.go
@@ -0,0 +1,25 @@
+package source
+
+import (
+       "testing"
+)
+
+func TestIgnoreDotFiles(t *testing.T) {
+       tests := []struct {
+               path   string
+               ignore bool
+       }{
+               {"barfoo.md", false},
+               {"foobar/barfoo.md", false},
+               {"foobar/.barfoo.md", true},
+               {".barfoo.md", true},
+               {".md", true},
+               {"", true},
+       }
+
+       for _, test := range tests {
+               if ignored := ignoreDotFile(test.path); test.ignore != ignored {
+                       t.Errorf("File not ignored.  Expected: %t, got: %t", test.ignore, ignored)
+               }
+       }
+}

diff --git a/source/filesystem.go b/source/filesystem.go
new file mode 100644 (file)
index 0000000..5434431
--- /dev/null
+++ b/source/filesystem.go
@@ -0,0 +1,72 @@
+package source
+
+import (
+       "io"
+       "os"
+       "path/filepath"
+)
+
+type Input interface {
+       Files() []*File
+}
+
+type File struct {
+       Name     string
+       Contents io.Reader
+}
+
+type Filesystem struct {
+       files      []*File
+       Base       string
+       AvoidPaths []string
+}
+
+func (f *Filesystem) Files() []*File {
+       f.captureFiles()
+       return f.files
+}
+
+func (f *Filesystem) add(name string, reader io.Reader) {
+       f.files = append(f.files, &File{Name: name, Contents: reader})
+}
+
+func (f *Filesystem) captureFiles() {
+
+       walker := func(path string, fi os.FileInfo, err error) error {
+               if err != nil {
+                       return nil
+               }
+
+               if fi.IsDir() {
+                       if f.avoid(path) {
+                               return filepath.SkipDir
+                       }
+                       return nil
+               } else {
+                       if ignoreDotFile(path) {
+                               return nil
+                       }
+                       file, err := os.Open(path)
+                       if err != nil {
+                               return err
+                       }
+                       f.add(path, file)
+                       return nil
+               }
+       }
+
+       filepath.Walk(f.Base, walker)
+}
+
+func (f *Filesystem) avoid(path string) bool {
+       for _, avoid := range f.AvoidPaths {
+               if avoid == path {
+                       return true
+               }
+       }
+       return false
+}
+
+func ignoreDotFile(path string) bool {
+       return filepath.Base(path)[0] == '.'
+}

diff --git a/source/filesystem_test.go b/source/filesystem_test.go
new file mode 100644 (file)
index 0000000..2aac9b0
--- /dev/null
+++ b/source/filesystem_test.go
@@ -0,0 +1,32 @@
+package source
+
+import (
+       "bytes"
+       "testing"
+)
+
+func TestEmptySourceFilesystem(t *testing.T) {
+       src := new(Filesystem)
+       if len(src.Files()) != 0 {
+               t.Errorf("new filesystem should contain 0 files.")
+       }
+}
+
+func TestAddFile(t *testing.T) {
+       src := new(Filesystem)
+       src.add("foobar", bytes.NewReader([]byte("aaa")))
+       if len(src.Files()) != 1 {
+               t.Errorf("Files() should return 1 file")
+       }
+
+       f := src.Files()[0]
+       if f.Name != "foobar" {
+               t.Errorf("File name should be 'foobar', got: %s", f.Name)
+       }
+
+       b := new(bytes.Buffer)
+       b.ReadFrom(f.Contents)
+       if b.String() != "aaa" {
+               t.Errorf("File contents should be 'aaa', got: %s", b.String())
+       }
+}

diff --git a/target/file.go b/target/file.go
index 2df708fcf86fb495ca45add1195533bf0126d4aa..29c019f4ebe334e9648f37638ee2b370cd2592da 100644 (file)
--- a/target/file.go
+++ b/target/file.go
@@ -35,15 +35,16 @@ func (fs *Filesystem) Publish(path string, r io.Reader) (err error) {
        }
 
        path, _ = filepath.Split(translated)
-       dest := filepath.Join(fs.PublishDir, path)
-       ospath := filepath.FromSlash(dest)
+       ospath := filepath.FromSlash(path)
 
-       err = os.MkdirAll(ospath, 0764) // rwx, rw, r
-       if err != nil {
-               return
+       if ospath != "" {
+               err = os.MkdirAll(ospath, 0764) // rwx, rw, r
+               if err != nil {
+                       return
+               }
        }
 
-       file, err := os.Create(filepath.Join(fs.PublishDir, translated))
+       file, err := os.Create(translated)
        if err != nil {
                return
        }
@@ -61,6 +62,9 @@ func (fs *Filesystem) Translate(src string) (dest string, err error) {
        dir, file := path.Split(src)
        ext := fs.extension(path.Ext(file))
        name := filename(file)
+       if fs.PublishDir != "" {
+               dir = path.Join(fs.PublishDir, dir)
+       }
 
        if fs.UglyUrls {
                return path.Join(dir, fmt.Sprintf("%s%s", name, ext)), nil