Feature: GetJson and GetJson in short codes or other layout files.
For more details please see: http://cyrillschumacher.com/2014/12/21/dynamic-pages-with-gohugo.io/
//Flags that are to be added to commands.
var BuildWatch, Draft, Future, UglyUrls, Verbose, Logging, VerboseLog, DisableRSS, DisableSitemap, PluralizeListTitles, NoTimes bool
-var Source, Destination, Theme, BaseUrl, CfgFile, LogFile, Editor string
+var Source, CacheDir, Destination, Theme, BaseUrl, CfgFile, LogFile, Editor string
//Execute adds all child commands to the root command HugoCmd and sets flags appropriately.
func Execute() {
HugoCmd.PersistentFlags().BoolVar(&DisableRSS, "disableRSS", false, "Do not build RSS files")
HugoCmd.PersistentFlags().BoolVar(&DisableSitemap, "disableSitemap", false, "Do not build Sitemap file")
HugoCmd.PersistentFlags().StringVarP(&Source, "source", "s", "", "filesystem path to read files relative from")
+ HugoCmd.PersistentFlags().StringVarP(&CacheDir, "cacheDir", "", "$TMPDIR/hugo_cache/", "filesystem path to cache directory")
HugoCmd.PersistentFlags().StringVarP(&Destination, "destination", "d", "", "filesystem path to write files to")
HugoCmd.PersistentFlags().StringVarP(&Theme, "theme", "t", "", "theme to use (located in /themes/THEMENAME/)")
HugoCmd.PersistentFlags().BoolVarP(&Verbose, "verbose", "v", false, "verbose output")
viper.Set("WorkingDir", dir)
}
+ if CacheDir != "" {
+ if helpers.FilePathSeparator != CacheDir[len(CacheDir)-1:] {
+ CacheDir = CacheDir + helpers.FilePathSeparator
+ }
+ viper.Set("CacheDir", CacheDir)
+ } else {
+ viper.Set("CacheDir", helpers.GetTempDir("hugo_cache", hugofs.SourceFs))
+ }
+
if VerboseLog || Logging || (viper.IsSet("LogFile") && viper.GetString("LogFile") != "") {
if viper.IsSet("LogFile") && viper.GetString("LogFile") != "" {
jww.SetLogFile(viper.GetString("LogFile"))
_, err = io.Copy(file, r)
return
}
+
+// GetTempDir returns the OS default temp directory with trailing slash
+// if subPath is not empty then it will be created recursively
+func GetTempDir(subPath string, fs afero.Fs) string {
+ dir := os.TempDir()
+ if FilePathSeparator != dir[len(dir)-1:] {
+ dir = dir + FilePathSeparator
+ }
+ if subPath != "" {
+ dir = dir + MakePath(subPath)
+
+ if exists, _ := Exists(dir, fs); exists {
+ return dir
+ }
+
+ err := fs.MkdirAll(dir, 0777) // rwx, rw, r
+ if err != nil {
+ panic(err)
+ }
+ if FilePathSeparator != dir[len(dir)-1:] {
+ dir = dir + FilePathSeparator
+ }
+ }
+ return dir
+}
reader.Seek(0, 0)
}
}
+
+func TestGetTempDir(t *testing.T) {
+ dir := os.TempDir()
+ if FilePathSeparator != dir[len(dir)-1:] {
+ dir = dir + FilePathSeparator
+ }
+ testDir := "hugoTestFolder" + FilePathSeparator
+ tests := []struct {
+ input string
+ expected string
+ }{
+ {"", dir},
+ {testDir + " Foo bar ", dir + testDir + "--Foo-bar" + FilePathSeparator},
+ {testDir + "Foo.Bar/foo_Bar-Foo", dir + testDir + "Foo.Bar/foo_Bar-Foo" + FilePathSeparator},
+ {testDir + "fOO,bar:foo%bAR", dir + testDir + "fOObarfoobAR" + FilePathSeparator},
+ {testDir + "FOo/BaR.html", dir + testDir + "FOo/BaR.html" + FilePathSeparator},
+ {testDir + "трям/трям", dir + testDir + "трям/трям" + FilePathSeparator},
+ {testDir + "은행", dir + testDir + "은행" + FilePathSeparator},
+ {testDir + "Банковский кассир", dir + testDir + "Банковский-кассир" + FilePathSeparator},
+ }
+
+ for _, test := range tests {
+ output := GetTempDir(test.input, new(afero.MemMapFs))
+ if output != test.expected {
+ t.Errorf("Expected %#v, got %#v\n", test.expected, output)
+ }
+ }
+}
--- /dev/null
+// Copyright © 2013-14 Steve Francia <spf@spf13.com>.
+//
+// Licensed under the Simple Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://opensource.org/licenses/Simple-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tpl
+
+import (
+ "bytes"
+ "encoding/csv"
+ "encoding/json"
+ "errors"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+ "strings"
+ "sync"
+
+ "github.com/spf13/afero"
+ "github.com/spf13/hugo/helpers"
+ "github.com/spf13/hugo/hugofs"
+ jww "github.com/spf13/jwalterweatherman"
+ "github.com/spf13/viper"
+)
+
+var remoteUrlLock = &remoteLock{m: make(map[string]*sync.Mutex)}
+
+type remoteLock struct {
+ sync.RWMutex
+ m map[string]*sync.Mutex
+}
+
+// resLock locks an URL during download
+func (l *remoteLock) UrlLock(url string) {
+ l.Lock()
+ if _, ok := l.m[url]; !ok {
+ l.m[url] = &sync.Mutex{}
+ }
+ l.Unlock() // call this Unlock before the next lock will be called. NFI why but defer doesn't work.
+ l.m[url].Lock()
+}
+
+// resUnlock unlocks an URL when the download has been finished. Use only in defer calls.
+func (l *remoteLock) UrlUnlock(url string) {
+ l.RLock()
+ defer l.RUnlock()
+ if um, ok := l.m[url]; ok {
+ um.Unlock()
+ }
+}
+
+// getFileID returns the cache ID for a string
+func getCacheFileID(id string) string {
+ return viper.GetString("CacheDir") + url.QueryEscape(id)
+}
+
+// resGetCache returns the content for an ID from the file cache or an error
+// if the file is not found returns nil,nil
+func resGetCache(id string, fs afero.Fs) ([]byte, error) {
+ fID := getCacheFileID(id)
+ isExists, err := helpers.Exists(fID, fs)
+ if err != nil {
+ return nil, err
+ }
+ if !isExists {
+ return nil, nil
+ }
+
+ f, err := fs.Open(fID)
+ if err != nil {
+ return nil, err
+ }
+
+ return ioutil.ReadAll(f)
+}
+
+// resWriteCache writes bytes to an ID into the file cache
+func resWriteCache(id string, c []byte, fs afero.Fs) error {
+ fID := getCacheFileID(id)
+ f, err := fs.Create(fID)
+ if err != nil {
+ return err
+ }
+ n, err := f.Write(c)
+ if n == 0 {
+ return errors.New("No bytes written to file: " + fID)
+ }
+ return err
+}
+
+// resGetRemote loads the content of a remote file. This method is thread safe.
+func resGetRemote(url string, fs afero.Fs, hc *http.Client) ([]byte, error) {
+
+ c, err := resGetCache(url, fs)
+ if c != nil && err == nil {
+ return c, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ // avoid race condition with locks, block other goroutines if the current url is processing
+ remoteUrlLock.UrlLock(url)
+ defer func() { remoteUrlLock.UrlUnlock(url) }()
+
+ // avoid multiple locks due to calling resGetCache twice
+ c, err = resGetCache(url, fs)
+ if c != nil && err == nil {
+ return c, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ jww.INFO.Printf("Downloading: %s ...", url)
+ res, err := hc.Get(url)
+ if err != nil {
+ return nil, err
+ }
+ c, err = ioutil.ReadAll(res.Body)
+ res.Body.Close()
+ if err != nil {
+ return nil, err
+ }
+ err = resWriteCache(url, c, fs)
+ if err != nil {
+ return nil, err
+ }
+ jww.INFO.Printf("... and cached to: %s", getCacheFileID(url))
+ return c, nil
+}
+
+// resGetLocal loads the content of a local file
+func resGetLocal(url string, fs afero.Fs) ([]byte, error) {
+ p := ""
+ if viper.GetString("WorkingDir") != "" {
+ p = viper.GetString("WorkingDir")
+ if helpers.FilePathSeparator != p[len(p)-1:] {
+ p = p + helpers.FilePathSeparator
+ }
+ }
+ jFile := p + url
+ if e, err := helpers.Exists(jFile, fs); !e {
+ return nil, err
+ }
+
+ f, err := fs.Open(jFile)
+ if err != nil {
+ return nil, err
+ }
+ return ioutil.ReadAll(f)
+}
+
+// resGetResource loads the content of a local or remote file
+func resGetResource(url string) ([]byte, error) {
+ if url == "" {
+ return nil, nil
+ }
+ if strings.Contains(url, "://") {
+ return resGetRemote(url, hugofs.SourceFs, http.DefaultClient)
+ }
+ return resGetLocal(url, hugofs.SourceFs)
+}
+
+// GetJson expects the url to a resource which can either be a local or a remote one.
+// GetJson returns nil or parsed JSON to use in a short code.
+func GetJson(url string) interface{} {
+ c, err := resGetResource(url)
+ if err != nil {
+ jww.ERROR.Printf("Failed to get json resource %s with error message %s", url, err)
+ return nil
+ }
+
+ var v interface{}
+ err = json.Unmarshal(c, &v)
+ if err != nil {
+ jww.ERROR.Printf("Cannot read json from resource %s with error message %s", url, err)
+ return nil
+ }
+ return v
+}
+
+// parseCsv parses bytes of csv data into a slice slice string or an error
+func parseCsv(c []byte, sep string) ([][]string, error) {
+ if len(sep) != 1 {
+ return nil, errors.New("Incorrect length of csv separator: " + sep)
+ }
+ b := bytes.NewReader(c)
+ r := csv.NewReader(b)
+ rSep := []rune(sep)
+ r.Comma = rSep[0]
+ r.FieldsPerRecord = 0
+ return r.ReadAll()
+}
+
+// GetCsv expects the url to a resource which can either be a local or a remote one and the type
+// of the data separator which can be comma, semi-colon, pipe, but only one character.
+// GetCsv returns nil or a slice slice to use in a short code.
+func GetCsv(url string, sep string) [][]string {
+
+ c, err := resGetResource(url)
+ if err != nil {
+ jww.ERROR.Printf("Failed to get csv resource %s with error message %s", url, err)
+ return nil
+ }
+ d, err := parseCsv(c, sep)
+ if err != nil {
+ jww.ERROR.Printf("Failed to read csv resource %s with error message %s", url, err)
+ return nil
+ }
+ return d
+}
--- /dev/null
+// Copyright © 2013-14 Steve Francia <spf@spf13.com>.
+//
+// Licensed under the Simple Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://opensource.org/licenses/Simple-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tpl
+
+import (
+ "bytes"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strings"
+ "testing"
+
+ "github.com/spf13/afero"
+ "github.com/spf13/hugo/helpers"
+)
+
+func TestScpCache(t *testing.T) {
+
+ tests := []struct {
+ path string
+ content []byte
+ }{
+ {"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`)},
+ {"fOO,bar:foo%bAR", []byte(`T€st Content 123 fOO,bar:foo%bAR`)},
+ {"FOo/BaR.html", []byte(`FOo/BaR.html T€st Content 123`)},
+ {"трям/трям", []byte(`T€st трям/трям Content 123`)},
+ {"은행", []byte(`T€st C은행ontent 123`)},
+ {"Банковский кассир", []byte(`Банковский кассир T€st Content 123`)},
+ }
+
+ fs := new(afero.MemMapFs)
+
+ for _, test := range tests {
+ c, err := resGetCache(test.path, fs)
+ if err != nil {
+ t.Errorf("Error getting cache: %s", err)
+ }
+ if c != nil {
+ t.Errorf("There is content where there should not be anything: %s", string(c))
+ }
+
+ err = resWriteCache(test.path, test.content, fs)
+ if err != nil {
+ t.Errorf("Error writing cache: %s", err)
+ }
+
+ c, err = resGetCache(test.path, fs)
+ if err != nil {
+ t.Errorf("Error getting cache after writing: %s", err)
+ }
+ if bytes.Compare(c, test.content) != 0 {
+ t.Errorf("\nExpected: %s\nActual: %s\n", string(test.content), string(c))
+ }
+ }
+}
+
+func TestScpGetLocal(t *testing.T) {
+ fs := new(afero.MemMapFs)
+ ps := helpers.FilePathSeparator
+ tests := []struct {
+ path string
+ content []byte
+ }{
+ {"testpath" + ps + "test.txt", []byte(`T€st Content 123 fOO,bar:foo%bAR`)},
+ {"FOo" + ps + "BaR.html", []byte(`FOo/BaR.html T€st Content 123`)},
+ {"трям" + ps + "трям", []byte(`T€st трям/трям Content 123`)},
+ {"은행", []byte(`T€st C은행ontent 123`)},
+ {"Банковский кассир", []byte(`Банковский кассир T€st Content 123`)},
+ }
+
+ for _, test := range tests {
+ r := bytes.NewReader(test.content)
+ err := helpers.WriteToDisk(test.path, r, fs)
+ if err != nil {
+ t.Error(err)
+ }
+
+ c, err := resGetLocal(test.path, fs)
+ if err != nil {
+ t.Errorf("Error getting resource content: %s", err)
+ }
+ if bytes.Compare(c, test.content) != 0 {
+ t.Errorf("\nExpected: %s\nActual: %s\n", string(test.content), string(c))
+ }
+ }
+
+}
+
+func getTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*httptest.Server, *http.Client) {
+ testServer := httptest.NewServer(http.HandlerFunc(handler))
+ client := &http.Client{
+ Transport: &http.Transport{Proxy: func(*http.Request) (*url.URL, error) { return url.Parse(testServer.URL) }},
+ }
+ return testServer, client
+}
+
+func TestScpGetRemote(t *testing.T) {
+ fs := new(afero.MemMapFs)
+
+ tests := []struct {
+ path string
+ content []byte
+ }{
+ {"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`)},
+ {"http://Doppel.Gänger/foo_Bar-Foo", []byte(`T€st Cont€nt 123`)},
+ {"http://Doppel.Gänger/Fizz_Bazz-Foo", []byte(`T€st Банковский кассир Cont€nt 123`)},
+ }
+
+ for _, test := range tests {
+
+ srv, cl := getTestServer(func(w http.ResponseWriter, r *http.Request) {
+ w.Write(test.content)
+ })
+ defer func() { srv.Close() }()
+
+ c, err := resGetRemote(test.path, fs, cl)
+ if err != nil {
+ t.Errorf("Error getting resource content: %s", err)
+ }
+ if bytes.Compare(c, test.content) != 0 {
+ t.Errorf("\nNet Expected: %s\nNet Actual: %s\n", string(test.content), string(c))
+ }
+ cc, cErr := resGetCache(test.path, fs)
+ if cErr != nil {
+ t.Error(cErr)
+ }
+ if bytes.Compare(cc, test.content) != 0 {
+ t.Errorf("\nCache Expected: %s\nCache Actual: %s\n", string(test.content), string(c))
+ }
+ }
+}
+
+func TestParseCsv(t *testing.T) {
+
+ tests := []struct {
+ csv []byte
+ sep string
+ exp string
+ err bool
+ }{
+ {[]byte("a,b,c\nd,e,f\n"), "", "", true},
+ {[]byte("a,b,c\nd,e,f\n"), "~/", "", true},
+ {[]byte("a,b,c\nd,e,f"), "|", "a,b,cd,e,f", false},
+ {[]byte("q,w,e\nd,e,f"), ",", "qwedef", false},
+ {[]byte("a|b|c\nd|e|f|g"), "|", "abcdefg", true},
+ {[]byte("z|y|c\nd|e|f"), "|", "zycdef", false},
+ }
+ for _, test := range tests {
+ csv, err := parseCsv(test.csv, test.sep)
+ if test.err && err == nil {
+ t.Error("Expecting an error")
+ }
+ if test.err {
+ continue
+ }
+ if !test.err && err != nil {
+ t.Error(err)
+ }
+
+ act := ""
+ for _, v := range csv {
+ act = act + strings.Join(v, "")
+ }
+
+ if act != test.exp {
+ t.Errorf("\nExpected: %s\nActual: %s\n%#v\n", test.exp, act, csv)
+ }
+
+ }
+}
projects / brevno-suite / hugo / commitdiff