".",
"mem"
]
- revision = "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c"
- version = "v1.0.2"
+ revision = "63644898a8da0bc22138abf860edaf5277b6102e"
+ version = "v1.1.0"
[[projects]]
name = "github.com/spf13/cast"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
- inputs-digest = "13ab39f8bfafadc12c05726e565ee3f3d94bf7d6c0e8adf04056de0691bf2dd6"
+ inputs-digest = "edb250b53926de21df1740c379c76351b7e9b110c96a77078a10ba69bf31a2d4"
solver-name = "gps-cdcl"
solver-version = 1
[[constraint]]
name = "github.com/spf13/afero"
- version = "^1.0.1"
+ version = "^1.1.0"
[[constraint]]
name = "github.com/spf13/cast"
c.Logger.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", path, err)
return nil
}
- linkfi, err := helpers.LstatIfOs(c.Fs.Source, link)
+ linkfi, err := helpers.LstatIfPossible(c.Fs.Source, link)
if err != nil {
c.Logger.ERROR.Printf("Cannot stat %q: %s", link, err)
return nil
// SymbolicWalk will log anny ERRORs
_ = helpers.SymbolicWalk(c.Fs.Source, dataDir, regularWalker)
- _ = helpers.SymbolicWalk(c.Fs.Source, c.PathSpec().AbsPathify(c.Cfg.GetString("contentDir")), symLinkWalker)
_ = helpers.SymbolicWalk(c.Fs.Source, i18nDir, regularWalker)
_ = helpers.SymbolicWalk(c.Fs.Source, layoutDir, regularWalker)
+
+ for _, contentDir := range c.PathSpec().ContentDirs() {
+ _ = helpers.SymbolicWalk(c.Fs.Source, contentDir.Value, symLinkWalker)
+ }
+
for _, staticDir := range staticDirs {
_ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker)
}
"github.com/spf13/cast"
)
+// KeyValueStr is a string tuple.
+type KeyValueStr struct {
+ Key string
+ Value string
+}
+
// KeyValues holds an key and a slice of values.
type KeyValues struct {
Key interface{}
return err
}
- contentPath := s.PathSpec.AbsPathify(filepath.Join(s.Cfg.GetString("contentDir"), targetPath))
+ // The site may have multiple content dirs, and we currently do not know which contentDir the
+ // user wants to create this content in. We should improve on this, but we start by testing if the
+ // provided path points to an existing dir. If so, use it as is.
+ var contentPath string
+ var exists bool
+ targetDir := filepath.Dir(targetPath)
+
+ if targetDir != "" && targetDir != "." {
+ exists, _ = helpers.Exists(targetDir, ps.Fs.Source)
+ }
+
+ if exists {
+ contentPath = targetPath
+ } else {
+ contentPath = s.PathSpec.AbsPathify(filepath.Join(s.Cfg.GetString("contentDir"), targetPath))
+ }
if err := helpers.SafeWriteToDisk(contentPath, bytes.NewReader(content), s.Fs.Source); err != nil {
return err
err error
)
- sp := source.NewSourceSpec(s.Deps.Cfg, s.Deps.Fs)
+ ps, err := helpers.NewPathSpec(s.Deps.Fs, s.Deps.Cfg)
+ sp := source.NewSourceSpec(ps, ps.Fs.Source)
+ if err != nil {
+ return nil, err
+ }
f := sp.NewFileInfo("", targetPath, false, nil)
name := f.TranslationBaseName()
+
if name == "index" || name == "_index" {
// Page bundles; the directory name will hopefully have a better name.
dir := strings.TrimSuffix(f.Dir(), helpers.FilePathSeparator)
for i, v := range c.expected {
found := strings.Contains(content, v)
if !found {
- t.Errorf("[%d] %q missing from output:\n%q", i, v, content)
+ t.Fatalf("[%d] %q missing from output:\n%q", i, v, content)
}
}
}
return nil, err
}
- sp := source.NewSourceSpec(cfg.Language, fs)
+ sp := source.NewSourceSpec(ps, fs.Source)
d := &Deps{
Fs: fs,
Title string
Weight int
+ Disabled bool
+
+ // If set per language, this tells Hugo that all content files without any
+ // language indicator (e.g. my-page.en.md) is in this language.
+ // This is usually a path relative to the working dir, but it can be an
+ // absolute directory referenece. It is what we get.
+ ContentDir string
+
Cfg config.Provider
// These are params declared in the [params] section of the language merged with the
params[k] = v
}
ToLowerMap(params)
- l := &Language{Lang: lang, Cfg: cfg, params: params, settings: make(map[string]interface{})}
+
+ defaultContentDir := cfg.GetString("contentDir")
+ if defaultContentDir == "" {
+ panic("contentDir not set")
+ }
+
+ l := &Language{Lang: lang, ContentDir: defaultContentDir, Cfg: cfg, params: params, settings: make(map[string]interface{})}
return l
}
func TestGetGlobalOnlySetting(t *testing.T) {
v := viper.New()
+ v.Set("defaultContentLanguageInSubdir", true)
+ v.Set("contentDir", "content")
+ v.Set("paginatePath", "page")
lang := NewDefaultLanguage(v)
lang.Set("defaultContentLanguageInSubdir", false)
lang.Set("paginatePath", "side")
- v.Set("defaultContentLanguageInSubdir", true)
- v.Set("paginatePath", "page")
require.True(t, lang.GetBool("defaultContentLanguageInSubdir"))
require.Equal(t, "side", lang.GetString("paginatePath"))
v := viper.New()
v.Set("p1", "p1cfg")
+ v.Set("contentDir", "content")
lang := NewDefaultLanguage(v)
lang.SetParam("p1", "p1p")
ErrThemeUndefined = errors.New("no theme set")
// ErrWalkRootTooShort is returned when the root specified for a file walk is shorter than 4 characters.
- ErrWalkRootTooShort = errors.New("Path too short. Stop walking.")
+ ErrPathTooShort = errors.New("file path is too short")
)
// filepathPathBridge is a bridge for common functionality in filepath vs path
// Sanity check
if len(root) < 4 {
- return ErrWalkRootTooShort
+ return ErrPathTooShort
}
// Handle the root first
}
func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) {
- fileInfo, err := LstatIfOs(fs, path)
+ fileInfo, err := LstatIfPossible(fs, path)
realPath := path
if err != nil {
if err != nil {
return nil, "", fmt.Errorf("Cannot read symbolic link '%s', error was: %s", path, err)
}
- fileInfo, err = LstatIfOs(fs, link)
+ fileInfo, err = LstatIfPossible(fs, link)
if err != nil {
return nil, "", fmt.Errorf("Cannot stat '%s', error was: %s", link, err)
}
return realPath, nil
}
-// Code copied from Afero's path.go
-// if the filesystem is OsFs use Lstat, else use fs.Stat
-func LstatIfOs(fs afero.Fs, path string) (info os.FileInfo, err error) {
- _, ok := fs.(*afero.OsFs)
- if ok {
- info, err = os.Lstat(path)
- } else {
- info, err = fs.Stat(path)
+// LstatIfPossible can be used to call Lstat if possible, else Stat.
+func LstatIfPossible(fs afero.Fs, path string) (os.FileInfo, error) {
+ if lstater, ok := fs.(afero.Lstater); ok {
+ fi, _, err := lstater.LstatIfPossible(path)
+ return fi, err
}
- return
+
+ return fs.Stat(path)
}
// SafeWriteToDisk is the same as WriteToDisk
for _, test := range tests {
v := viper.New()
- l := NewDefaultLanguage(v)
+ v.Set("contentDir", "content")
v.Set("removePathAccents", test.removeAccents)
+
+ l := NewDefaultLanguage(v)
p, err := NewPathSpec(hugofs.NewMem(v), l)
require.NoError(t, err)
func TestMakePathSanitized(t *testing.T) {
v := viper.New()
+ v.Set("contentDir", "content")
+
l := NewDefaultLanguage(v)
p, _ := NewPathSpec(hugofs.NewMem(v), l)
v := viper.New()
v.Set("disablePathToLower", true)
+ v.Set("contentDir", "content")
l := NewDefaultLanguage(v)
p, _ := NewPathSpec(hugofs.NewMem(v), l)
"fmt"
"strings"
+ "github.com/spf13/afero"
+
+ "github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/hugofs"
"github.com/spf13/cast"
theme string
// Directories
- contentDir string
- themesDir string
- layoutDir string
- workingDir string
- staticDirs []string
+ contentDir string
+ themesDir string
+ layoutDir string
+ workingDir string
+ staticDirs []string
+ absContentDirs []types.KeyValueStr
+
PublishDir string
// The PathSpec looks up its config settings in both the current language
// The file systems to use
Fs *hugofs.Fs
+ // The fine grained filesystems in play (resources, content etc.).
+ BaseFs *hugofs.BaseFs
+
// The config provider to use
Cfg config.Provider
}
languages = l
}
+ defaultContentLanguage := cfg.GetString("defaultContentLanguage")
+
+ // We will eventually pull out this badly placed path logic.
+ contentDir := cfg.GetString("contentDir")
+ workingDir := cfg.GetString("workingDir")
+ resourceDir := cfg.GetString("resourceDir")
+ publishDir := cfg.GetString("publishDir")
+
+ if len(languages) == 0 {
+ // We have some old tests that does not test the entire chain, hence
+ // they have no languages. So create one so we get the proper filesystem.
+ languages = Languages{&Language{Lang: "en", ContentDir: contentDir}}
+ }
+
+ absPuslishDir := AbsPathify(workingDir, publishDir)
+ if !strings.HasSuffix(absPuslishDir, FilePathSeparator) {
+ absPuslishDir += FilePathSeparator
+ }
+ // If root, remove the second '/'
+ if absPuslishDir == "//" {
+ absPuslishDir = FilePathSeparator
+ }
+ absResourcesDir := AbsPathify(workingDir, resourceDir)
+ if !strings.HasSuffix(absResourcesDir, FilePathSeparator) {
+ absResourcesDir += FilePathSeparator
+ }
+ if absResourcesDir == "//" {
+ absResourcesDir = FilePathSeparator
+ }
+
+ contentFs, absContentDirs, err := createContentFs(fs.Source, workingDir, defaultContentLanguage, languages)
+ if err != nil {
+ return nil, err
+ }
+
+ // Make sure we don't have any overlapping content dirs. That will never work.
+ for i, d1 := range absContentDirs {
+ for j, d2 := range absContentDirs {
+ if i == j {
+ continue
+ }
+ if strings.HasPrefix(d1.Value, d2.Value) || strings.HasPrefix(d2.Value, d1.Value) {
+ return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2)
+ }
+ }
+ }
+
+ resourcesFs := afero.NewBasePathFs(fs.Source, absResourcesDir)
+ publishFs := afero.NewBasePathFs(fs.Destination, absPuslishDir)
+
+ baseFs := &hugofs.BaseFs{
+ ContentFs: contentFs,
+ ResourcesFs: resourcesFs,
+ PublishFs: publishFs,
+ }
+
ps := &PathSpec{
Fs: fs,
+ BaseFs: baseFs,
Cfg: cfg,
disablePathToLower: cfg.GetBool("disablePathToLower"),
removePathAccents: cfg.GetBool("removePathAccents"),
Language: language,
Languages: languages,
defaultContentLanguageInSubdir: cfg.GetBool("defaultContentLanguageInSubdir"),
- defaultContentLanguage: cfg.GetString("defaultContentLanguage"),
+ defaultContentLanguage: defaultContentLanguage,
paginatePath: cfg.GetString("paginatePath"),
BaseURL: baseURL,
- contentDir: cfg.GetString("contentDir"),
+ contentDir: contentDir,
themesDir: cfg.GetString("themesDir"),
layoutDir: cfg.GetString("layoutDir"),
- workingDir: cfg.GetString("workingDir"),
+ workingDir: workingDir,
staticDirs: staticDirs,
+ absContentDirs: absContentDirs,
theme: cfg.GetString("theme"),
ProcessingStats: NewProcessingStats(lang),
}
}
}
- publishDir := ps.AbsPathify(cfg.GetString("publishDir")) + FilePathSeparator
- // If root, remove the second '/'
- if publishDir == "//" {
- publishDir = FilePathSeparator
- }
-
- ps.PublishDir = publishDir
+ // TODO(bep) remove this, eventually
+ ps.PublishDir = absPuslishDir
return ps, nil
}
return out
}
+func createContentFs(fs afero.Fs,
+ workingDir,
+ defaultContentLanguage string,
+ languages Languages) (afero.Fs, []types.KeyValueStr, error) {
+
+ var contentLanguages Languages
+ var contentDirSeen = make(map[string]bool)
+ languageSet := make(map[string]bool)
+
+ // The default content language needs to be first.
+ for _, language := range languages {
+ if language.Lang == defaultContentLanguage {
+ contentLanguages = append(contentLanguages, language)
+ contentDirSeen[language.ContentDir] = true
+ }
+ languageSet[language.Lang] = true
+ }
+
+ for _, language := range languages {
+ if contentDirSeen[language.ContentDir] {
+ continue
+ }
+ if language.ContentDir == "" {
+ language.ContentDir = defaultContentLanguage
+ }
+ contentDirSeen[language.ContentDir] = true
+ contentLanguages = append(contentLanguages, language)
+
+ }
+
+ var absContentDirs []types.KeyValueStr
+
+ fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs)
+ return fs, absContentDirs, err
+
+}
+
+func createContentOverlayFs(source afero.Fs,
+ workingDir string,
+ languages Languages,
+ languageSet map[string]bool,
+ absContentDirs *[]types.KeyValueStr) (afero.Fs, error) {
+ if len(languages) == 0 {
+ return source, nil
+ }
+
+ language := languages[0]
+
+ contentDir := language.ContentDir
+ if contentDir == "" {
+ panic("missing contentDir")
+ }
+
+ absContentDir := AbsPathify(workingDir, language.ContentDir)
+ if !strings.HasSuffix(absContentDir, FilePathSeparator) {
+ absContentDir += FilePathSeparator
+ }
+
+ // If root, remove the second '/'
+ if absContentDir == "//" {
+ absContentDir = FilePathSeparator
+ }
+
+ if len(absContentDir) < 6 {
+ return nil, fmt.Errorf("invalid content dir %q: %s", absContentDir, ErrPathTooShort)
+ }
+
+ *absContentDirs = append(*absContentDirs, types.KeyValueStr{Key: language.Lang, Value: absContentDir})
+
+ overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir))
+ if len(languages) == 1 {
+ return overlay, nil
+ }
+
+ base, err := createContentOverlayFs(source, workingDir, languages[1:], languageSet, absContentDirs)
+ if err != nil {
+ return nil, err
+ }
+
+ return hugofs.NewLanguageCompositeFs(base, overlay), nil
+
+}
+
+// RelContentDir tries to create a path relative to the content root from
+// the given filename. The return value is the path and language code.
+func (p *PathSpec) RelContentDir(filename string) (string, string) {
+ for _, dir := range p.absContentDirs {
+ if strings.HasPrefix(filename, dir.Value) {
+ rel := strings.TrimPrefix(filename, dir.Value)
+ return strings.TrimPrefix(rel, FilePathSeparator), dir.Key
+ }
+ }
+ // Either not a content dir or already relative.
+ return filename, ""
+}
+
+// ContentDirs returns all the content dirs (absolute paths).
+func (p *PathSpec) ContentDirs() []types.KeyValueStr {
+ return p.absContentDirs
+}
+
// PaginatePath returns the configured root path used for paginator pages.
func (p *PathSpec) PaginatePath() string {
return p.paginatePath
func TestNewPathSpecFromConfig(t *testing.T) {
v := viper.New()
+ v.Set("contentDir", "content")
l := NewLanguage("no", v)
v.Set("disablePathToLower", true)
v.Set("removePathAccents", true)
func newTestCfg(fs *hugofs.Fs) *viper.Viper {
v := viper.New()
+ v.Set("contentDir", "content")
v.SetFs(fs.Source)
func TestURLize(t *testing.T) {
v := viper.New()
+ v.Set("contentDir", "content")
l := NewDefaultLanguage(v)
p, _ := NewPathSpec(hugofs.NewMem(v), l)
for _, test := range tests {
v.Set("baseURL", test.baseURL)
+ v.Set("contentDir", "content")
l := NewLanguage(lang, v)
p, _ := NewPathSpec(hugofs.NewMem(v), l)
for i, test := range tests {
v.Set("baseURL", test.baseURL)
v.Set("canonifyURLs", test.canonify)
+ v.Set("contentDir", "content")
l := NewLanguage(lang, v)
p, _ := NewPathSpec(hugofs.NewMem(v), l)
for i, d := range data {
v := viper.New()
v.Set("uglyURLs", d.ugly)
+ v.Set("contentDir", "content")
l := NewDefaultLanguage(v)
p, _ := NewPathSpec(hugofs.NewMem(v), l)
--- /dev/null
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+ "github.com/spf13/afero"
+)
+
+// BaseFs contains the core base filesystems used by Hugo. The name "base" is used
+// to underline that even if they can be composites, they all have a base path set to a specific
+// resource folder, e.g "/my-project/content". So, no absolute filenames needed.
+type BaseFs struct {
+ // The filesystem used to capture content. This can be a composite and
+ // language aware file system.
+ ContentFs afero.Fs
+
+ // The filesystem used to store resources (processed images etc.).
+ // This usually maps to /my-project/resources.
+ ResourcesFs afero.Fs
+
+ // The filesystem used to publish the rendered site.
+ // This usually maps to /my-project/public.
+ PublishFs afero.Fs
+}
--- /dev/null
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+ "github.com/spf13/afero"
+)
+
+var (
+ _ afero.Fs = (*languageCompositeFs)(nil)
+ _ afero.Lstater = (*languageCompositeFs)(nil)
+)
+
+type languageCompositeFs struct {
+ *afero.CopyOnWriteFs
+}
+
+// NewLanguageCompositeFs creates a composite and language aware filesystem.
+// This is a hybrid filesystem. To get a specific file in Open, Stat etc., use the full filename
+// to the target filesystem. This information is available in Readdir, Stat etc. via the
+// special LanguageFileInfo FileInfo implementation.
+func NewLanguageCompositeFs(base afero.Fs, overlay *LanguageFs) afero.Fs {
+ return afero.NewReadOnlyFs(&languageCompositeFs{afero.NewCopyOnWriteFs(base, overlay).(*afero.CopyOnWriteFs)})
+}
+
+// Open takes the full path to the file in the target filesystem. If it is a directory, it gets merged
+// using the language as a weight.
+func (fs *languageCompositeFs) Open(name string) (afero.File, error) {
+ f, err := fs.CopyOnWriteFs.Open(name)
+ if err != nil {
+ return nil, err
+ }
+
+ fu, ok := f.(*afero.UnionFile)
+ if ok {
+ // This is a directory: Merge it.
+ fu.Merger = LanguageDirsMerger
+ }
+ return f, nil
+}
--- /dev/null
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+ "path/filepath"
+
+ "strings"
+
+ "testing"
+
+ "github.com/spf13/afero"
+ "github.com/stretchr/testify/require"
+)
+
+func TestCompositeLanguagFsTest(t *testing.T) {
+ assert := require.New(t)
+
+ languages := map[string]bool{
+ "sv": true,
+ "en": true,
+ "nn": true,
+ }
+ msv := afero.NewMemMapFs()
+ baseSv := "/content/sv"
+ lfssv := NewLanguageFs("sv", languages, afero.NewBasePathFs(msv, baseSv))
+ mnn := afero.NewMemMapFs()
+ baseNn := "/content/nn"
+ lfsnn := NewLanguageFs("nn", languages, afero.NewBasePathFs(mnn, baseNn))
+ men := afero.NewMemMapFs()
+ baseEn := "/content/en"
+ lfsen := NewLanguageFs("en", languages, afero.NewBasePathFs(men, baseEn))
+
+ // The order will be sv, en, nn
+ composite := NewLanguageCompositeFs(lfsnn, lfsen)
+ composite = NewLanguageCompositeFs(composite, lfssv)
+
+ afero.WriteFile(msv, filepath.Join(baseSv, "f1.txt"), []byte("some sv"), 0755)
+ afero.WriteFile(mnn, filepath.Join(baseNn, "f1.txt"), []byte("some nn"), 0755)
+ afero.WriteFile(men, filepath.Join(baseEn, "f1.txt"), []byte("some en"), 0755)
+
+ // Swedish is the top layer.
+ assertLangFile(t, composite, "f1.txt", "sv")
+
+ afero.WriteFile(msv, filepath.Join(baseSv, "f2.en.txt"), []byte("some sv"), 0755)
+ afero.WriteFile(mnn, filepath.Join(baseNn, "f2.en.txt"), []byte("some nn"), 0755)
+ afero.WriteFile(men, filepath.Join(baseEn, "f2.en.txt"), []byte("some en"), 0755)
+
+ // English is in the middle, but the most specific language match wins.
+ //assertLangFile(t, composite, "f2.en.txt", "en")
+
+ // Fetch some specific language versions
+ assertLangFile(t, composite, filepath.Join(baseNn, "f2.en.txt"), "nn")
+ assertLangFile(t, composite, filepath.Join(baseEn, "f2.en.txt"), "en")
+ assertLangFile(t, composite, filepath.Join(baseSv, "f2.en.txt"), "sv")
+
+ // Read the root
+ f, err := composite.Open("/")
+ assert.NoError(err)
+ defer f.Close()
+ files, err := f.Readdir(-1)
+ assert.Equal(4, len(files))
+ expected := map[string]bool{
+ filepath.FromSlash("/content/en/f1.txt"): true,
+ filepath.FromSlash("/content/nn/f1.txt"): true,
+ filepath.FromSlash("/content/sv/f1.txt"): true,
+ filepath.FromSlash("/content/en/f2.en.txt"): true,
+ }
+ got := make(map[string]bool)
+
+ for _, fi := range files {
+ fil, ok := fi.(*LanguageFileInfo)
+ assert.True(ok)
+ got[fil.Filename()] = true
+ }
+ assert.Equal(expected, got)
+}
+
+func assertLangFile(t testing.TB, fs afero.Fs, filename, match string) {
+ f, err := fs.Open(filename)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+ b, err := afero.ReadAll(f)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ s := string(b)
+ if !strings.Contains(s, match) {
+ t.Fatalf("got %q expected it to contain %q", s, match)
+
+ }
+}
--- /dev/null
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/spf13/afero"
+)
+
+const hugoFsMarker = "__hugofs"
+
+var (
+ _ LanguageAnnouncer = (*LanguageFileInfo)(nil)
+ _ FilePather = (*LanguageFileInfo)(nil)
+ _ afero.Lstater = (*LanguageFs)(nil)
+)
+
+// LanguageAnnouncer is aware of its language.
+type LanguageAnnouncer interface {
+ Lang() string
+ TranslationBaseName() string
+}
+
+// FilePather is aware of its file's location.
+type FilePather interface {
+ // Filename gets the full path and filename to the file.
+ Filename() string
+
+ // Path gets the content relative path including file name and extension.
+ // The directory is relative to the content root where "content" is a broad term.
+ Path() string
+
+ // RealName is FileInfo.Name in its original form.
+ RealName() string
+
+ BaseDir() string
+}
+
+var LanguageDirsMerger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) {
+ m := make(map[string]*LanguageFileInfo)
+
+ for _, fi := range lofi {
+ fil, ok := fi.(*LanguageFileInfo)
+ if !ok {
+ return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi)
+ }
+ m[fil.virtualName] = fil
+ }
+
+ for _, fi := range bofi {
+ fil, ok := fi.(*LanguageFileInfo)
+ if !ok {
+ return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi)
+ }
+ existing, found := m[fil.virtualName]
+
+ if !found || existing.weight < fil.weight {
+ m[fil.virtualName] = fil
+ }
+ }
+
+ merged := make([]os.FileInfo, len(m))
+ i := 0
+ for _, v := range m {
+ merged[i] = v
+ i++
+ }
+
+ return merged, nil
+}
+
+type LanguageFileInfo struct {
+ os.FileInfo
+ lang string
+ baseDir string
+ realFilename string
+ relFilename string
+ name string
+ realName string
+ virtualName string
+ translationBaseName string
+
+ // We add some weight to the files in their own language's content directory.
+ weight int
+}
+
+func (fi *LanguageFileInfo) Filename() string {
+ return fi.realFilename
+}
+
+func (fi *LanguageFileInfo) Path() string {
+ return fi.relFilename
+}
+
+func (fi *LanguageFileInfo) RealName() string {
+ return fi.realName
+}
+
+func (fi *LanguageFileInfo) BaseDir() string {
+ return fi.baseDir
+}
+
+func (fi *LanguageFileInfo) Lang() string {
+ return fi.lang
+}
+
+// TranslationBaseName returns the base filename without any extension or language
+// identificator.
+func (fi *LanguageFileInfo) TranslationBaseName() string {
+ return fi.translationBaseName
+}
+
+// Name is the name of the file within this filesystem without any path info.
+// It will be marked with language information so we can identify it as ours.
+func (fi *LanguageFileInfo) Name() string {
+ return fi.name
+}
+
+type languageFile struct {
+ afero.File
+ fs *LanguageFs
+}
+
+// Readdir creates FileInfo entries by calling Lstat if possible.
+func (l *languageFile) Readdir(c int) (ofi []os.FileInfo, err error) {
+ names, err := l.File.Readdirnames(c)
+ if err != nil {
+ return nil, err
+ }
+
+ fis := make([]os.FileInfo, len(names))
+
+ for i, name := range names {
+ fi, _, err := l.fs.LstatIfPossible(filepath.Join(l.Name(), name))
+
+ if err != nil {
+ return nil, err
+ }
+ fis[i] = fi
+ }
+
+ return fis, err
+}
+
+type LanguageFs struct {
+ // This Fs is usually created with a BasePathFs
+ basePath string
+ lang string
+ nameMarker string
+ languages map[string]bool
+ afero.Fs
+}
+
+func NewLanguageFs(lang string, languages map[string]bool, fs afero.Fs) *LanguageFs {
+ if lang == "" {
+ panic("no lang set for the language fs")
+ }
+ var basePath string
+
+ if bfs, ok := fs.(*afero.BasePathFs); ok {
+ basePath, _ = bfs.RealPath("")
+ }
+
+ marker := hugoFsMarker + "_" + lang + "_"
+
+ return &LanguageFs{lang: lang, languages: languages, basePath: basePath, Fs: fs, nameMarker: marker}
+}
+
+func (fs *LanguageFs) Lang() string {
+ return fs.lang
+}
+
+func (fs *LanguageFs) Stat(name string) (os.FileInfo, error) {
+ name, err := fs.realName(name)
+ if err != nil {
+ return nil, err
+ }
+
+ fi, err := fs.Fs.Stat(name)
+ if err != nil {
+ return nil, err
+ }
+
+ return fs.newLanguageFileInfo(name, fi)
+}
+
+func (fs *LanguageFs) Open(name string) (afero.File, error) {
+ name, err := fs.realName(name)
+ if err != nil {
+ return nil, err
+ }
+ f, err := fs.Fs.Open(name)
+
+ if err != nil {
+ return nil, err
+ }
+ return &languageFile{File: f, fs: fs}, nil
+}
+
+func (fs *LanguageFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
+ name, err := fs.realName(name)
+ if err != nil {
+ return nil, false, err
+ }
+
+ var fi os.FileInfo
+ var b bool
+
+ if lif, ok := fs.Fs.(afero.Lstater); ok {
+ fi, b, err = lif.LstatIfPossible(name)
+ } else {
+ fi, err = fs.Fs.Stat(name)
+ }
+
+ if err != nil {
+ return nil, b, err
+ }
+
+ lfi, err := fs.newLanguageFileInfo(name, fi)
+
+ return lfi, b, err
+}
+
+func (fs *LanguageFs) realPath(name string) (string, error) {
+ if baseFs, ok := fs.Fs.(*afero.BasePathFs); ok {
+ return baseFs.RealPath(name)
+ }
+ return name, nil
+}
+
+func (fs *LanguageFs) realName(name string) (string, error) {
+ if strings.Contains(name, hugoFsMarker) {
+ if !strings.Contains(name, fs.nameMarker) {
+ return "", os.ErrNotExist
+ }
+ return strings.Replace(name, fs.nameMarker, "", 1), nil
+ }
+
+ if fs.basePath == "" {
+ return name, nil
+ }
+
+ return strings.TrimPrefix(name, fs.basePath), nil
+}
+
+func (fs *LanguageFs) newLanguageFileInfo(filename string, fi os.FileInfo) (*LanguageFileInfo, error) {
+ filename = filepath.Clean(filename)
+ _, name := filepath.Split(filename)
+
+ realName := name
+ virtualName := name
+
+ realPath, err := fs.realPath(filename)
+ if err != nil {
+ return nil, err
+ }
+
+ lang := fs.Lang()
+
+ baseNameNoExt := ""
+
+ if !fi.IsDir() {
+
+ // Try to extract the language from the file name.
+ // Any valid language identificator in the name will win over the
+ // language set on the file system, e.g. "mypost.en.md".
+ baseName := filepath.Base(name)
+ ext := filepath.Ext(baseName)
+ baseNameNoExt = baseName
+
+ if ext != "" {
+ baseNameNoExt = strings.TrimSuffix(baseNameNoExt, ext)
+ }
+
+ fileLangExt := filepath.Ext(baseNameNoExt)
+ fileLang := strings.TrimPrefix(fileLangExt, ".")
+
+ if fs.languages[fileLang] {
+ lang = fileLang
+ }
+
+ baseNameNoExt = strings.TrimSuffix(baseNameNoExt, fileLangExt)
+
+ // This connects the filename to the filesystem, not the language.
+ virtualName = baseNameNoExt + "." + lang + ext
+
+ name = fs.nameMarker + name
+ }
+
+ weight := 1
+ // If this file's language belongs in this directory, add some weight to it
+ // to make it more important.
+ if lang == fs.Lang() {
+ weight = 2
+ }
+
+ if fi.IsDir() {
+ // For directories we always want to start from the union view.
+ realPath = strings.TrimPrefix(realPath, fs.basePath)
+ }
+
+ return &LanguageFileInfo{
+ lang: lang,
+ weight: weight,
+ realFilename: realPath,
+ realName: realName,
+ relFilename: strings.TrimPrefix(strings.TrimPrefix(realPath, fs.basePath), string(os.PathSeparator)),
+ name: name,
+ virtualName: virtualName,
+ translationBaseName: baseNameNoExt,
+ baseDir: fs.basePath,
+ FileInfo: fi}, nil
+}
--- /dev/null
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+ "path/filepath"
+ "testing"
+
+ "github.com/spf13/afero"
+ "github.com/stretchr/testify/require"
+)
+
+func TestLanguagFs(t *testing.T) {
+ languages := map[string]bool{
+ "sv": true,
+ }
+ base := filepath.FromSlash("/my/base")
+ assert := require.New(t)
+ m := afero.NewMemMapFs()
+ bfs := afero.NewBasePathFs(m, base)
+ lfs := NewLanguageFs("sv", languages, bfs)
+ assert.NotNil(lfs)
+ assert.Equal("sv", lfs.Lang())
+ err := afero.WriteFile(lfs, filepath.FromSlash("sect/page.md"), []byte("abc"), 0777)
+ assert.NoError(err)
+ fi, err := lfs.Stat(filepath.FromSlash("sect/page.md"))
+ assert.NoError(err)
+ assert.Equal("__hugofs_sv_page.md", fi.Name())
+
+ languager, ok := fi.(LanguageAnnouncer)
+ assert.True(ok)
+
+ assert.Equal("sv", languager.Lang())
+
+ lfi, ok := fi.(*LanguageFileInfo)
+ assert.True(ok)
+ assert.Equal(filepath.FromSlash("/my/base/sect/page.md"), lfi.Filename())
+ assert.Equal(filepath.FromSlash("sect/page.md"), lfi.Path())
+ assert.Equal("page.sv.md", lfi.virtualName)
+ assert.Equal("__hugofs_sv_page.md", lfi.Name())
+ assert.Equal("page.md", lfi.RealName())
+
+}
} else {
languages = make(map[string]interface{})
for k, v := range languagesFromConfig {
- isDisabled := false
for _, disabled := range disableLanguages {
if disabled == defaultLang {
return fmt.Errorf("cannot disable default language %q", defaultLang)
}
if strings.EqualFold(k, disabled) {
- isDisabled = true
+ v.(map[string]interface{})["disabled"] = true
break
}
}
- if !isDisabled {
- languages[k] = v
- }
-
+ languages[k] = v
}
}
writeSource(t, fs, "content/sect/p1.md", fmt.Sprintf(pageTemplate, "P1", "- tag1"))
- writeNewContentFile(t, fs, "Category Terms", "2017-01-01", "content/categories/_index.md", 10)
- writeNewContentFile(t, fs, "Tag1 List", "2017-01-01", "content/tags/tag1/_index.md", 10)
+ writeNewContentFile(t, fs.Source, "Category Terms", "2017-01-01", "content/categories/_index.md", 10)
+ writeNewContentFile(t, fs.Source, "Tag1 List", "2017-01-01", "content/tags/tag1/_index.md", 10)
h, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg})
package hugolib
import (
- "os"
"strings"
"github.com/gohugoio/hugo/helpers"
var (
_ source.File = (*fileInfo)(nil)
_ source.ReadableFile = (*fileInfo)(nil)
+ _ pathLangFile = (*fileInfo)(nil)
)
+// A partial interface to prevent ambigous compiler error.
+type basePather interface {
+ Filename() string
+ RealName() string
+ BaseDir() string
+}
+
type fileInfo struct {
bundleTp bundleDirType
+
source.ReadableFile
+ basePather
+
overriddenLang string
// Set if the content language for this file is disabled.
return fi.ReadableFile.Lang()
}
+func (fi *fileInfo) Filename() string {
+ return fi.basePather.Filename()
+}
+
func (fi *fileInfo) isOwner() bool {
return fi.bundleTp > bundleNot
}
return contentFileExtensionsSet[fi.Ext()]
}
-func newFileInfo(sp *source.SourceSpec, baseDir, filename string, fi os.FileInfo, tp bundleDirType) *fileInfo {
+func newFileInfo(sp *source.SourceSpec, baseDir, filename string, fi pathLangFileFi, tp bundleDirType) *fileInfo {
baseFi := sp.NewFileInfo(baseDir, filename, tp == bundleLeaf, fi)
f := &fileInfo{
bundleTp: tp,
ReadableFile: baseFi,
+ basePather: fi,
}
lang := f.Lang()
+++ /dev/null
-// Copyright 2017-present The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hugolib
-
-import (
- "testing"
-
- "path/filepath"
-
- "github.com/gohugoio/hugo/source"
- "github.com/stretchr/testify/require"
-)
-
-func TestBundleFileInfo(t *testing.T) {
- t.Parallel()
-
- assert := require.New(t)
- cfg, fs := newTestBundleSourcesMultilingual(t)
- sourceSpec := source.NewSourceSpec(cfg, fs)
-
- for _, this := range []struct {
- filename string
- check func(f *fileInfo)
- }{
- {"/path/to/file.md", func(fi *fileInfo) {
- assert.Equal("md", fi.Ext())
- assert.Equal("en", fi.Lang())
- assert.False(fi.isOwner())
- assert.True(fi.isContentFile())
- }},
- {"/path/to/file.JPG", func(fi *fileInfo) {
- assert.Equal("jpg", fi.Ext())
- assert.False(fi.isContentFile())
- }},
- {"/path/to/file.nn.png", func(fi *fileInfo) {
- assert.Equal("png", fi.Ext())
- assert.Equal("nn", fi.Lang())
- assert.Equal("file", fi.TranslationBaseName())
- assert.False(fi.isContentFile())
- }},
- } {
- fi := newFileInfo(
- sourceSpec,
- filepath.FromSlash("/work/base"),
- filepath.FromSlash(this.filename),
- nil, bundleNot)
- this.check(fi)
- }
-
-}
// GetContentPage finds a Page with content given the absolute filename.
// Returns nil if none found.
func (h *HugoSites) GetContentPage(filename string) *Page {
- s := h.Sites[0]
- contendDir := filepath.Join(s.PathSpec.AbsPathify(s.Cfg.GetString("contentDir")))
- if !strings.HasPrefix(filename, contendDir) {
- return nil
- }
-
- rel := strings.TrimPrefix(filename, contendDir)
- rel = strings.TrimPrefix(rel, helpers.FilePathSeparator)
-
for _, s := range h.Sites {
-
- pos := s.rawAllPages.findPagePosByFilePath(rel)
-
+ pos := s.rawAllPages.findPagePosByFilename(filename)
if pos == -1 {
continue
}
}
// If not found already, this may be bundled in another content file.
- rel = filepath.Dir(rel)
- for _, s := range h.Sites {
-
- pos := s.rawAllPages.findFirstPagePosByFilePathPrefix(rel)
+ dir := filepath.Dir(filename)
+ for _, s := range h.Sites {
+ pos := s.rawAllPages.findPagePosByFilnamePrefix(dir)
if pos == -1 {
continue
}
return s.rawAllPages[pos]
}
-
return nil
-
}
// NewHugoSites creates a new collection of sites given the input sites, building
var contentChangeTracker *contentChangeMap
- // Only needed in server mode.
- // TODO(bep) clean up the running vs watching terms
- if cfg.Running {
- contentChangeTracker = &contentChangeMap{symContent: make(map[string]map[string]bool)}
- }
-
h := &HugoSites{
- running: cfg.Running,
- multilingual: langConfig,
- multihost: cfg.Cfg.GetBool("multihost"),
- ContentChanges: contentChangeTracker,
- Sites: sites}
+ running: cfg.Running,
+ multilingual: langConfig,
+ multihost: cfg.Cfg.GetBool("multihost"),
+ Sites: sites}
for _, s := range sites {
s.owner = h
h.Deps = sites[0].Deps
+ // Only needed in server mode.
+ // TODO(bep) clean up the running vs watching terms
+ if cfg.Running {
+ contentChangeTracker = &contentChangeMap{pathSpec: h.PathSpec, symContent: make(map[string]map[string]bool)}
+ h.ContentChanges = contentChangeTracker
+ }
+
if err := h.initGitInfo(); err != nil {
return nil, err
}
d.OutputFormatsConfig = s.outputFormatsConfig
s.Deps = d
}
+
s.resourceSpec, err = resource.NewSpec(s.Deps.PathSpec, s.mediaTypesConfig)
if err != nil {
return err
languages := getLanguages(cfg.Cfg)
for _, lang := range languages {
+ if lang.Disabled {
+ continue
+ }
var s *Site
var err error
cfg.Language = lang
return nil
}
-func (h *HugoSites) removePageByPath(path string) {
+func (h *HugoSites) removePageByFilename(filename string) {
for _, s := range h.Sites {
- s.removePageByPath(path)
+ s.removePageFilename(filename)
}
}
branches []string
leafs []string
+ pathSpec *helpers.PathSpec
+
// Hugo supports symlinked content (both directories and files). This
// can lead to situations where the same file can be referenced from several
// locations in /content -- which is really cool, but also means we have to
func (m *contentChangeMap) add(filename string, tp bundleDirType) {
m.mu.Lock()
- dir := filepath.Dir(filename)
+ dir := filepath.Dir(filename) + helpers.FilePathSeparator
switch tp {
case bundleBranch:
m.branches = append(m.branches, dir)
// Track the addition of bundle dirs.
func (m *contentChangeMap) handleBundles(b *bundleDirs) {
for _, bd := range b.bundles {
- m.add(bd.fi.Filename(), bd.tp)
+ m.add(bd.fi.Path(), bd.tp)
}
}
m.mu.RLock()
defer m.mu.RUnlock()
- dir, name := filepath.Split(filename)
- dir = strings.TrimSuffix(dir, helpers.FilePathSeparator)
- fileTp, isContent := classifyBundledFile(name)
-
- // If the file itself is a bundle, no need to look further:
- if fileTp > bundleNot {
- return dir, dir, fileTp
+ // Bundles share resources, so we need to start from the virtual root.
+ relPath, _ := m.pathSpec.RelContentDir(filename)
+ dir, name := filepath.Split(relPath)
+ if !strings.HasSuffix(dir, helpers.FilePathSeparator) {
+ dir += helpers.FilePathSeparator
}
+ fileTp, _ := classifyBundledFile(name)
+
// This may be a member of a bundle. Start with branch bundles, the most specific.
- if !isContent {
+ if fileTp != bundleLeaf {
for i, b := range m.branches {
if b == dir {
m.branches = append(m.branches[:i], m.branches[i+1:]...)
- return dir, dir, bundleBranch
+ return dir, b, bundleBranch
}
}
}
for i, l := range m.leafs {
if strings.HasPrefix(dir, l) {
m.leafs = append(m.leafs[:i], m.leafs[i+1:]...)
- return dir, dir, bundleLeaf
+ return dir, l, bundleLeaf
}
}
import (
"bytes"
"fmt"
+ "io"
"strings"
"testing"
// t.Parallel() not supported, see https://github.com/fortytw2/leaktest/issues/4
// This leaktest seems to be a little bit shaky on Travis.
if !isCI() {
- defer leaktest.CheckTimeout(t, 30*time.Second)()
+ defer leaktest.CheckTimeout(t, 10*time.Second)()
}
assert := require.New(t)
b.AssertFileContent("public/fr/sect/doc1/index.html", "Single", "Shortcode: Bonjour")
b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Single", "Shortcode: Hello")
+ contentFs := b.H.BaseFs.ContentFs
+
for i, this := range []struct {
preFunc func(t *testing.T)
events []fsnotify.Event
},
{
func(t *testing.T) {
- writeNewContentFile(t, fs, "new_en_1", "2016-07-31", "content/new1.en.md", -5)
- writeNewContentFile(t, fs, "new_en_2", "1989-07-30", "content/new2.en.md", -10)
- writeNewContentFile(t, fs, "new_fr_1", "2016-07-30", "content/new1.fr.md", 10)
+ writeNewContentFile(t, contentFs, "new_en_1", "2016-07-31", "new1.en.md", -5)
+ writeNewContentFile(t, contentFs, "new_en_2", "1989-07-30", "new2.en.md", -10)
+ writeNewContentFile(t, contentFs, "new_fr_1", "2016-07-30", "new1.fr.md", 10)
},
[]fsnotify.Event{
{Name: filepath.FromSlash("content/new1.en.md"), Op: fsnotify.Create},
},
{
func(t *testing.T) {
- p := "content/sect/doc1.en.md"
- doc1 := readSource(t, fs, p)
+ p := "sect/doc1.en.md"
+ doc1 := readFileFromFs(t, contentFs, p)
doc1 += "CHANGED"
- writeSource(t, fs, p, doc1)
+ writeToFs(t, contentFs, p, doc1)
},
[]fsnotify.Event{{Name: filepath.FromSlash("content/sect/doc1.en.md"), Op: fsnotify.Write}},
func(t *testing.T) {
// Rename a file
{
func(t *testing.T) {
- if err := fs.Source.Rename("content/new1.en.md", "content/new1renamed.en.md"); err != nil {
+ if err := contentFs.Rename("new1.en.md", "new1renamed.en.md"); err != nil {
t.Fatalf("Rename failed: %s", err)
}
},
title = "Svenska"
`
- writeNewContentFile(t, fs, "Swedish Contentfile", "2016-01-01", "content/sect/doc1.sv.md", 10)
+ writeNewContentFile(t, fs.Source, "Swedish Contentfile", "2016-01-01", "content/sect/doc1.sv.md", 10)
// replace the config
b.WithNewConfig(newConfig)
if err != nil {
// Print some debug info
root := strings.Split(filename, helpers.FilePathSeparator)[0]
- afero.Walk(fs, root, func(path string, info os.FileInfo, err error) error {
- if info != nil && !info.IsDir() {
- fmt.Println(" ", path)
- }
-
- return nil
- })
+ printFs(fs, root, os.Stdout)
Fatalf(t, "Failed to read file: %s", err)
}
return string(b)
}
+func printFs(fs afero.Fs, path string, w io.Writer) {
+ if fs == nil {
+ return
+ }
+ afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {
+ if info != nil && !info.IsDir() {
+ s := path
+ if lang, ok := info.(hugofs.LanguageAnnouncer); ok {
+ s = s + "\tLANG: " + lang.Lang()
+ }
+ if fp, ok := info.(hugofs.FilePather); ok {
+ s = s + "\tRF: " + fp.Filename() + "\tBP: " + fp.BaseDir()
+ }
+ fmt.Fprintln(w, " ", s)
+ }
+ return nil
+ })
+}
+
const testPageTemplate = `---
title: "%s"
publishdate: "%s"
return fmt.Sprintf(testPageTemplate, title, date, weight, title)
}
-func writeNewContentFile(t *testing.T, fs *hugofs.Fs, title, date, filename string, weight int) {
+func writeNewContentFile(t *testing.T, fs afero.Fs, title, date, filename string, weight int) {
content := newTestPage(title, date, weight)
- writeSource(t, fs, filename, content)
+ writeToFs(t, fs, filename, content)
}
type multiSiteTestBuilder struct {
--- /dev/null
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+/*
+
+/en/p1.md
+/nn/p1.md
+
+.Readdir
+
+- Name() => p1.en.md, p1.nn.md
+
+.Stat(name)
+
+.Open() --- real file name
+
+
+*/
+
+func TestLanguageContentRoot(t *testing.T) {
+ t.Parallel()
+ assert := require.New(t)
+
+ config := `
+baseURL = "https://example.org/"
+
+defaultContentLanguage = "en"
+defaultContentLanguageInSubdir = true
+
+contentDir = "content/main"
+workingDir = "/my/project"
+
+[Languages]
+[Languages.en]
+weight = 10
+title = "In English"
+languageName = "English"
+
+[Languages.nn]
+weight = 20
+title = "På Norsk"
+languageName = "Norsk"
+# This tells Hugo that all content in this directory is in the Norwegian language.
+# It does not have to have the "my-page.nn.md" format. It can, but that is optional.
+contentDir = "content/norsk"
+
+[Languages.sv]
+weight = 30
+title = "På Svenska"
+languageName = "Svensk"
+contentDir = "content/svensk"
+`
+
+ pageTemplate := `
+---
+title: %s
+slug: %s
+weight: %d
+---
+
+Content.
+
+`
+
+ pageBundleTemplate := `
+---
+title: %s
+weight: %d
+---
+
+Content.
+
+`
+ var contentFiles []string
+ section := "sect"
+
+ var contentRoot = func(lang string) string {
+ contentRoot := "content/main"
+
+ switch lang {
+ case "nn":
+ contentRoot = "content/norsk"
+ case "sv":
+ contentRoot = "content/svensk"
+ }
+ return contentRoot + "/" + section
+ }
+
+ for _, lang := range []string{"en", "nn", "sv"} {
+ for j := 1; j <= 10; j++ {
+ if (lang == "nn" || lang == "en") && j%4 == 0 {
+ // Skip 4 and 8 for nn
+ // We also skip it for en, but that is added to the Swedish directory below.
+ continue
+ }
+
+ if lang == "sv" && j%5 == 0 {
+ // Skip 5 and 10 for sv
+ continue
+ }
+
+ base := fmt.Sprintf("p-%s-%d", lang, j)
+ slug := fmt.Sprintf("%s", base)
+ langID := ""
+
+ if lang == "sv" && j%4 == 0 {
+ // Put an English page in the Swedish content dir.
+ langID = ".en"
+ }
+
+ if lang == "en" && j == 8 {
+ // This should win over the sv variant above.
+ langID = ".en"
+ }
+
+ slug += langID
+
+ contentRoot := contentRoot(lang)
+
+ filename := filepath.Join(contentRoot, fmt.Sprintf("page%d%s.md", j, langID))
+ contentFiles = append(contentFiles, filename, fmt.Sprintf(pageTemplate, slug, slug, j))
+ }
+ }
+
+ // Put common translations in all of them
+ for i, lang := range []string{"en", "nn", "sv"} {
+ contentRoot := contentRoot(lang)
+
+ slug := fmt.Sprintf("common_%s", lang)
+
+ filename := filepath.Join(contentRoot, "common.md")
+ contentFiles = append(contentFiles, filename, fmt.Sprintf(pageTemplate, slug, slug, 100+i))
+
+ for j, lang2 := range []string{"en", "nn", "sv"} {
+ filename := filepath.Join(contentRoot, fmt.Sprintf("translated_all.%s.md", lang2))
+ langSlug := slug + "_translated_all_" + lang2
+ contentFiles = append(contentFiles, filename, fmt.Sprintf(pageTemplate, langSlug, langSlug, 200+i+j))
+ }
+
+ for j, lang2 := range []string{"sv", "nn"} {
+ if lang == "en" {
+ continue
+ }
+ filename := filepath.Join(contentRoot, fmt.Sprintf("translated_some.%s.md", lang2))
+ langSlug := slug + "_translated_some_" + lang2
+ contentFiles = append(contentFiles, filename, fmt.Sprintf(pageTemplate, langSlug, langSlug, 300+i+j))
+ }
+ }
+
+ // Add a bundle with some images
+ for i, lang := range []string{"en", "nn", "sv"} {
+ contentRoot := contentRoot(lang)
+ slug := fmt.Sprintf("bundle_%s", lang)
+ filename := filepath.Join(contentRoot, "mybundle", "index.md")
+ contentFiles = append(contentFiles, filename, fmt.Sprintf(pageBundleTemplate, slug, 400+i))
+ if lang == "en" {
+ imageFilename := filepath.Join(contentRoot, "mybundle", "logo.png")
+ contentFiles = append(contentFiles, imageFilename, "PNG Data")
+ }
+ imageFilename := filepath.Join(contentRoot, "mybundle", "featured.png")
+ contentFiles = append(contentFiles, imageFilename, fmt.Sprintf("PNG Data for %s", lang))
+
+ // Add some bundled pages
+ contentFiles = append(contentFiles, filepath.Join(contentRoot, "mybundle", "p1.md"), fmt.Sprintf(pageBundleTemplate, slug, 401+i))
+ contentFiles = append(contentFiles, filepath.Join(contentRoot, "mybundle", "sub", "p1.md"), fmt.Sprintf(pageBundleTemplate, slug, 402+i))
+
+ }
+
+ b := newTestSitesBuilder(t)
+ b.WithWorkingDir("/my/project").WithConfigFile("toml", config).WithContent(contentFiles...).CreateSites()
+
+ _ = os.Stdout
+ //printFs(b.H.BaseFs.ContentFs, "/", os.Stdout)
+
+ b.Build(BuildCfg{})
+
+ assert.Equal(3, len(b.H.Sites))
+
+ enSite := b.H.Sites[0]
+ nnSite := b.H.Sites[1]
+ svSite := b.H.Sites[2]
+
+ //dumpPages(nnSite.RegularPages...)
+ assert.Equal(12, len(nnSite.RegularPages))
+ assert.Equal(13, len(enSite.RegularPages))
+
+ assert.Equal(10, len(svSite.RegularPages))
+
+ for i, p := range enSite.RegularPages {
+ j := i + 1
+ msg := fmt.Sprintf("Test %d", j)
+ assert.Equal("en", p.Lang(), msg)
+ assert.Equal("sect", p.Section())
+ if j < 9 {
+ if j%4 == 0 {
+ assert.Contains(p.Title(), fmt.Sprintf("p-sv-%d.en", i+1), msg)
+ } else {
+ assert.Contains(p.Title(), "p-en", msg)
+ }
+ }
+ }
+
+ // Check bundles
+ bundleEn := enSite.RegularPages[len(enSite.RegularPages)-1]
+ bundleNn := nnSite.RegularPages[len(nnSite.RegularPages)-1]
+ bundleSv := svSite.RegularPages[len(svSite.RegularPages)-1]
+
+ assert.Equal("/en/sect/mybundle/", bundleEn.RelPermalink())
+ assert.Equal("/sv/sect/mybundle/", bundleSv.RelPermalink())
+
+ assert.Equal(4, len(bundleEn.Resources))
+ assert.Equal(4, len(bundleNn.Resources))
+ assert.Equal(4, len(bundleSv.Resources))
+
+ assert.Equal("/en/sect/mybundle/logo.png", bundleEn.Resources.GetMatch("logo*").RelPermalink())
+ assert.Equal("/nn/sect/mybundle/logo.png", bundleNn.Resources.GetMatch("logo*").RelPermalink())
+ assert.Equal("/sv/sect/mybundle/logo.png", bundleSv.Resources.GetMatch("logo*").RelPermalink())
+
+ b.AssertFileContent("/my/project/public/sv/sect/mybundle/featured.png", "PNG Data for sv")
+ b.AssertFileContent("/my/project/public/nn/sect/mybundle/featured.png", "PNG Data for nn")
+ b.AssertFileContent("/my/project/public/en/sect/mybundle/featured.png", "PNG Data for en")
+ b.AssertFileContent("/my/project/public/en/sect/mybundle/logo.png", "PNG Data")
+ b.AssertFileContent("/my/project/public/sv/sect/mybundle/logo.png", "PNG Data")
+ b.AssertFileContent("/my/project/public/nn/sect/mybundle/logo.png", "PNG Data")
+
+ nnSect := nnSite.getPage(KindSection, "sect")
+ assert.NotNil(nnSect)
+ assert.Equal(12, len(nnSect.Pages))
+ nnHome, _ := nnSite.Info.Home()
+ assert.Equal("/nn/", nnHome.RelPermalink())
+
+}
writeSource(t, fs, "content/sect2/p4.md", fmt.Sprintf(menuPageTemplate, "p4", 4, "main", "atitle4", 10))
writeSource(t, fs, "content/sect3/p5.md", fmt.Sprintf(menuPageTemplate, "p5", 5, "main", "atitle5", 5))
- writeNewContentFile(t, fs, "Section One", "2017-01-01", "content/sect1/_index.md", 100)
- writeNewContentFile(t, fs, "Section Five", "2017-01-01", "content/sect5/_index.md", 10)
+ writeNewContentFile(t, fs.Source, "Section One", "2017-01-01", "content/sect1/_index.md", 100)
+ writeNewContentFile(t, fs.Source, "Section Five", "2017-01-01", "content/sect5/_index.md", 10)
err := h.Build(BuildCfg{})
language.LanguageName = cast.ToString(v)
case "weight":
language.Weight = cast.ToInt(v)
+ case "contentdir":
+ language.ContentDir = cast.ToString(v)
+ case "disabled":
+ language.Disabled = cast.ToBool(v)
case "params":
m := cast.ToStringMap(v)
// Needed for case insensitive fetching of params values
return fmt.Sprintf("Pages(%d)", len(ps))
}
-func (ps Pages) findPagePosByFilePath(inPath string) int {
+func (ps Pages) findPagePosByFilename(filename string) int {
for i, x := range ps {
- if x.Source.Path() == inPath {
+ if x.Source.Filename() == filename {
return i
}
}
return ps
}
-func (ps Pages) findFirstPagePosByFilePathPrefix(prefix string) int {
+func (ps Pages) findPagePosByFilnamePrefix(prefix string) int {
if prefix == "" {
return -1
}
+
+ lenDiff := -1
+ currPos := -1
+ prefixLen := len(prefix)
+
+ // Find the closest match
for i, x := range ps {
- if strings.HasPrefix(x.Source.Path(), prefix) {
- return i
+ if strings.HasPrefix(x.Source.Filename(), prefix) {
+ diff := len(x.Source.Filename()) - prefixLen
+ if lenDiff == -1 || diff < lenDiff {
+ lenDiff = diff
+ currPos = i
+ }
}
}
- return -1
+ return currPos
}
// findPagePos Given a page, it will find the position in Pages
"fmt"
"math"
"runtime"
- "strings"
// Use this until errgroup gets ported to context
// See https://github.com/golang/go/issues/19781
)
type siteContentProcessor struct {
- baseDir string
-
site *Site
handleContent contentHandler
fileSinglesChan chan *fileInfo
// These assets should be just copied to destination.
- fileAssetsChan chan []string
+ fileAssetsChan chan []pathLangFile
numWorkers int
}
}
-func (s *siteContentProcessor) processAssets(assets []string) {
+func (s *siteContentProcessor) processAssets(assets []pathLangFile) {
select {
case s.fileAssetsChan <- assets:
case <-s.ctx.Done():
}
}
-func newSiteContentProcessor(ctx context.Context, baseDir string, partialBuild bool, s *Site) *siteContentProcessor {
+func newSiteContentProcessor(ctx context.Context, partialBuild bool, s *Site) *siteContentProcessor {
numWorkers := 12
if n := runtime.NumCPU() * 3; n > numWorkers {
numWorkers = n
return &siteContentProcessor{
ctx: ctx,
partialBuild: partialBuild,
- baseDir: baseDir,
site: s,
handleContent: newHandlerChain(s),
fileBundlesChan: make(chan *bundleDir, numWorkers),
fileSinglesChan: make(chan *fileInfo, numWorkers),
- fileAssetsChan: make(chan []string, numWorkers),
+ fileAssetsChan: make(chan []pathLangFile, numWorkers),
numWorkers: numWorkers,
pagesChan: make(chan *Page, numWorkers),
}
g2.Go(func() error {
for {
select {
- case filenames, ok := <-s.fileAssetsChan:
+ case files, ok := <-s.fileAssetsChan:
if !ok {
return nil
}
- for _, filename := range filenames {
- name := strings.TrimPrefix(filename, s.baseDir)
- f, err := s.site.Fs.Source.Open(filename)
+ for _, file := range files {
+ f, err := s.site.BaseFs.ContentFs.Open(file.Filename())
if err != nil {
- return err
+ return fmt.Errorf("failed to open assets file: %s", err)
}
-
- err = s.site.publish(&s.site.PathSpec.ProcessingStats.Files, name, f)
+ err = s.site.publish(&s.site.PathSpec.ProcessingStats.Files, file.Path(), f)
f.Close()
if err != nil {
return err
}
func (s *siteContentProcessor) readAndConvertContentFile(file *fileInfo) error {
- ctx := &handlerContext{source: file, baseDir: s.baseDir, pages: s.pagesChan}
+ ctx := &handlerContext{source: file, pages: s.pagesChan}
return s.handleContent(ctx).err
}
func (s *siteContentProcessor) readAndConvertContentBundle(bundle *bundleDir) error {
- ctx := &handlerContext{bundle: bundle, baseDir: s.baseDir, pages: s.pagesChan}
+ ctx := &handlerContext{bundle: bundle, pages: s.pagesChan}
return s.handleContent(ctx).err
}
"errors"
"fmt"
"os"
+ "path"
"path/filepath"
"runtime"
"strings"
"sync"
+ "github.com/spf13/afero"
+
+ "github.com/gohugoio/hugo/hugofs"
+
"github.com/gohugoio/hugo/helpers"
"golang.org/x/sync/errgroup"
"github.com/gohugoio/hugo/source"
- "github.com/spf13/afero"
jww "github.com/spf13/jwalterweatherman"
)
fs afero.Fs
logger *jww.Notepad
- baseDir string
-
// Filenames limits the content to process to a list of filenames/directories.
// This is used for partial building in server mode.
filenames []string
sourceSpec *source.SourceSpec,
handler captureResultHandler,
contentChanges *contentChangeMap,
- baseDir string, filenames ...string) *capturer {
+ filenames ...string) *capturer {
numWorkers := 4
if n := runtime.NumCPU(); n > numWorkers {
sem: make(chan bool, numWorkers),
handler: handler,
sourceSpec: sourceSpec,
+ fs: sourceSpec.Fs,
logger: logger,
contentChanges: contentChanges,
- fs: sourceSpec.Fs.Source, baseDir: baseDir, seen: make(map[string]bool),
- filenames: filenames}
+ seen: make(map[string]bool),
+ filenames: filenames}
return c
}
// these channels.
type captureResultHandler interface {
handleSingles(fis ...*fileInfo)
- handleCopyFiles(filenames ...string)
+ handleCopyFiles(fis ...pathLangFile)
captureBundlesHandler
}
}
}
-func (c *captureResultHandlerChain) handleCopyFiles(filenames ...string) {
+func (c *captureResultHandlerChain) handleCopyFiles(files ...pathLangFile) {
for _, h := range c.handlers {
if hh, ok := h.(captureResultHandler); ok {
- hh.handleCopyFiles(filenames...)
+ hh.handleCopyFiles(files...)
}
}
}
return err
}
default:
- fi, _, err := c.getRealFileInfo(resolvedFilename)
+ fi, err := c.resolveRealPath(resolvedFilename)
if os.IsNotExist(err) {
// File has been deleted.
continue
// Just in case the owning dir is a new symlink -- this will
// create the proper mapping for it.
- c.getRealFileInfo(dir)
+ c.resolveRealPath(dir)
- f, active := c.newFileInfo(resolvedFilename, fi, tp)
+ f, active := c.newFileInfo(fi, tp)
if active {
c.copyOrHandleSingle(f)
}
return c.capturePartial(c.filenames...)
}
- err := c.handleDir(c.baseDir)
+ err := c.handleDir(helpers.FilePathSeparator)
if err != nil {
return err
}
func (c *capturer) handleBranchDir(dirname string) error {
files, err := c.readDir(dirname)
if err != nil {
+
return err
}
for _, fi := range files {
if !fi.IsDir() {
- tp, _ := classifyBundledFile(fi.Name())
+ tp, _ := classifyBundledFile(fi.RealName())
if dirType == bundleNot {
dirType = tp
}
dirs := newBundleDirs(bundleBranch, c)
- for _, fi := range files {
+ var secondPass []*fileInfo
+ // Handle potential bundle headers first.
+ for _, fi := range files {
if fi.IsDir() {
continue
}
- tp, isContent := classifyBundledFile(fi.Name())
+ tp, isContent := classifyBundledFile(fi.RealName())
+
+ f, active := c.newFileInfo(fi, tp)
- f, active := c.newFileInfo(fi.filename, fi.FileInfo, tp)
if !active {
continue
}
- if f.isOwner() {
- dirs.addBundleHeader(f)
- } else if !isContent {
- // This is a partial update -- we only care about the files that
- // is in this bundle.
- dirs.addBundleFiles(f)
+
+ if !f.isOwner() {
+ if !isContent {
+ // This is a partial update -- we only care about the files that
+ // is in this bundle.
+ secondPass = append(secondPass, f)
+ }
+ continue
}
+ dirs.addBundleHeader(f)
+ }
+
+ for _, f := range secondPass {
+ dirs.addBundleFiles(f)
}
c.handler.handleBundles(dirs)
}
func (c *capturer) handleDir(dirname string) error {
+
files, err := c.readDir(dirname)
if err != nil {
return err
for i, fi := range files {
if !fi.IsDir() {
- tp, isContent := classifyBundledFile(fi.Name())
+ tp, isContent := classifyBundledFile(fi.RealName())
+
fileBundleTypes[i] = tp
if !isBranch {
isBranch = tp == bundleBranch
var fileInfos = make([]*fileInfo, 0, len(files))
for i, fi := range files {
+
currentType := bundleNot
if !fi.IsDir() {
if bundleType == bundleNot && currentType != bundleNot {
bundleType = currentType
}
- f, active := c.newFileInfo(fi.filename, fi.FileInfo, currentType)
+
+ f, active := c.newFileInfo(fi, currentType)
+
if !active {
continue
}
for _, fi := range fileInfos {
if fi.FileInfo().IsDir() {
// Handle potential nested bundles.
- filename := fi.Filename()
- if err := c.handleNestedDir(filename); err != nil {
+ if err := c.handleNestedDir(fi.Path()); err != nil {
return err
}
} else if bundleType == bundleNot || (!fi.isOwner() && fi.isContentFile()) {
func (c *capturer) handleNonBundle(
dirname string,
- fileInfos []fileInfoName,
+ fileInfos pathLangFileFis,
singlesOnly bool) error {
for _, fi := range fileInfos {
if fi.IsDir() {
- if err := c.handleNestedDir(fi.filename); err != nil {
+ if err := c.handleNestedDir(fi.Filename()); err != nil {
return err
}
} else {
if singlesOnly {
- f, active := c.newFileInfo(fi.filename, fi, bundleNot)
+ f, active := c.newFileInfo(fi, bundleNot)
if !active {
continue
}
c.handler.handleSingles(f)
} else {
- c.handler.handleCopyFiles(fi.filename)
+ c.handler.handleCopyFiles(fi)
}
}
}
c.handler.handleSingles(fi)
} else {
// These do not currently need any further processing.
- c.handler.handleCopyFiles(fi.Filename())
+ c.handler.handleCopyFiles(fi)
}
}
fileInfos = append(fileInfos, fis...)
}
}
- err := c.collectFiles(fi.Filename(), collector)
+ err := c.collectFiles(fi.Path(), collector)
if err != nil {
return nil, err
}
}
func (c *capturer) collectFiles(dirname string, handleFiles func(fis ...*fileInfo)) error {
+
filesInDir, err := c.readDir(dirname)
if err != nil {
return err
for _, fi := range filesInDir {
if fi.IsDir() {
- err := c.collectFiles(fi.filename, handleFiles)
+ err := c.collectFiles(fi.Filename(), handleFiles)
if err != nil {
return err
}
} else {
- f, active := c.newFileInfo(fi.filename, fi.FileInfo, bundleNot)
+ f, active := c.newFileInfo(fi, bundleNot)
if active {
handleFiles(f)
}
return nil
}
-func (c *capturer) readDir(dirname string) ([]fileInfoName, error) {
+func (c *capturer) readDir(dirname string) (pathLangFileFis, error) {
if c.sourceSpec.IgnoreFile(dirname) {
return nil, nil
}
dir, err := c.fs.Open(dirname)
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("readDir: %s", err)
}
defer dir.Close()
- names, err := dir.Readdirnames(-1)
+ fis, err := dir.Readdir(-1)
if err != nil {
return nil, err
}
- fis := make([]fileInfoName, 0, len(names))
+ pfis := make(pathLangFileFis, 0, len(fis))
- for _, name := range names {
- filename := filepath.Join(dirname, name)
- if !c.sourceSpec.IgnoreFile(filename) {
- fi, _, err := c.getRealFileInfo(filename)
+ for _, fi := range fis {
+ fip := fi.(pathLangFileFi)
+
+ if !c.sourceSpec.IgnoreFile(fip.Filename()) {
+
+ err := c.resolveRealPathIn(fip)
if err != nil {
// It may have been deleted in the meantime.
return nil, err
}
- fis = append(fis, fileInfoName{filename: filename, FileInfo: fi})
+ pfis = append(pfis, fip)
}
}
- return fis, nil
+ return pfis, nil
}
-func (c *capturer) newFileInfo(filename string, fi os.FileInfo, tp bundleDirType) (*fileInfo, bool) {
- f := newFileInfo(c.sourceSpec, c.baseDir, filename, fi, tp)
+func (c *capturer) newFileInfo(fi pathLangFileFi, tp bundleDirType) (*fileInfo, bool) {
+ f := newFileInfo(c.sourceSpec, "", "", fi, tp)
return f, !f.disabled
}
-type fileInfoName struct {
+type pathLangFile interface {
+ hugofs.LanguageAnnouncer
+ hugofs.FilePather
+}
+
+type pathLangFileFi interface {
os.FileInfo
- filename string
+ pathLangFile
}
+type pathLangFileFis []pathLangFileFi
+
type bundleDirs struct {
tp bundleDirType
// Maps languages to bundles.
b.bundles[fi.Lang()] = dir
}
- dir.resources[fi.Filename()] = fi
+ dir.resources[fi.Path()] = fi
}
func (b *bundleDirs) addBundleFiles(fi *fileInfo) {
dir := filepath.ToSlash(fi.Dir())
p := dir + fi.TranslationBaseName() + "." + fi.Ext()
for lang, bdir := range b.bundles {
- key := lang + p
+ key := path.Join(lang, p)
+
// Given mypage.de.md (German translation) and mypage.md we pick the most
- // the specific for that language.
+ // specific for that language.
if fi.Lang() == lang || !b.langOverrides[key] {
bdir.resources[key] = fi
}
return false
}
-func (c *capturer) getRealFileInfo(path string) (os.FileInfo, string, error) {
- fileInfo, err := c.lstatIfOs(path)
- realPath := path
-
+func (c *capturer) resolveRealPath(path string) (pathLangFileFi, error) {
+ fileInfo, err := c.lstatIfPossible(path)
if err != nil {
- return nil, "", err
+ return nil, err
}
+ return fileInfo, c.resolveRealPathIn(fileInfo)
+}
+
+func (c *capturer) resolveRealPathIn(fileInfo pathLangFileFi) error {
+
+ basePath := fileInfo.BaseDir()
+ path := fileInfo.Filename()
+
+ realPath := path
if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {
link, err := filepath.EvalSymlinks(path)
if err != nil {
- return nil, "", fmt.Errorf("Cannot read symbolic link %q, error was: %s", path, err)
+ return fmt.Errorf("Cannot read symbolic link %q, error was: %s", path, err)
}
- fileInfo, err = c.lstatIfOs(link)
+ // This is a file on the outside of any base fs, so we have to use the os package.
+ sfi, err := os.Stat(link)
if err != nil {
- return nil, "", fmt.Errorf("Cannot stat %q, error was: %s", link, err)
+ return fmt.Errorf("Cannot stat %q, error was: %s", link, err)
+ }
+
+ // TODO(bep) improve all of this.
+ if a, ok := fileInfo.(*hugofs.LanguageFileInfo); ok {
+ a.FileInfo = sfi
}
realPath = link
- if realPath != path && fileInfo.IsDir() && c.isSeen(realPath) {
+ if realPath != path && sfi.IsDir() && c.isSeen(realPath) {
// Avoid cyclic symlinks.
// Note that this may prevent some uses that isn't cyclic and also
// potential useful, but this implementation is both robust and simple:
// We stop at the first directory that we have seen before, e.g.
// /content/blog will only be processed once.
- return nil, realPath, errSkipCyclicDir
+ return errSkipCyclicDir
}
if c.contentChanges != nil {
// Keep track of symbolic links in watch mode.
var from, to string
- if fileInfo.IsDir() {
+ if sfi.IsDir() {
from = realPath
to = path
from = from + helpers.FilePathSeparator
}
- baseDir := c.baseDir
- if !strings.HasSuffix(baseDir, helpers.FilePathSeparator) {
- baseDir = baseDir + helpers.FilePathSeparator
+ if !strings.HasSuffix(basePath, helpers.FilePathSeparator) {
+ basePath = basePath + helpers.FilePathSeparator
}
- if strings.HasPrefix(from, baseDir) {
+ if strings.HasPrefix(from, basePath) {
// With symbolic links inside /content we need to keep
// a reference to both. This may be confusing with --navigateToChanged
// but the user has chosen this him or herself.
}
}
- return fileInfo, realPath, nil
+ return nil
}
-func (c *capturer) lstatIfOs(path string) (os.FileInfo, error) {
- return helpers.LstatIfOs(c.fs, path)
+func (c *capturer) lstatIfPossible(path string) (pathLangFileFi, error) {
+ fi, err := helpers.LstatIfPossible(c.fs, path)
+ if err != nil {
+ return nil, err
+ }
+ return fi.(pathLangFileFi), nil
}
import (
"fmt"
+ "os"
"path"
"path/filepath"
"sort"
s.dirKeys = append(s.dirKeys, keys...)
}
-func (s *storeFilenames) handleCopyFiles(names ...string) {
+func (s *storeFilenames) handleCopyFiles(files ...pathLangFile) {
s.Lock()
defer s.Unlock()
- for _, name := range names {
- s.copyNames = append(s.copyNames, filepath.ToSlash(name))
+ for _, file := range files {
+ s.copyNames = append(s.copyNames, filepath.ToSlash(file.Filename()))
}
-
}
func (s *storeFilenames) sortedStr() string {
func TestPageBundlerCaptureSymlinks(t *testing.T) {
assert := require.New(t)
- cfg, fs, workDir := newTestBundleSymbolicSources(t)
- contentDir := "base"
- sourceSpec := source.NewSourceSpec(cfg, fs)
+ ps, workDir := newTestBundleSymbolicSources(t)
+ sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs)
fileStore := &storeFilenames{}
logger := newErrorLogger()
- c := newCapturer(logger, sourceSpec, fileStore, nil, filepath.Join(workDir, contentDir))
+ c := newCapturer(logger, sourceSpec, fileStore, nil)
assert.NoError(c.capture())
/base/symbolic3/s1.png
/base/symbolic3/s2.png
`
+
got := strings.Replace(fileStore.sortedStr(), filepath.ToSlash(workDir), "", -1)
got = strings.Replace(got, "//", "/", -1)
}
}
-func TestPageBundlerCapture(t *testing.T) {
+func TestPageBundlerCaptureBasic(t *testing.T) {
t.Parallel()
assert := require.New(t)
- cfg, fs := newTestBundleSources(t)
+ fs, cfg := newTestBundleSources(t)
+ assert.NoError(loadDefaultSettingsFor(cfg))
+ assert.NoError(loadLanguageSettings(cfg, nil))
+ ps, err := helpers.NewPathSpec(fs, cfg)
+ assert.NoError(err)
- sourceSpec := source.NewSourceSpec(cfg, fs)
+ sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs)
fileStore := &storeFilenames{}
- c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil, filepath.FromSlash("/work/base"))
+ c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil)
assert.NoError(c.capture())
+ printFs(fs.Source, "", os.Stdout)
+
expected := `
F:
/work/base/_1.md
t.Parallel()
assert := require.New(t)
- cfg, fs := newTestBundleSourcesMultilingual(t)
- sourceSpec := source.NewSourceSpec(cfg, fs)
+ fs, cfg := newTestBundleSourcesMultilingual(t)
+ assert.NoError(loadDefaultSettingsFor(cfg))
+ assert.NoError(loadLanguageSettings(cfg, nil))
+
+ ps, err := helpers.NewPathSpec(fs, cfg)
+ assert.NoError(err)
+
+ sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs)
fileStore := &storeFilenames{}
- c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil, filepath.FromSlash("/work/base"))
+ c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil)
assert.NoError(c.capture())
if expected != got {
diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got))
t.Log(got)
- t.Fatalf("Failed:\n%s", diff)
+ t.Fatalf("Failed:\n%s", strings.Join(diff, "\n"))
}
}
type noOpFileStore int
-func (noOpFileStore) handleSingles(fis ...*fileInfo) {}
-func (noOpFileStore) handleBundles(b *bundleDirs) {}
-func (noOpFileStore) handleCopyFiles(names ...string) {}
+func (noOpFileStore) handleSingles(fis ...*fileInfo) {}
+func (noOpFileStore) handleBundles(b *bundleDirs) {}
+func (noOpFileStore) handleCopyFiles(files ...pathLangFile) {}
func BenchmarkPageBundlerCapture(b *testing.B) {
capturers := make([]*capturer, b.N)
for i := 0; i < b.N; i++ {
cfg, fs := newTestCfg()
- sourceSpec := source.NewSourceSpec(cfg, fs)
+ ps, _ := helpers.NewPathSpec(fs, cfg)
+ sourceSpec := source.NewSourceSpec(ps, fs.Source)
base := fmt.Sprintf("base%d", i)
for j := 1; j <= 5; j++ {
bundle *bundleDir
- // The source baseDir, e.g. "/myproject/content/"
- baseDir string
-
source *fileInfo
// Relative path to the target.
return c.target
}
- return strings.TrimPrefix(c.source.Filename(), c.baseDir)
+ return c.source.Filename()
}
func (c *handlerContext) file() *fileInfo {
resource, err := c.s.resourceSpec.NewResourceFromFilename(
ctx.parentPage.subResourceTargetPathFactory,
- c.s.absPublishDir(),
ctx.source.Filename(), ctx.target)
return handlerResult{err: err, handled: true, resource: resource}
func (c *contentHandlers) copyFile() contentHandler {
return func(ctx *handlerContext) handlerResult {
- f, err := c.s.Fs.Source.Open(ctx.source.Filename())
+ f, err := c.s.BaseFs.ContentFs.Open(ctx.source.Filename())
if err != nil {
+ err := fmt.Errorf("failed to open file in copyFile: %s", err)
return handlerResult{err: err}
}
"strings"
"testing"
+ "github.com/gohugoio/hugo/helpers"
+
"io"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
)
-func TestPageBundlerSite(t *testing.T) {
+func TestPageBundlerSiteRegular(t *testing.T) {
t.Parallel()
for _, ugly := range []bool{false, true} {
func(t *testing.T) {
assert := require.New(t)
- cfg, fs := newTestBundleSources(t)
+ fs, cfg := newTestBundleSources(t)
+ assert.NoError(loadDefaultSettingsFor(cfg))
+ assert.NoError(loadLanguageSettings(cfg, nil))
cfg.Set("permalinks", map[string]string{
"a": ":sections/:filename",
assert.Equal(filepath.FromSlash("/work/base/b/my-bundle/c/logo.png"), image.(resource.Source).AbsSourceFilename())
assert.Equal("https://example.com/2017/pageslug/c/logo.png", image.Permalink())
+
+ printFs(th.Fs.Destination, "", os.Stdout)
th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content")
th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/c/logo.png"), "content")
func(t *testing.T) {
assert := require.New(t)
- cfg, fs := newTestBundleSourcesMultilingual(t)
-
+ fs, cfg := newTestBundleSourcesMultilingual(t)
cfg.Set("uglyURLs", ugly)
assert.NoError(loadDefaultSettingsFor(cfg))
t.Parallel()
assert := require.New(t)
- cfg, _ := newTestBundleSourcesMultilingual(t)
+ _, cfg := newTestBundleSourcesMultilingual(t)
cfg.Set("disableLanguages", []string{"en"})
t.Parallel()
assert := require.New(t)
- cfg, fs := newTestBundleSourcesMultilingual(t)
+ fs, cfg := newTestBundleSourcesMultilingual(t)
cfg.Set("disableLanguages", []string{"nn"})
assert.NoError(loadDefaultSettingsFor(cfg))
+ assert.NoError(loadLanguageSettings(cfg, nil))
+
sites, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg})
assert.NoError(err)
assert.Equal(1, len(sites.Sites))
func TestPageBundlerSiteWitSymbolicLinksInContent(t *testing.T) {
assert := require.New(t)
- cfg, fs, workDir := newTestBundleSymbolicSources(t)
+ ps, workDir := newTestBundleSymbolicSources(t)
+ cfg := ps.Cfg
+ fs := ps.Fs
s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: newErrorLogger()}, BuildCfg{})
}
-func newTestBundleSources(t *testing.T) (*viper.Viper, *hugofs.Fs) {
+func newTestBundleSources(t *testing.T) (*hugofs.Fs, *viper.Viper) {
cfg, fs := newTestCfg()
assert := require.New(t)
src.Close()
assert.NoError(err)
- return cfg, fs
+ return fs, cfg
+
}
-func newTestBundleSourcesMultilingual(t *testing.T) (*viper.Viper, *hugofs.Fs) {
+func newTestBundleSourcesMultilingual(t *testing.T) (*hugofs.Fs, *viper.Viper) {
cfg, fs := newTestCfg()
workDir := "/work"
writeSource(t, fs, filepath.Join(workDir, "base", "bf", "my-bf-bundle", "index.nn.md"), pageContent)
writeSource(t, fs, filepath.Join(workDir, "base", "bf", "my-bf-bundle", "page.md"), pageContent)
- return cfg, fs
+ return fs, cfg
}
-func newTestBundleSymbolicSources(t *testing.T) (*viper.Viper, *hugofs.Fs, string) {
+func newTestBundleSymbolicSources(t *testing.T) (*helpers.PathSpec, string) {
assert := require.New(t)
// We need to use the OS fs for this.
cfg := viper.New()
cfg.Set("contentDir", contentDir)
cfg.Set("baseURL", "https://example.com")
+ if err := loadLanguageSettings(cfg, nil); err != nil {
+ t.Fatal(err)
+ }
+
layout := `{{ .Title }}|{{ .Content }}`
pageContent := `---
slug: %s
os.Chdir(workDir)
assert.NoError(err)
- return cfg, fs, workDir
+ ps, _ := helpers.NewPathSpec(fs, cfg)
+
+ return ps, workDir
}
c.rawAllPages = append(c.rawAllPages, page)
}
-func (c *PageCollections) removePageByPath(path string) {
- if i := c.rawAllPages.findPagePosByFilePath(path); i >= 0 {
+func (c *PageCollections) removePageFilename(filename string) {
+ if i := c.rawAllPages.findPagePosByFilename(filename); i >= 0 {
c.clearResourceCacheForPage(c.rawAllPages[i])
c.rawAllPages = append(c.rawAllPages[:i], c.rawAllPages[i+1:]...)
}
if len(page.Resources) > 0 {
first := page.Resources[0]
dir := path.Dir(first.RelPermalink())
+ dir = strings.TrimPrefix(dir, page.LanguagePrefix())
// This is done to keep the memory usage in check when doing live reloads.
page.s.resourceSpec.DeleteCacheByPrefix(dir)
}
}
}
-func TestPageKind(t *testing.T) {
- t.Parallel()
- const sep = helpers.FilePathSeparator
- var tests = []struct {
- file string
- kind string
- }{
- {"_index.md", KindHome},
- {"about.md", KindPage},
- {"sectionA" + sep + "_index.md", KindSection},
- {"sectionA" + sep + "about.md", KindPage},
- {"categories" + sep + "_index.md", KindTaxonomyTerm},
- {"categories" + sep + "categoryA" + sep + "_index.md", KindTaxonomy},
- {"tags" + sep + "_index.md", KindTaxonomyTerm},
- {"tags" + sep + "tagA" + sep + "_index.md", KindTaxonomy},
-
- // nn is configured as a language
- {"_index.nn.md", KindHome},
- {"about.nn.md", KindPage},
- {"sectionA" + sep + "_index.nn.md", KindSection},
- {"sectionA" + sep + "about.nn.md", KindPage},
-
- // should NOT be categorized as KindHome
- {"_indexNOT.md", KindPage},
-
- // To be consistent with FileInfo.TranslationBaseName(),
- // language codes not explicitly configured for the site
- // are not treated as such. "fr" is not configured as
- // a language in the test site, so ALL of the
- // following should be KindPage
- {"_index.fr.md", KindPage}, //not KindHome
- {"about.fr.md", KindPage},
- {"sectionA" + sep + "_index.fr.md", KindPage}, // KindSection
- {"sectionA" + sep + "about.fr.md", KindPage},
- }
-
- for _, test := range tests {
- s := newTestSite(t, "languages.nn.languageName", "Nynorsk")
- taxonomies := make(map[string]string)
- taxonomies["tag"] = "tags"
- taxonomies["category"] = "categories"
- s.Taxonomies = make(TaxonomyList)
- for _, plural := range taxonomies {
- s.Taxonomies[plural] = make(Taxonomy)
- }
-
- p, _ := s.NewPage(test.file)
- p.setValuesForKind(s)
- if p.Kind != test.kind {
- t.Errorf("for %s expected p.Kind == %s, got %s", test.file, test.kind, p.Kind)
- }
- }
-}
-
func TestDegenerateInvalidFrontMatterShortDelim(t *testing.T) {
t.Parallel()
var tests = []struct {
// GC requires a build first.
func (h *HugoSites) GC() (int, error) {
s := h.Sites[0]
- imageCacheDir := s.resourceSpec.AbsGenImagePath
+ fs := h.PathSpec.BaseFs.ResourcesFs
+
+ imageCacheDir := s.resourceSpec.GenImagePath
if len(imageCacheDir) < 10 {
panic("invalid image cache")
}
counter := 0
- err := afero.Walk(s.Fs.Source, imageCacheDir, func(path string, info os.FileInfo, err error) error {
+ err := afero.Walk(fs, imageCacheDir, func(path string, info os.FileInfo, err error) error {
if info == nil {
return nil
}
}
if info.IsDir() {
- f, err := s.Fs.Source.Open(path)
+ f, err := fs.Open(path)
if err != nil {
return nil
}
inUse := isInUse(path)
if !inUse {
- err := s.Fs.Source.Remove(path)
+ err := fs.Remove(path)
if err != nil && !os.IsNotExist(err) {
s.Log.ERROR.Printf("Failed to remove %q: %s", path, err)
} else {
}
}
if removed && isContentFile(ev.Name) {
- path, _ := helpers.GetRelativePath(ev.Name, s.getContentDir(ev.Name))
-
- h.removePageByPath(path)
+ h.removePageByFilename(ev.Name)
}
sourceReallyChanged = append(sourceReallyChanged, ev)
func (s *Site) readData(f source.ReadableFile) (interface{}, error) {
file, err := f.Open()
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("readData: failed to open data file: %s", err)
}
defer file.Close()
content := helpers.ReaderToBytes(file)
}
}
-func (c *contentCaptureResultHandler) handleCopyFiles(filenames ...string) {
+func (c *contentCaptureResultHandler) handleCopyFiles(files ...pathLangFile) {
for _, proc := range c.contentProcessors {
- proc.processAssets(filenames)
+ proc.processAssets(files)
}
}
ctx := context.Background()
g, ctx := errgroup.WithContext(ctx)
- sourceSpec := source.NewSourceSpec(s.owner.Cfg, s.Fs)
- baseDir := s.absContentDir()
defaultContentLanguage := s.SourceSpec.DefaultContentLanguage
contentProcessors := make(map[string]*siteContentProcessor)
var defaultContentProcessor *siteContentProcessor
sites := s.owner.langSite()
for k, v := range sites {
- proc := newSiteContentProcessor(ctx, baseDir, len(filenames) > 0, v)
+ if v.Language.Disabled {
+ continue
+ }
+ proc := newSiteContentProcessor(ctx, len(filenames) > 0, v)
contentProcessors[k] = proc
if k == defaultContentLanguage {
defaultContentProcessor = proc
mainHandler := &contentCaptureResultHandler{contentProcessors: contentProcessors, defaultContentProcessor: defaultContentProcessor}
+ sourceSpec := source.NewSourceSpec(s.PathSpec, s.BaseFs.ContentFs)
+
if s.running() {
// Need to track changes.
bundleMap = s.owner.ContentChanges
handler = mainHandler
}
- c := newCapturer(s.Log, sourceSpec, handler, bundleMap, baseDir, filenames...)
+ c := newCapturer(s.Log, sourceSpec, handler, bundleMap, filenames...)
err1 := c.capture()
notUgly := s.getPage(KindPage, "sect1/p1.md")
assert.NotNil(notUgly)
+ assert.Equal("sect1", notUgly.Section())
assert.Equal("/sect1/p1/", notUgly.RelPermalink())
ugly := s.getPage(KindPage, "sect2/p2.md")
assert.NotNil(ugly)
+ assert.Equal("sect2", ugly.Section())
assert.Equal("/sect2/p2.html", ugly.RelPermalink())
}
writeSource(t, fs, "content/p3.md", fmt.Sprintf(pageTemplate, "t2/c12", "- tag2", "- cat2", "- o1", "- pl1"))
writeSource(t, fs, "content/p4.md", fmt.Sprintf(pageTemplate, "Hello World", "", "", "- \"Hello Hugo world\"", "- pl1"))
- writeNewContentFile(t, fs, "Category Terms", "2017-01-01", "content/categories/_index.md", 10)
- writeNewContentFile(t, fs, "Tag1 List", "2017-01-01", "content/tags/Tag1/_index.md", 10)
+ writeNewContentFile(t, fs.Source, "Category Terms", "2017-01-01", "content/categories/_index.md", 10)
+ writeNewContentFile(t, fs.Source, "Tag1 List", "2017-01-01", "content/tags/Tag1/_index.md", 10)
err := h.Build(BuildCfg{})
// Default toml
configFormat string
+ // Default is empty.
+ // TODO(bep) revisit this and consider always setting it to something.
+ // Consider this in relation to using the BaseFs.PublishFs to all publishing.
+ workingDir string
+
// Base data/content
contentFilePairs []string
templateFilePairs []string
return s
}
+func (s *sitesBuilder) WithWorkingDir(dir string) *sitesBuilder {
+ s.workingDir = dir
+ return s
+}
+
func (s *sitesBuilder) WithConfigTemplate(data interface{}, format, configTemplate string) *sitesBuilder {
if format == "" {
format = "toml"
}
for i := 0; i < len(filenameContent); i += 2 {
filename, content := filenameContent[i], filenameContent[i+1]
- writeSource(s.T, s.Fs, filepath.Join(folder, filename), content)
+ target := folder
+ // TODO(bep) clean up this magic.
+ if strings.HasPrefix(filename, folder) {
+ target = ""
+ }
+
+ if s.workingDir != "" {
+ target = filepath.Join(s.workingDir, target)
+ }
+
+ writeSource(s.T, s.Fs, filepath.Join(target, filename), content)
}
return s
}
v := viper.New()
// Easier to reason about in tests.
v.Set("disablePathToLower", true)
+ v.Set("contentDir", "content")
fs := hugofs.NewDefault(v)
ps, _ := helpers.NewPathSpec(fs, v)
return ps
var actual, expected string
v := viper.New()
v.SetDefault("defaultContentLanguage", "en")
+ v.Set("contentDir", "content")
// Test without and with placeholders
for _, enablePlaceholders := range []bool{false, true} {
// Update updates the i18n func in the provided Deps.
func (tp *TranslationProvider) Update(d *deps.Deps) error {
dir := d.PathSpec.AbsPathify(d.Cfg.GetString("i18nDir"))
- sp := source.NewSourceSpec(d.Cfg, d.Fs)
+ sp := source.NewSourceSpec(d.PathSpec, d.Fs.Source)
sources := []source.Input{sp.NewFilesystem(dir)}
themeI18nDir, err := d.PathSpec.GetThemeI18nDirPath()
config image.Config
)
- f, err = i.spec.Fs.Source.Open(i.AbsSourceFilename())
+ f, err = i.sourceFs().Open(i.AbsSourceFilename())
if err != nil {
return
}
i.config = config
})
- return err
+ if err != nil {
+ return fmt.Errorf("failed to load image config: %s", err)
+ }
+
+ return nil
}
func (i *Image) decodeSource() (image.Image, error) {
- file, err := i.spec.Fs.Source.Open(i.AbsSourceFilename())
+ file, err := i.sourceFs().Open(i.AbsSourceFilename())
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("failed to open image for decode: %s", err)
}
defer file.Close()
img, _, err := image.Decode(file)
func (i *Image) copyToDestination(src string) error {
var res error
i.copyToDestinationInit.Do(func() {
- target := filepath.Join(i.absPublishDir, i.target())
+ target := i.target()
// Fast path:
// This is a processed version of the original.
// If it exists on destination with the same filename and file size, it is
// the same file, so no need to transfer it again.
- if fi, err := i.spec.Fs.Destination.Stat(target); err == nil && fi.Size() == i.osFileInfo.Size() {
+ if fi, err := i.spec.BaseFs.PublishFs.Stat(target); err == nil && fi.Size() == i.osFileInfo.Size() {
return
}
- in, err := i.spec.Fs.Source.Open(src)
+ in, err := i.sourceFs().Open(src)
if err != nil {
res = err
return
}
defer in.Close()
- out, err := i.spec.Fs.Destination.Create(target)
+ out, err := i.spec.BaseFs.PublishFs.Create(target)
if err != nil && os.IsNotExist(err) {
// When called from shortcodes, the target directory may not exist yet.
// See https://github.com/gohugoio/hugo/issues/4202
- if err = i.spec.Fs.Destination.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil {
+ if err = i.spec.BaseFs.PublishFs.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil {
res = err
return
}
- out, err = i.spec.Fs.Destination.Create(target)
+ out, err = i.spec.BaseFs.PublishFs.Create(target)
if err != nil {
res = err
return
}
})
- return res
+ if res != nil {
+ return fmt.Errorf("failed to copy image to destination: %s", res)
+ }
+ return nil
}
func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resourceCacheFilename, filename string) error {
- target := filepath.Join(i.absPublishDir, filename)
+ target := filepath.Clean(filename)
- file1, err := i.spec.Fs.Destination.Create(target)
+ file1, err := i.spec.BaseFs.PublishFs.Create(target)
if err != nil && os.IsNotExist(err) {
// When called from shortcodes, the target directory may not exist yet.
// See https://github.com/gohugoio/hugo/issues/4202
- if err = i.spec.Fs.Destination.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil {
+ if err = i.spec.BaseFs.PublishFs.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil {
return err
}
- file1, err = i.spec.Fs.Destination.Create(target)
+ file1, err = i.spec.BaseFs.PublishFs.Create(target)
if err != nil {
return err
}
if resourceCacheFilename != "" {
// Also save it to the image resource cache for later reuse.
- if err = i.spec.Fs.Source.MkdirAll(filepath.Dir(resourceCacheFilename), os.FileMode(0755)); err != nil {
+ if err = i.spec.BaseFs.ResourcesFs.MkdirAll(filepath.Dir(resourceCacheFilename), os.FileMode(0755)); err != nil {
return err
}
- file2, err := i.spec.Fs.Source.Create(resourceCacheFilename)
+ file2, err := i.spec.BaseFs.ResourcesFs.Create(resourceCacheFilename)
if err != nil {
return err
}
)
type imageCache struct {
- absPublishDir string
- absCacheDir string
- pathSpec *helpers.PathSpec
- mu sync.RWMutex
+ cacheDir string
+ pathSpec *helpers.PathSpec
+ mu sync.RWMutex
store map[string]*Image
}
parent.createMu.Lock()
defer parent.createMu.Unlock()
- cacheFilename := filepath.Join(c.absCacheDir, key)
+ cacheFilename := filepath.Join(c.cacheDir, key)
// The definition of this counter is not that we have processed that amount
// (e.g. resized etc.), it can be fetched from file cache,
// but the count of processed image variations for this site.
c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages)
- exists, err := helpers.Exists(cacheFilename, c.pathSpec.Fs.Source)
+ exists, err := helpers.Exists(cacheFilename, c.pathSpec.BaseFs.ResourcesFs)
if err != nil {
return nil, err
}
if exists {
img = parent.clone()
img.relTargetPath.file = relTarget.file
- img.absSourceFilename = cacheFilename
+ img.sourceFilename = cacheFilename
+ // We have to look resources file system for this.
+ img.overriddenSourceFs = img.spec.BaseFs.ResourcesFs
} else {
img, err = create(cacheFilename)
if err != nil {
}
-func newImageCache(ps *helpers.PathSpec, absCacheDir, absPublishDir string) *imageCache {
- return &imageCache{pathSpec: ps, store: make(map[string]*Image), absCacheDir: absCacheDir, absPublishDir: absPublishDir}
+func newImageCache(ps *helpers.PathSpec, cacheDir string) *imageCache {
+ return &imageCache{pathSpec: ps, store: make(map[string]*Image), cacheDir: cacheDir}
}
func timeTrack(start time.Time, name string) {
import (
"fmt"
"math/rand"
+ "os"
"path/filepath"
"strconv"
"testing"
}
}
-func TestImageTransform(t *testing.T) {
+func TestImageTransformBasic(t *testing.T) {
assert := require.New(t)
image := fetchSunset(assert)
+ printFs(image.sourceFs(), "", os.Stdout)
+
assert.Equal("/a/sunset.jpg", image.RelPermalink())
assert.Equal("image", image.ResourceType())
assert.NoError(err)
assert.Equal(320, resized0x.Width())
assert.Equal(200, resized0x.Height())
- assertFileCache(assert, image.spec.Fs, resized0x.RelPermalink(), 320, 200)
+ assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resized0x.RelPermalink(), 320, 200)
resizedx0, err := image.Resize("200x")
assert.NoError(err)
assert.Equal(200, resizedx0.Width())
assert.Equal(125, resizedx0.Height())
- assertFileCache(assert, image.spec.Fs, resizedx0.RelPermalink(), 200, 125)
+ assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedx0.RelPermalink(), 200, 125)
resizedAndRotated, err := image.Resize("x200 r90")
assert.NoError(err)
assert.Equal(125, resizedAndRotated.Width())
assert.Equal(200, resizedAndRotated.Height())
- assertFileCache(assert, image.spec.Fs, resizedAndRotated.RelPermalink(), 125, 200)
+ assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedAndRotated.RelPermalink(), 125, 200)
assert.Equal("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_300x200_resize_q68_linear.jpg", resized.RelPermalink())
assert.Equal(300, resized.Width())
assert.Equal("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_bottomleft.jpg", filled.RelPermalink())
assert.Equal(200, filled.Width())
assert.Equal(100, filled.Height())
- assertFileCache(assert, image.spec.Fs, filled.RelPermalink(), 200, 100)
+ assertFileCache(assert, image.spec.BaseFs.ResourcesFs, filled.RelPermalink(), 200, 100)
smart, err := image.Fill("200x100 smart")
assert.NoError(err)
assert.Equal(fmt.Sprintf("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_smart%d.jpg", smartCropVersionNumber), smart.RelPermalink())
assert.Equal(200, smart.Width())
assert.Equal(100, smart.Height())
- assertFileCache(assert, image.spec.Fs, smart.RelPermalink(), 200, 100)
+ assertFileCache(assert, image.spec.BaseFs.ResourcesFs, smart.RelPermalink(), 200, 100)
// Check cache
filledAgain, err := image.Fill("200x100 bottomLeft")
assert.NoError(err)
assert.True(filled == filledAgain)
- assertFileCache(assert, image.spec.Fs, filledAgain.RelPermalink(), 200, 100)
+ assertFileCache(assert, image.spec.BaseFs.ResourcesFs, filledAgain.RelPermalink(), 200, 100)
}
assert.Equal("/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png", resized.RelPermalink())
assert.Equal(101, resized.Width())
- assertFileCache(assert, image.spec.Fs, resized.RelPermalink(), 101, 101)
- publishedImageFilename := filepath.Join("/public", resized.RelPermalink())
- assertImageFile(assert, image.spec.Fs, publishedImageFilename, 101, 101)
- assert.NoError(image.spec.Fs.Destination.Remove(publishedImageFilename))
+ assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resized.RelPermalink(), 101, 101)
+ publishedImageFilename := filepath.Clean(resized.RelPermalink())
+ assertImageFile(assert, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101)
+ assert.NoError(image.spec.BaseFs.PublishFs.Remove(publishedImageFilename))
// Cleare mem cache to simulate reading from the file cache.
resized.spec.imageCache.clear()
assert.NoError(err)
assert.Equal("/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png", resizedAgain.RelPermalink())
assert.Equal(101, resizedAgain.Width())
- assertFileCache(assert, image.spec.Fs, resizedAgain.RelPermalink(), 101, 101)
- assertImageFile(assert, image.spec.Fs, publishedImageFilename, 101, 101)
+ assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedAgain.RelPermalink(), 101, 101)
+ assertImageFile(assert, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101)
}
"strings"
"sync"
+ "github.com/spf13/afero"
+
"github.com/spf13/cast"
"github.com/gobwas/glob"
type Spec struct {
*helpers.PathSpec
+
mimeTypes media.Types
// Holds default filter settings etc.
imageCache *imageCache
- AbsGenImagePath string
+ GenImagePath string
}
func NewSpec(s *helpers.PathSpec, mimeTypes media.Types) (*Spec, error) {
}
s.GetLayoutDirPath()
- genImagePath := s.AbsPathify(filepath.Join(s.Cfg.GetString("resourceDir"), "_gen", "images"))
+ genImagePath := filepath.FromSlash("_gen/images")
- return &Spec{AbsGenImagePath: genImagePath, PathSpec: s, imaging: &imaging, mimeTypes: mimeTypes, imageCache: newImageCache(
- s,
- // We're going to write a cache pruning routine later, so make it extremely
- // unlikely that the user shoots him or herself in the foot
- // and this is set to a value that represents data he/she
- // cares about. This should be set in stone once released.
- genImagePath,
- s.AbsPathify(s.Cfg.GetString("publishDir")))}, nil
+ return &Spec{PathSpec: s,
+ GenImagePath: genImagePath,
+ imaging: &imaging, mimeTypes: mimeTypes, imageCache: newImageCache(
+ s,
+ // We're going to write a cache pruning routine later, so make it extremely
+ // unlikely that the user shoots him or herself in the foot
+ // and this is set to a value that represents data he/she
+ // cares about. This should be set in stone once released.
+ genImagePath,
+ )}, nil
}
func (r *Spec) NewResourceFromFile(
targetPathBuilder func(base string) string,
- absPublishDir string,
file source.File, relTargetFilename string) (Resource, error) {
- return r.newResource(targetPathBuilder, absPublishDir, file.Filename(), file.FileInfo(), relTargetFilename)
+ return r.newResource(targetPathBuilder, file.Filename(), file.FileInfo(), relTargetFilename)
}
func (r *Spec) NewResourceFromFilename(
targetPathBuilder func(base string) string,
- absPublishDir,
absSourceFilename, relTargetFilename string) (Resource, error) {
- fi, err := r.Fs.Source.Stat(absSourceFilename)
+ fi, err := r.sourceFs().Stat(absSourceFilename)
if err != nil {
return nil, err
}
- return r.newResource(targetPathBuilder, absPublishDir, absSourceFilename, fi, relTargetFilename)
+ return r.newResource(targetPathBuilder, absSourceFilename, fi, relTargetFilename)
+}
+
+func (r *Spec) sourceFs() afero.Fs {
+ return r.PathSpec.BaseFs.ContentFs
}
func (r *Spec) newResource(
targetPathBuilder func(base string) string,
- absPublishDir,
absSourceFilename string, fi os.FileInfo, relTargetFilename string) (Resource, error) {
var mimeType string
}
}
- gr := r.newGenericResource(targetPathBuilder, fi, absPublishDir, absSourceFilename, relTargetFilename, mimeType)
+ gr := r.newGenericResource(targetPathBuilder, fi, absSourceFilename, relTargetFilename, mimeType)
if mimeType == "image" {
ext := strings.ToLower(helpers.Ext(absSourceFilename))
return gr, nil
}
- f, err := r.Fs.Source.Open(absSourceFilename)
+ f, err := gr.sourceFs().Open(absSourceFilename)
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("failed to open image source file: %s", err)
}
defer f.Close()
params map[string]interface{}
// Absolute filename to the source, including any content folder path.
- absSourceFilename string
- absPublishDir string
- resourceType string
- osFileInfo os.FileInfo
+ // Note that this is absolute in relation to the filesystem it is stored in.
+ // It can be a base path filesystem, and then this filename will not match
+ // the path to the file on the real filesystem.
+ sourceFilename string
+
+ // This may be set to tell us to look in another filesystem for this resource.
+ // We, by default, use the sourceFs filesystem in the spec below.
+ overriddenSourceFs afero.Fs
+
+ spec *Spec
+
+ resourceType string
+ osFileInfo os.FileInfo
- spec *Spec
targetPathBuilder func(rel string) string
}
+func (l *genericResource) sourceFs() afero.Fs {
+ if l.overriddenSourceFs != nil {
+ return l.overriddenSourceFs
+ }
+ return l.spec.sourceFs()
+}
+
func (l *genericResource) Permalink() string {
return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetPath.path(), false), l.spec.BaseURL.String())
}
}
func (l *genericResource) AbsSourceFilename() string {
- return l.absSourceFilename
+ return l.sourceFilename
}
func (l *genericResource) Publish() error {
- f, err := l.spec.Fs.Source.Open(l.AbsSourceFilename())
+ f, err := l.sourceFs().Open(l.AbsSourceFilename())
if err != nil {
return err
}
defer f.Close()
-
- target := filepath.Join(l.absPublishDir, l.target())
-
- return helpers.WriteToDisk(target, f, l.spec.Fs.Destination)
+ return helpers.WriteToDisk(l.target(), f, l.spec.BaseFs.PublishFs)
}
const counterPlaceHolder = ":counter"
func (r *Spec) newGenericResource(
targetPathBuilder func(base string) string,
osFileInfo os.FileInfo,
- absPublishDir,
- absSourceFilename,
+ sourceFilename,
baseFilename,
resourceType string) *genericResource {
return &genericResource{
targetPathBuilder: targetPathBuilder,
osFileInfo: osFileInfo,
- absPublishDir: absPublishDir,
- absSourceFilename: absSourceFilename,
+ sourceFilename: sourceFilename,
relTargetPath: dirFile{dir: fpath, file: fname},
resourceType: resourceType,
spec: r,
assert := require.New(t)
spec := newTestResourceSpec(assert)
- r := spec.newGenericResource(nil, nil, "/public", "/a/foo.css", "foo.css", "css")
+ r := spec.newGenericResource(nil, nil, "/a/foo.css", "foo.css", "css")
assert.Equal("https://example.com/foo.css", r.Permalink())
assert.Equal("/foo.css", r.RelPermalink())
factory := func(s string) string {
return path.Join("/foo", s)
}
- r := spec.newGenericResource(factory, nil, "/public", "/a/foo.css", "foo.css", "css")
+ r := spec.newGenericResource(factory, nil, "/a/foo.css", "foo.css", "css")
assert.Equal("https://example.com/foo/foo.css", r.Permalink())
assert.Equal("/foo/foo.css", r.RelPermalink())
assert := require.New(t)
spec := newTestResourceSpec(assert)
- writeSource(t, spec.Fs, "/project/a/b/logo.png", "image")
- writeSource(t, spec.Fs, "/root/a/b/data.json", "json")
+ writeSource(t, spec.Fs, "content/a/b/logo.png", "image")
+ writeSource(t, spec.Fs, "content/a/b/data.json", "json")
- r, err := spec.NewResourceFromFilename(nil, "/public",
- filepath.FromSlash("/project/a/b/logo.png"), filepath.FromSlash("a/b/logo.png"))
+ r, err := spec.NewResourceFromFilename(nil,
+ filepath.FromSlash("a/b/logo.png"), filepath.FromSlash("a/b/logo.png"))
assert.NoError(err)
assert.NotNil(r)
assert.Equal("/a/b/logo.png", r.RelPermalink())
assert.Equal("https://example.com/a/b/logo.png", r.Permalink())
- r, err = spec.NewResourceFromFilename(nil, "/public", "/root/a/b/data.json", "a/b/data.json")
+ r, err = spec.NewResourceFromFilename(nil, "a/b/data.json", "a/b/data.json")
assert.NoError(err)
assert.NotNil(r)
assert := require.New(t)
spec := newTestResourceSpecForBaseURL(assert, "https://example.com/docs")
- writeSource(t, spec.Fs, "/project/a/b/logo.png", "image")
+ writeSource(t, spec.Fs, "content/a/b/logo.png", "image")
- r, err := spec.NewResourceFromFilename(nil, "/public",
- filepath.FromSlash("/project/a/b/logo.png"), filepath.FromSlash("a/b/logo.png"))
+ r, err := spec.NewResourceFromFilename(nil,
+ filepath.FromSlash("a/b/logo.png"), filepath.FromSlash("a/b/logo.png"))
assert.NoError(err)
assert.NotNil(r)
assert := require.New(t)
spec := newTestResourceSpec(assert)
resources := Resources{
- spec.newGenericResource(nil, nil, "/public", "/a/foo1.css", "foo1.css", "css"),
- spec.newGenericResource(nil, nil, "/public", "/a/logo.png", "logo.css", "image"),
- spec.newGenericResource(nil, nil, "/public", "/a/foo2.css", "foo2.css", "css"),
- spec.newGenericResource(nil, nil, "/public", "/a/foo3.css", "foo3.css", "css")}
+ spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"),
+ spec.newGenericResource(nil, nil, "/a/logo.png", "logo.css", "image"),
+ spec.newGenericResource(nil, nil, "/a/foo2.css", "foo2.css", "css"),
+ spec.newGenericResource(nil, nil, "/a/foo3.css", "foo3.css", "css")}
assert.Len(resources.ByType("css"), 3)
assert.Len(resources.ByType("image"), 1)
assert := require.New(t)
spec := newTestResourceSpec(assert)
resources := Resources{
- spec.newGenericResource(nil, nil, "/public", "/a/foo1.css", "foo1.css", "css"),
- spec.newGenericResource(nil, nil, "/public", "/a/logo1.png", "logo1.png", "image"),
- spec.newGenericResource(nil, nil, "/public", "/b/Logo2.png", "Logo2.png", "image"),
- spec.newGenericResource(nil, nil, "/public", "/b/foo2.css", "foo2.css", "css"),
- spec.newGenericResource(nil, nil, "/public", "/b/foo3.css", "foo3.css", "css")}
+ spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"),
+ spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image"),
+ spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image"),
+ spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css"),
+ spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css")}
assert.Nil(resources.GetByPrefix("asdf"))
assert.Equal("/logo1.png", resources.GetByPrefix("logo").RelPermalink())
assert := require.New(t)
spec := newTestResourceSpec(assert)
resources := Resources{
- spec.newGenericResource(nil, nil, "/public", "/a/foo1.css", "foo1.css", "css"),
- spec.newGenericResource(nil, nil, "/public", "/a/logo1.png", "logo1.png", "image"),
- spec.newGenericResource(nil, nil, "/public", "/b/Logo2.png", "Logo2.png", "image"),
- spec.newGenericResource(nil, nil, "/public", "/b/foo2.css", "foo2.css", "css"),
- spec.newGenericResource(nil, nil, "/public", "/b/foo3.css", "foo3.css", "css"),
- spec.newGenericResource(nil, nil, "/public", "/b/c/foo4.css", "c/foo4.css", "css"),
- spec.newGenericResource(nil, nil, "/public", "/b/c/foo5.css", "c/foo5.css", "css"),
- spec.newGenericResource(nil, nil, "/public", "/b/c/d/foo6.css", "c/d/foo6.css", "css"),
+ spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"),
+ spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image"),
+ spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image"),
+ spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css"),
+ spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css"),
+ spec.newGenericResource(nil, nil, "/b/c/foo4.css", "c/foo4.css", "css"),
+ spec.newGenericResource(nil, nil, "/b/c/foo5.css", "c/foo5.css", "css"),
+ spec.newGenericResource(nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", "css"),
}
assert.Equal("/logo1.png", resources.GetMatch("logo*").RelPermalink())
}},
} {
- foo2 = spec.newGenericResource(nil, nil, "/public", "/b/foo2.css", "foo2.css", "css")
- logo2 = spec.newGenericResource(nil, nil, "/public", "/b/Logo2.png", "Logo2.png", "image")
- foo1 = spec.newGenericResource(nil, nil, "/public", "/a/foo1.css", "foo1.css", "css")
- logo1 = spec.newGenericResource(nil, nil, "/public", "/a/logo1.png", "logo1.png", "image")
- foo3 = spec.newGenericResource(nil, nil, "/public", "/b/foo3.css", "foo3.css", "css")
- logo3 = spec.newGenericResource(nil, nil, "/public", "/b/logo3.png", "logo3.png", "image")
+ foo2 = spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css")
+ logo2 = spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image")
+ foo1 = spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css")
+ logo1 = spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image")
+ foo3 = spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css")
+ logo3 = spec.newGenericResource(nil, nil, "/b/logo3.png", "logo3.png", "image")
resources = Resources{
foo2,
a100 := strings.Repeat("a", 100)
pattern := "a*a*a*a*a*a*a*a*b"
- resources := Resources{spec.newGenericResource(nil, nil, "/public", "/a/"+a100, a100, "css")}
+ resources := Resources{spec.newGenericResource(nil, nil, "/a/"+a100, a100, "css")}
b.ResetTimer()
for i := 0; i < b.N; i++ {
for i := 0; i < 30; i++ {
name := fmt.Sprintf("abcde%d_%d.css", i%5, i)
- resources = append(resources, spec.newGenericResource(nil, nil, "/public", "/a/"+name, name, "css"))
+ resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css"))
}
for i := 0; i < 30; i++ {
name := fmt.Sprintf("efghi%d_%d.css", i%5, i)
- resources = append(resources, spec.newGenericResource(nil, nil, "/public", "/a/"+name, name, "css"))
+ resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css"))
}
for i := 0; i < 30; i++ {
name := fmt.Sprintf("jklmn%d_%d.css", i%5, i)
- resources = append(resources, spec.newGenericResource(nil, nil, "/public", "/b/sub/"+name, "sub/"+name, "css"))
+ resources = append(resources, spec.newGenericResource(nil, nil, "/b/sub/"+name, "sub/"+name, "css"))
}
return resources
}
for i := 0; i < 20; i++ {
name := fmt.Sprintf("foo%d_%d.css", i%5, i)
- resources = append(resources, spec.newGenericResource(nil, nil, "/public", "/a/"+name, name, "css"))
+ resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css"))
}
b.StartTimer()
"path/filepath"
"testing"
+ "fmt"
"image"
"io"
"io/ioutil"
func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *Spec {
cfg := viper.New()
cfg.Set("baseURL", baseURL)
- cfg.Set("resourceDir", "/res")
+ cfg.Set("resourceDir", "resources")
+ cfg.Set("contentDir", "content")
imagingCfg := map[string]interface{}{
"resampleFilter": "linear",
workDir = "/private" + workDir
}
- contentDir := "base"
cfg.Set("workingDir", workDir)
- cfg.Set("contentDir", contentDir)
+ cfg.Set("contentDir", filepath.Join(workDir, "content"))
cfg.Set("resourceDir", filepath.Join(workDir, "res"))
fs := hugofs.NewFrom(hugofs.Os, cfg)
src, err := os.Open(filepath.FromSlash("testdata/" + name))
assert.NoError(err)
- workingDir := spec.Cfg.GetString("workingDir")
- f := filepath.Join(workingDir, name)
-
- out, err := spec.Fs.Source.Create(f)
+ assert.NoError(spec.BaseFs.ContentFs.MkdirAll(filepath.Dir(name), 0755))
+ out, err := spec.BaseFs.ContentFs.Create(name)
assert.NoError(err)
_, err = io.Copy(out, src)
out.Close()
return path.Join("/a", s)
}
- r, err := spec.NewResourceFromFilename(factory, "/public", f, name)
+ r, err := spec.NewResourceFromFilename(factory, name, name)
assert.NoError(err)
return r
}
-func assertImageFile(assert *require.Assertions, fs *hugofs.Fs, filename string, width, height int) {
- f, err := fs.Source.Open(filename)
+func assertImageFile(assert *require.Assertions, fs afero.Fs, filename string, width, height int) {
+ f, err := fs.Open(filename)
+ if err != nil {
+ printFs(fs, "", os.Stdout)
+ }
assert.NoError(err)
defer f.Close()
assert.Equal(height, config.Height)
}
-func assertFileCache(assert *require.Assertions, fs *hugofs.Fs, filename string, width, height int) {
- assertImageFile(assert, fs, filepath.Join("/res/_gen/images", filename), width, height)
+func assertFileCache(assert *require.Assertions, fs afero.Fs, filename string, width, height int) {
+ assertImageFile(assert, fs, filepath.Join("_gen/images", filename), width, height)
}
func writeSource(t testing.TB, fs *hugofs.Fs, filename, content string) {
t.Fatalf("Failed to write file: %s", err)
}
}
+
+func printFs(fs afero.Fs, path string, w io.Writer) {
+ if fs == nil {
+ return
+ }
+ afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {
+ if info != nil && !info.IsDir() {
+ s := path
+ if lang, ok := info.(hugofs.LanguageAnnouncer); ok {
+ s = s + "\t" + lang.Lang()
+ }
+ if fp, ok := info.(hugofs.FilePather); ok {
+ s += "\tFilename: " + fp.Filename() + "\tBase: " + fp.BaseDir()
+ }
+ fmt.Fprintln(w, " ", s)
+ }
+ return nil
+ })
+}
"path/filepath"
"testing"
+ "github.com/gohugoio/hugo/helpers"
+
"github.com/gohugoio/hugo/hugofs"
"github.com/spf13/viper"
+ "github.com/stretchr/testify/require"
)
func TestIgnoreDotFilesAndDirectories(t *testing.T) {
+ assert := require.New(t)
tests := []struct {
path string
{"foobar/.barfoo.md", true, nil},
{".barfoo.md", true, nil},
{".md", true, nil},
- {"", true, nil},
{"foobar/barfoo.md~", true, nil},
{".foobar/barfoo.md~", true, nil},
{"foobar~/barfoo.md", false, nil},
for i, test := range tests {
v := viper.New()
+ v.Set("contentDir", "content")
v.Set("ignoreFiles", test.ignoreFilesRegexpes)
+ fs := hugofs.NewMem(v)
+ ps, err := helpers.NewPathSpec(fs, v)
+ assert.NoError(err)
- s := NewSourceSpec(v, hugofs.NewMem(v))
+ s := NewSourceSpec(ps, fs.Source)
if ignored := s.IgnoreFile(filepath.FromSlash(test.path)); test.ignore != ignored {
t.Errorf("[%d] File not ignored", i)
for i, test := range tests {
msg := fmt.Sprintf("Test %d", i)
v := viper.New()
+ v.Set("contentDir", "content")
+
fs := hugofs.NewMem(v)
cfg := test.setup(v, fs)
cfg.Set("workingDir", filepath.FromSlash("/work"))
v.Set("workingDir", filepath.FromSlash("/work"))
v.Set("theme", "mytheme")
v.Set("themesDir", "themes")
+ v.Set("contentDir", "content")
v.Set("staticDir", []string{"s1", "s2"})
v.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(v)})
package source
import (
+ "fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
+ "github.com/spf13/afero"
+
+ "github.com/gohugoio/hugo/hugofs"
+
"github.com/gohugoio/hugo/helpers"
)
// Absolute filename to the file on disk.
filename string
- fi os.FileInfo
+
+ sp *SourceSpec
+
+ fi os.FileInfo
// Derived from filename
ext string // Extension without any "."
uniqueID string
- sp *SourceSpec
-
lazyInit sync.Once
}
fi.lazyInit.Do(func() {
relDir := strings.Trim(fi.relDir, helpers.FilePathSeparator)
parts := strings.Split(relDir, helpers.FilePathSeparator)
-
var section string
if (!fi.isLeafBundle && len(parts) == 1) || len(parts) > 1 {
section = parts[0]
func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, fi os.FileInfo) *FileInfo {
+ var lang, translationBaseName, relPath string
+
+ if fp, ok := fi.(hugofs.FilePather); ok {
+ filename = fp.Filename()
+ baseDir = fp.BaseDir()
+ relPath = fp.Path()
+ }
+
+ if fl, ok := fi.(hugofs.LanguageAnnouncer); ok {
+ lang = fl.Lang()
+ translationBaseName = fl.TranslationBaseName()
+ }
+
dir, name := filepath.Split(filename)
if !strings.HasSuffix(dir, helpers.FilePathSeparator) {
dir = dir + helpers.FilePathSeparator
relDir = strings.TrimPrefix(relDir, helpers.FilePathSeparator)
- relPath := filepath.Join(relDir, name)
+ if relPath == "" {
+ relPath = filepath.Join(relDir, name)
+ }
ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), "."))
baseName := helpers.Filename(name)
- lang := strings.TrimPrefix(filepath.Ext(baseName), ".")
- var translationBaseName string
-
- if _, ok := sp.Languages[lang]; lang == "" || !ok {
- lang = sp.DefaultContentLanguage
- translationBaseName = baseName
- } else {
- translationBaseName = helpers.Filename(baseName)
+ if translationBaseName == "" {
+ // This is usyally provided by the filesystem. But this FileInfo is also
+ // created in a standalone context when doing "hugo new". This is
+ // an approximate implementation, which is "good enough" in that case.
+ translationBaseName = strings.TrimSuffix(baseName, ext)
+ fileLangExt := filepath.Ext(translationBaseName)
+ translationBaseName = strings.TrimSuffix(translationBaseName, fileLangExt)
}
f := &FileInfo{
// Open implements ReadableFile.
func (fi *FileInfo) Open() (io.ReadCloser, error) {
- return fi.sp.Fs.Source.Open(fi.Filename())
+ f, err := fi.sp.PathSpec.Fs.Source.Open(fi.Filename())
+ return f, err
+}
+
+func printFs(fs afero.Fs, path string, w io.Writer) {
+ if fs == nil {
+ return
+ }
+ afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {
+
+ if info != nil && !info.IsDir() {
+
+ s := path
+ if lang, ok := info.(hugofs.LanguageAnnouncer); ok {
+ s = s + "\t" + lang.Lang()
+ }
+ if fp, ok := info.(hugofs.FilePather); ok {
+ s = s + "\t" + fp.Filename()
+ }
+ fmt.Fprintln(w, " ", s)
+ }
+ return nil
+ })
}
"path/filepath"
"testing"
+ "github.com/gohugoio/hugo/helpers"
+
+ "github.com/spf13/viper"
+
+ "github.com/gohugoio/hugo/hugofs"
+ "github.com/spf13/afero"
"github.com/stretchr/testify/require"
)
assert.Equal(filepath.FromSlash("b/"), f.Dir())
assert.Equal(filepath.FromSlash("b/page.md"), f.Path())
assert.Equal("b", f.Section())
+ assert.Equal(filepath.FromSlash("page"), f.TranslationBaseName())
+ assert.Equal(filepath.FromSlash("page"), f.BaseFileName())
}},
{filepath.FromSlash("/a/"), filepath.FromSlash("/a/b/c/d/page.md"), func(f *FileInfo) {
}
}
+
+func TestFileInfoLanguage(t *testing.T) {
+ assert := require.New(t)
+ langs := map[string]bool{
+ "sv": true,
+ "en": true,
+ }
+
+ m := afero.NewMemMapFs()
+ lfs := hugofs.NewLanguageFs("sv", langs, m)
+ v := viper.New()
+ v.Set("contentDir", "content")
+
+ fs := hugofs.NewFrom(m, v)
+
+ ps, err := helpers.NewPathSpec(fs, v)
+ assert.NoError(err)
+ s := SourceSpec{Fs: lfs, PathSpec: ps}
+ s.Languages = map[string]interface{}{
+ "en": true,
+ }
+
+ err = afero.WriteFile(lfs, "page.md", []byte("abc"), 0777)
+ assert.NoError(err)
+ err = afero.WriteFile(lfs, "page.en.md", []byte("abc"), 0777)
+ assert.NoError(err)
+
+ sv, _ := lfs.Stat("page.md")
+ en, _ := lfs.Stat("page.en.md")
+
+ fiSv := s.NewFileInfo("", "page.md", false, sv)
+ fiEn := s.NewFileInfo("", "page.en.md", false, en)
+
+ assert.Equal("sv", fiSv.Lang())
+ assert.Equal("en", fiEn.Lang())
+}
if f.Fs == nil {
panic("Must have a fs")
}
- err := helpers.SymbolicWalk(f.Fs.Source, f.Base, walker)
+ err := helpers.SymbolicWalk(f.Fs, f.Base, walker)
if err != nil {
jww.ERROR.Println(err)
- if err == helpers.ErrWalkRootTooShort {
+ if err == helpers.ErrPathTooShort {
panic("The root path is too short. If this is a test, make sure to init the content paths.")
}
}
jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filename, err)
return false, nil
}
- linkfi, err := f.Fs.Source.Stat(link)
+ linkfi, err := f.Fs.Stat(link)
if err != nil {
jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)
return false, nil
"runtime"
"testing"
+ "github.com/gohugoio/hugo/helpers"
+
"github.com/gohugoio/hugo/hugofs"
"github.com/spf13/viper"
func newTestSourceSpec() SourceSpec {
v := viper.New()
- return SourceSpec{Fs: hugofs.NewMem(v), Cfg: v}
+ v.Set("contentDir", "content")
+ ps, _ := helpers.NewPathSpec(hugofs.NewMem(v), v)
+ return SourceSpec{Fs: hugofs.NewMem(v).Source, PathSpec: ps}
}
"path/filepath"
"regexp"
- "github.com/gohugoio/hugo/config"
+ "github.com/spf13/afero"
+
"github.com/gohugoio/hugo/helpers"
- "github.com/gohugoio/hugo/hugofs"
"github.com/spf13/cast"
)
// SourceSpec abstracts language-specific file creation.
// TODO(bep) rename to Spec
type SourceSpec struct {
- Cfg config.Provider
- Fs *hugofs.Fs
+ *helpers.PathSpec
+
+ Fs afero.Fs
// This is set if the ignoreFiles config is set.
ignoreFilesRe []*regexp.Regexp
DisabledLanguages map[string]bool
}
-// NewSourceSpec initializes SourceSpec using languages from a given configuration.
-func NewSourceSpec(cfg config.Provider, fs *hugofs.Fs) *SourceSpec {
+// NewSourceSpec initializes SourceSpec using languages the given filesystem and PathSpec.
+func NewSourceSpec(ps *helpers.PathSpec, fs afero.Fs) *SourceSpec {
+ cfg := ps.Cfg
defaultLang := cfg.GetString("defaultContentLanguage")
languages := cfg.GetStringMap("languages")
}
}
- return &SourceSpec{ignoreFilesRe: regexps, Cfg: cfg, Fs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet}
+ return &SourceSpec{ignoreFilesRe: regexps, PathSpec: ps, Fs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet}
}
func (s *SourceSpec) IgnoreFile(filename string) bool {
+ if filename == "" {
+ if _, ok := s.Fs.(*afero.OsFs); ok {
+ return true
+ }
+ return false
+ }
+
base := filepath.Base(filename)
if len(base) > 0 {
}
func (s *SourceSpec) IsRegularSourceFile(filename string) (bool, error) {
- fi, err := helpers.LstatIfOs(s.Fs.Source, filename)
+ fi, err := helpers.LstatIfPossible(s.Fs, filename)
if err != nil {
return false, err
}
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
link, err := filepath.EvalSymlinks(filename)
- fi, err = helpers.LstatIfOs(s.Fs.Source, link)
+ fi, err = helpers.LstatIfPossible(s.Fs, link)
if err != nil {
return false, err
}
func TestIsSet(t *testing.T) {
t.Parallel()
- ns := New(newDeps(viper.New()))
+ ns := newTestNs()
for i, test := range []struct {
a interface{}
Log: jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime),
}
}
+
+func newTestNs() *Namespace {
+ v := viper.New()
+ v.Set("contentDir", "content")
+ return New(newDeps(v))
+}
"strings"
"testing"
- "github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetCSV(t *testing.T) {
t.Parallel()
- ns := New(newDeps(viper.New()))
+ ns := newTestNs()
for i, test := range []struct {
sep string
func TestGetJSON(t *testing.T) {
t.Parallel()
- ns := New(newDeps(viper.New()))
+ ns := newTestNs()
for i, test := range []struct {
url string
func TestScpGetRemoteParallel(t *testing.T) {
t.Parallel()
- ns := New(newDeps(viper.New()))
+ ns := newTestNs()
content := []byte(`T€st Content 123`)
srv, cl := getTestServer(func(w http.ResponseWriter, r *http.Request) {
ContentSpec: cs,
}
}
+
+func newTestNs() *Namespace {
+ v := viper.New()
+ v.Set("contentDir", "content")
+ return New(newDeps(v))
+}
// New returns a new instance of the os-namespaced template functions.
func New(deps *deps.Deps) *Namespace {
+
+ // Since Hugo 0.38 we can have multiple content dirs. This can make it hard to
+ // reason about where the file is placed relative to the project root.
+ // To make the {{ readFile .Filename }} variant just work, we create a composite
+ // filesystem that first checks the work dir fs and then the content fs.
+ var rfs afero.Fs
+ if deps.Fs != nil {
+ rfs = deps.Fs.WorkingDir
+ if deps.PathSpec != nil && deps.PathSpec.BaseFs != nil {
+ rfs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(deps.PathSpec.BaseFs.ContentFs, deps.Fs.WorkingDir))
+ }
+ }
+
return &Namespace{
- deps: deps,
+ readFileFs: rfs,
+ deps: deps,
}
}
// Namespace provides template functions for the "os" namespace.
type Namespace struct {
- deps *deps.Deps
+ readFileFs afero.Fs
+ deps *deps.Deps
}
// Getenv retrieves the value of the environment variable named by the key.
return _os.Getenv(skey), nil
}
-// readFile reads the file named by filename relative to the given basepath
+// readFile reads the file named by filename in the given filesystem
// and returns the contents as a string.
// There is a upper size limit set at 1 megabytes.
-func readFile(fs *afero.BasePathFs, filename string) (string, error) {
+func readFile(fs afero.Fs, filename string) (string, error) {
if filename == "" {
return "", errors.New("readFile needs a filename")
}
return "", err
}
- return readFile(ns.deps.Fs.WorkingDir, s)
+ return readFile(ns.readFileFs, s)
}
// ReadDir lists the directory contents relative to the configured WorkingDir.
v.Set("workingDir", workingDir)
v.Set("multilingual", true)
+ v.Set("contentDir", "content")
v.Set("baseURL", "http://mysite.com/hugo/")
v.Set("CurrentContentLanguage", helpers.NewLanguage("en", v))
var data struct {
}
- config := newDepsConfig(viper.New())
+ v := viper.New()
+ v.Set("contentDir", "content")
+
+ config := newDepsConfig(v)
config.WithTemplate = func(templ tpl.TemplateHandler) error {
err := templ.AddTemplate("partials/"+name, partial)
"other": "<h1>Hi!</h1>",
}
v := viper.New()
+ v.Set("contentDir", "content")
fs := hugofs.NewMem(v)
//afero.WriteFile(fs.Source, filepath.Join(workingDir, "README.txt"), []byte("Hugo Rocks!"), 0755)
func TestRemarshal(t *testing.T) {
t.Parallel()
- ns := New(newDeps(viper.New()))
+ v := viper.New()
+ v.Set("contentDir", "content")
+ ns := New(newDeps(v))
assert := require.New(t)
tomlExample := `title = "Test Metadata"
func TestRemarshalComments(t *testing.T) {
t.Parallel()
- ns := New(newDeps(viper.New()))
+ v := viper.New()
+ v.Set("contentDir", "content")
+ ns := New(newDeps(v))
+
assert := require.New(t)
input := `
func TestTestRemarshalError(t *testing.T) {
t.Parallel()
- ns := New(newDeps(viper.New()))
+ v := viper.New()
+ v.Set("contentDir", "content")
+ ns := New(newDeps(v))
assert := require.New(t)
_, err := ns.Remarshal("asdf", "asdf")
func TestEmojify(t *testing.T) {
t.Parallel()
- ns := New(newDeps(viper.New()))
+ v := viper.New()
+ v.Set("contentDir", "content")
+ ns := New(newDeps(v))
for i, test := range []struct {
s interface{}
func TestHighlight(t *testing.T) {
t.Parallel()
- ns := New(newDeps(viper.New()))
+ v := viper.New()
+ v.Set("contentDir", "content")
+ ns := New(newDeps(v))
for i, test := range []struct {
s interface{}
func TestHTMLEscape(t *testing.T) {
t.Parallel()
- ns := New(newDeps(viper.New()))
+ v := viper.New()
+ v.Set("contentDir", "content")
+ ns := New(newDeps(v))
for i, test := range []struct {
s interface{}
func TestHTMLUnescape(t *testing.T) {
t.Parallel()
- ns := New(newDeps(viper.New()))
+ v := viper.New()
+ v.Set("contentDir", "content")
+ ns := New(newDeps(v))
for i, test := range []struct {
s interface{}
func TestMarkdownify(t *testing.T) {
t.Parallel()
- ns := New(newDeps(viper.New()))
+ v := viper.New()
+ v.Set("contentDir", "content")
+ ns := New(newDeps(v))
for i, test := range []struct {
s interface{}
assert := require.New(t)
- ns := New(newDeps(viper.New()))
+ v := viper.New()
+ v.Set("contentDir", "content")
+ ns := New(newDeps(v))
text := `
#First
func TestPlainify(t *testing.T) {
t.Parallel()
- ns := New(newDeps(viper.New()))
+ v := viper.New()
+ v.Set("contentDir", "content")
+ ns := New(newDeps(v))
for i, test := range []struct {
s interface{}
projects / brevno-suite / hugo / commitdiff