This commit introduces a new data structure to store pages and their resources.
This data structure is backed by radix trees.
This simplies tree operations, makes all pages a bundle, and paves the way for #6310.
It also solves a set of annoying issues (see list below).
Not a motivation behind this, but this commit also makes Hugo in general a little bit faster and more memory effective (see benchmarks). Especially for partial rebuilds on content edits, but also when taxonomies is in use.
```
name old time/op new time/op delta
SiteNew/Bundle_with_image/Edit-16 1.32ms ± 8% 1.00ms ± 9% -24.42% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file/Edit-16 1.28ms ± 0% 0.94ms ± 0% -26.26% (p=0.029 n=4+4)
SiteNew/Tags_and_categories/Edit-16 33.9ms ± 2% 21.8ms ± 1% -35.67% (p=0.029 n=4+4)
SiteNew/Canonify_URLs/Edit-16 40.6ms ± 1% 37.7ms ± 3% -7.20% (p=0.029 n=4+4)
SiteNew/Deep_content_tree/Edit-16 56.7ms ± 0% 51.7ms ± 1% -8.82% (p=0.029 n=4+4)
SiteNew/Many_HTML_templates/Edit-16 19.9ms ± 2% 18.3ms ± 3% -7.64% (p=0.029 n=4+4)
SiteNew/Page_collections/Edit-16 37.9ms ± 4% 34.0ms ± 2% -10.28% (p=0.029 n=4+4)
SiteNew/Bundle_with_image-16 10.7ms ± 0% 10.6ms ± 0% -1.15% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16 10.8ms ± 0% 10.7ms ± 0% -1.05% (p=0.029 n=4+4)
SiteNew/Tags_and_categories-16 43.2ms ± 1% 39.6ms ± 1% -8.35% (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16 47.6ms ± 1% 47.3ms ± 0% ~ (p=0.057 n=4+4)
SiteNew/Deep_content_tree-16 73.0ms ± 1% 74.2ms ± 1% ~ (p=0.114 n=4+4)
SiteNew/Many_HTML_templates-16 37.9ms ± 0% 38.1ms ± 1% ~ (p=0.114 n=4+4)
SiteNew/Page_collections-16 53.6ms ± 1% 54.7ms ± 1% +2.09% (p=0.029 n=4+4)
name old alloc/op new alloc/op delta
SiteNew/Bundle_with_image/Edit-16 486kB ± 0% 430kB ± 0% -11.47% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file/Edit-16 265kB ± 0% 209kB ± 0% -21.06% (p=0.029 n=4+4)
SiteNew/Tags_and_categories/Edit-16 13.6MB ± 0% 8.8MB ± 0% -34.93% (p=0.029 n=4+4)
SiteNew/Canonify_URLs/Edit-16 66.5MB ± 0% 63.9MB ± 0% -3.95% (p=0.029 n=4+4)
SiteNew/Deep_content_tree/Edit-16 28.8MB ± 0% 25.8MB ± 0% -10.55% (p=0.029 n=4+4)
SiteNew/Many_HTML_templates/Edit-16 6.16MB ± 0% 5.56MB ± 0% -9.86% (p=0.029 n=4+4)
SiteNew/Page_collections/Edit-16 16.9MB ± 0% 16.0MB ± 0% -5.19% (p=0.029 n=4+4)
SiteNew/Bundle_with_image-16 2.28MB ± 0% 2.29MB ± 0% +0.35% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16 2.07MB ± 0% 2.07MB ± 0% ~ (p=0.114 n=4+4)
SiteNew/Tags_and_categories-16 14.3MB ± 0% 13.2MB ± 0% -7.30% (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16 69.1MB ± 0% 69.0MB ± 0% ~ (p=0.343 n=4+4)
SiteNew/Deep_content_tree-16 31.3MB ± 0% 31.8MB ± 0% +1.49% (p=0.029 n=4+4)
SiteNew/Many_HTML_templates-16 10.8MB ± 0% 10.9MB ± 0% +1.11% (p=0.029 n=4+4)
SiteNew/Page_collections-16 21.4MB ± 0% 21.6MB ± 0% +1.15% (p=0.029 n=4+4)
name old allocs/op new allocs/op delta
SiteNew/Bundle_with_image/Edit-16 4.74k ± 0% 3.86k ± 0% -18.57% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file/Edit-16 4.73k ± 0% 3.85k ± 0% -18.58% (p=0.029 n=4+4)
SiteNew/Tags_and_categories/Edit-16 301k ± 0% 198k ± 0% -34.14% (p=0.029 n=4+4)
SiteNew/Canonify_URLs/Edit-16 389k ± 0% 373k ± 0% -4.07% (p=0.029 n=4+4)
SiteNew/Deep_content_tree/Edit-16 338k ± 0% 262k ± 0% -22.63% (p=0.029 n=4+4)
SiteNew/Many_HTML_templates/Edit-16 102k ± 0% 88k ± 0% -13.81% (p=0.029 n=4+4)
SiteNew/Page_collections/Edit-16 176k ± 0% 152k ± 0% -13.32% (p=0.029 n=4+4)
SiteNew/Bundle_with_image-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4)
SiteNew/Tags_and_categories-16 273k ± 0% 245k ± 0% -10.36% (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16 396k ± 0% 398k ± 0% +0.39% (p=0.029 n=4+4)
SiteNew/Deep_content_tree-16 317k ± 0% 325k ± 0% +2.53% (p=0.029 n=4+4)
SiteNew/Many_HTML_templates-16 146k ± 0% 147k ± 0% +0.98% (p=0.029 n=4+4)
SiteNew/Page_collections-16 210k ± 0% 215k ± 0% +2.44% (p=0.029 n=4+4)
```
Fixes #6312
Fixes #6087
Fixes #6738
Fixes #6412
Fixes #6743
Fixes #6875
Fixes #6034
Fixes #6902
Fixes #6173
Fixes #6590
import (
"bytes"
"fmt"
- "io"
"strings"
"time"
+ "github.com/gohugoio/hugo/parser/pageparser"
+
"github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/parser"
"github.com/gohugoio/hugo/parser/metadecoders"
- "github.com/gohugoio/hugo/parser/pageparser"
"github.com/pkg/errors"
return nil
}
- pf, err := parseContentFile(file)
+ pf, err := pageparser.ParseFrontMatterAndContent(file)
if err != nil {
site.Log.ERROR.Println(errMsg)
file.Close()
file.Close()
// better handling of dates in formats that don't have support for them
- if pf.frontMatterFormat == metadecoders.JSON || pf.frontMatterFormat == metadecoders.YAML || pf.frontMatterFormat == metadecoders.TOML {
- for k, v := range pf.frontMatter {
+ if pf.FrontMatterFormat == metadecoders.JSON || pf.FrontMatterFormat == metadecoders.YAML || pf.FrontMatterFormat == metadecoders.TOML {
+ for k, v := range pf.FrontMatter {
switch vv := v.(type) {
case time.Time:
- pf.frontMatter[k] = vv.Format(time.RFC3339)
+ pf.FrontMatter[k] = vv.Format(time.RFC3339)
}
}
}
var newContent bytes.Buffer
- err = parser.InterfaceToFrontMatter(pf.frontMatter, targetFormat, &newContent)
+ err = parser.InterfaceToFrontMatter(pf.FrontMatter, targetFormat, &newContent)
if err != nil {
site.Log.ERROR.Println(errMsg)
return err
}
- newContent.Write(pf.content)
+ newContent.Write(pf.Content)
newFilename := p.File().Filename()
// Everything after Front Matter
content []byte
}
-
-func parseContentFile(r io.Reader) (parsedFile, error) {
- var pf parsedFile
-
- psr, err := pageparser.Parse(r, pageparser.Config{})
- if err != nil {
- return pf, err
- }
-
- iter := psr.Iterator()
-
- walkFn := func(item pageparser.Item) bool {
- if pf.frontMatterSource != nil {
- // The rest is content.
- pf.content = psr.Input()[item.Pos:]
- // Done
- return false
- } else if item.IsFrontMatter() {
- pf.frontMatterFormat = metadecoders.FormatFromFrontMatterType(item.Type)
- pf.frontMatterSource = item.Val
- }
- return true
-
- }
-
- iter.PeekWalk(walkFn)
-
- metadata, err := metadecoders.Default.UnmarshalToMap(pf.frontMatterSource, pf.frontMatterFormat)
- if err != nil {
- return pf, err
- }
- pf.frontMatter = metadata
-
- return pf, nil
-
-}
"time"
"unicode"
+ "github.com/gohugoio/hugo/parser/pageparser"
+
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/parser/metadecoders"
return err
}
- pf, err := parseContentFile(bytes.NewReader(contentBytes))
+ pf, err := pageparser.ParseFrontMatterAndContent(bytes.NewReader(contentBytes))
if err != nil {
jww.ERROR.Println("Parse file error:", path)
return err
}
- newmetadata, err := convertJekyllMetaData(pf.frontMatter, postName, postDate, draft)
+ newmetadata, err := convertJekyllMetaData(pf.FrontMatter, postName, postDate, draft)
if err != nil {
jww.ERROR.Println("Convert metadata error:", path)
return err
}
- content, err := convertJekyllContent(newmetadata, string(pf.content))
+ content, err := convertJekyllContent(newmetadata, string(pf.Content))
if err != nil {
jww.ERROR.Println("Converting Jekyll error:", path)
return err
fmt.Fprintf(w, "%s", buf)
}
+// ErrorSender is a, typically, non-blocking error handler.
+type ErrorSender interface {
+ SendError(err error)
+}
+
// Recover is a helper function that can be used to capture panics.
// Put this at the top of a method/function that crashes in a template:
// defer herrors.Recover()
import (
"context"
"runtime"
+
"sort"
"sync"
"sync/atomic"
--- /dev/null
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+import "github.com/spf13/cast"
+
+// ToStringSlicePreserveString converts v to a string slice.
+// If v is a string, it will be wrapped in a string slice.
+func ToStringSlicePreserveString(v interface{}) []string {
+ if v == nil {
+ return nil
+ }
+ if sds, ok := v.(string); ok {
+ return []string{sds}
+ }
+ return cast.ToStringSlice(v)
+}
--- /dev/null
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+import (
+ "testing"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func TestToStringSlicePreserveString(t *testing.T) {
+ c := qt.New(t)
+
+ c.Assert(ToStringSlicePreserveString("Hugo"), qt.DeepEquals, []string{"Hugo"})
+ c.Assert(ToStringSlicePreserveString([]interface{}{"A", "B"}), qt.DeepEquals, []string{"A", "B"})
+ c.Assert(ToStringSlicePreserveString(nil), qt.IsNil)
+
+}
package config
import (
- "github.com/spf13/cast"
+ "github.com/gohugoio/hugo/common/types"
)
// Provider provides the configuration settings for Hugo.
// we do not attempt to split it into fields.
func GetStringSlicePreserveString(cfg Provider, key string) []string {
sd := cfg.Get(key)
- return toStringSlicePreserveString(sd)
-}
-
-func toStringSlicePreserveString(v interface{}) []string {
- if sds, ok := v.(string); ok {
- return []string{sds}
- }
- return cast.ToStringSlice(v)
+ return types.ToStringSlicePreserveString(sd)
}
// SetBaseTestDefaults provides some common config defaults used in tests.
Date: time.Now().Format(time.RFC3339),
Name: name,
File: f,
- Site: &s.Info,
+ Site: s.Info,
}
if archetypeFilename == "" {
github.com/bep/gitmap v1.1.1/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kYaY=
github.com/bep/golibsass v0.4.0 h1:B2jsNZuRgpsyzv0I5iubJYApDhib87RzjTcRhVOjg78=
github.com/bep/golibsass v0.4.0/go.mod h1:DL87K8Un/+pWUS75ggYv41bliGiolxzDKWJAq3eJ1MA=
+github.com/bep/golibsass v0.5.0 h1:b+Uxsk826Q35OmbenSmU65P+FJJQoVs2gI2mk1ba28s=
github.com/bep/golibsass v0.5.0/go.mod h1:DL87K8Un/+pWUS75ggYv41bliGiolxzDKWJAq3eJ1MA=
github.com/bep/tmc v0.5.1 h1:CsQnSC6MsomH64gw0cT5f+EwQDcvZz4AazKunFwTpuI=
github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0=
return pflag.NormalizedName(name)
}
-// DiffStringSlices returns the difference between two string slices.
-// Useful in tests.
-// See:
-// http://stackoverflow.com/questions/19374219/how-to-find-the-difference-between-two-slices-of-strings-in-golang
-func DiffStringSlices(slice1 []string, slice2 []string) []string {
- diffStr := []string{}
- m := map[string]int{}
-
- for _, s1Val := range slice1 {
- m[s1Val] = 1
- }
- for _, s2Val := range slice2 {
- m[s2Val] = m[s2Val] + 1
- }
-
- for mKey, mVal := range m {
- if mVal == 1 {
- diffStr = append(diffStr, mKey)
- }
- }
-
- return diffStr
-}
-
-// DiffStrings splits the strings into fields and runs it into DiffStringSlices.
-// Useful for tests.
-func DiffStrings(s1, s2 string) []string {
- return DiffStringSlices(strings.Fields(s1), strings.Fields(s2))
-}
-
// PrintFs prints the given filesystem to the given writer starting from the given path.
// This is useful for debugging.
func PrintFs(fs afero.Fs, path string, w io.Writer) {
"fmt"
"io"
"os"
+ "path"
"path/filepath"
"regexp"
"sort"
return file, strings.TrimPrefix(ext, ".")
}
-// Filename takes a path, strips out the extension,
+// Filename takes a file path, strips out the extension,
// and returns the name of the file.
func Filename(in string) (name string) {
name, _ = fileAndExt(in, fpb)
return
}
+// PathNoExt takes a path, strips out the extension,
+// and returns the name of the file.
+func PathNoExt(in string) string {
+ return strings.TrimSuffix(in, path.Ext(in))
+}
+
// FileAndExt returns the filename and any extension of a file path as
// two separate strings.
//
import (
"errors"
+ "fmt"
"reflect"
+ "strings"
qt "github.com/frankban/quicktest"
+ "github.com/gohugoio/hugo/htesting"
"github.com/google/go-cmp/cmp"
+ "github.com/spf13/cast"
)
+// IsSameString asserts that two strings are equal. The two strings
+// are normalized (whitespace removed) before doing a ==.
+// Also note that two strings can be the same even if they're of different
+// types.
+var IsSameString qt.Checker = &stringChecker{
+ argNames: []string{"got", "want"},
+}
+
// IsSameType asserts that got is the same type as want.
var IsSameType qt.Checker = &typeChecker{
argNames: []string{"got", "want"},
return nil
}
+type stringChecker struct {
+ argNames
+}
+
+// Check implements Checker.Check by checking that got and args[0] represents the same normalized text (whitespace etc. trimmed).
+func (c *stringChecker) Check(got interface{}, args []interface{}, note func(key string, value interface{})) (err error) {
+ s1, s2 := cast.ToString(got), cast.ToString(args[0])
+
+ if s1 == s2 {
+ return nil
+ }
+
+ s1, s2 = normalizeString(s1), normalizeString(s2)
+
+ if s1 == s2 {
+ return nil
+ }
+
+ return fmt.Errorf("values are not the same text: %s", htesting.DiffStrings(s1, s2))
+}
+
+func normalizeString(s string) string {
+ lines := strings.Split(strings.TrimSpace(s), "\n")
+ for i, line := range lines {
+ lines[i] = strings.TrimSpace(line)
+ }
+
+ return strings.Join(lines, "\n")
+}
+
// DeepAllowUnexported creates an option to allow compare of unexported types
// in the given list of types.
// see https://github.com/google/go-cmp/issues/40#issuecomment-328615283
func RandIntn(n int) int {
return rnd.Intn(n)
}
+
+// DiffStringSlices returns the difference between two string slices.
+// Useful in tests.
+// See:
+// http://stackoverflow.com/questions/19374219/how-to-find-the-difference-between-two-slices-of-strings-in-golang
+func DiffStringSlices(slice1 []string, slice2 []string) []string {
+ diffStr := []string{}
+ m := map[string]int{}
+
+ for _, s1Val := range slice1 {
+ m[s1Val] = 1
+ }
+ for _, s2Val := range slice2 {
+ m[s2Val] = m[s2Val] + 1
+ }
+
+ for mKey, mVal := range m {
+ if mVal == 1 {
+ diffStr = append(diffStr, mKey)
+ }
+ }
+
+ return diffStr
+}
+
+// DiffStrings splits the strings into fields and runs it into DiffStringSlices.
+// Useful for tests.
+func DiffStrings(s1, s2 string) []string {
+ return DiffStringSlices(strings.Fields(s1), strings.Fields(s2))
+}
// NewBaseFileDecorator decorates the given Fs to provide the real filename
// and an Opener func.
-func NewBaseFileDecorator(fs afero.Fs) afero.Fs {
+func NewBaseFileDecorator(fs afero.Fs, callbacks ...func(fi FileMetaInfo)) afero.Fs {
+
ffs := &baseFileDecoratorFs{Fs: fs}
decorator := func(fi os.FileInfo, filename string) (os.FileInfo, error) {
return ffs.open(filename)
}
- return decorateFileInfo(fi, ffs, opener, filename, "", meta), nil
+ fim := decorateFileInfo(fi, ffs, opener, filename, "", meta)
+
+ for _, cb := range callbacks {
+ cb(fim)
+ }
+
+ return fim, nil
+
}
ffs.decorate = decorator
metaKeyBaseDir = "baseDir" // Abs base directory of source file.
metaKeyMountRoot = "mountRoot"
+ metaKeyModule = "module"
metaKeyOriginalFilename = "originalFilename"
metaKeyName = "name"
metaKeyPath = "path"
return f.stringV(metaKeyName)
}
-func (f FileMeta) Classifier() string {
- c := f.stringV(metaKeyClassifier)
- if c != "" {
- return c
+func (f FileMeta) Classifier() files.ContentClass {
+ c, found := f[metaKeyClassifier]
+ if found {
+ return c.(files.ContentClass)
}
return files.ContentClassFile // For sorting
return f.stringV(metaKeyMountRoot)
}
+func (f FileMeta) Module() string {
+ return f.stringV(metaKeyModule)
+}
+
func (f FileMeta) Weight() int {
return f.GetInt(metaKeyWeight)
}
return contentFileExtensionsSet[ext]
}
+type ContentClass string
+
const (
- ContentClassLeaf = "leaf"
- ContentClassBranch = "branch"
- ContentClassFile = "zfile" // Sort below
- ContentClassContent = "zcontent"
+ ContentClassLeaf ContentClass = "leaf"
+ ContentClassBranch ContentClass = "branch"
+ ContentClassFile ContentClass = "zfile" // Sort below
+ ContentClassContent ContentClass = "zcontent"
)
-func ClassifyContentFile(filename string) string {
+func (c ContentClass) IsBundle() bool {
+ return c == ContentClassLeaf || c == ContentClassBranch
+}
+
+func ClassifyContentFile(filename string) ContentClass {
if !IsContentFile(filename) {
return ContentClassFile
}
}
func (fs *FilterFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
- panic("not implemented")
+ return fs.fs.Open(name)
}
func (fs *FilterFs) ReadDir(name string) ([]os.FileInfo, error) {
rm.Meta[metaKeyBaseDir] = rm.ToBasedir
rm.Meta[metaKeyMountRoot] = rm.path
+ rm.Meta[metaKeyModule] = rm.Module
meta := copyFileMeta(rm.Meta)
From string // The virtual mount.
To string // The source directory or file.
ToBasedir string // The base of To. May be empty if an absolute path was provided.
+ Module string // The module path/ID.
Meta FileMeta // File metadata (lang etc.)
fi FileMetaInfo
"bytes"
"errors"
"fmt"
- "html/template"
"io"
"path"
"path/filepath"
"github.com/gohugoio/hugo/tpl"
)
-var defaultAliasTemplates *template.Template
-
type aliasHandler struct {
t tpl.TemplateHandler
log *loggers.Logger
"bytes"
"fmt"
"path"
+ "strings"
"testing"
qt "github.com/frankban/quicktest"
b.Build(BuildCfg{})
b.AssertFileContent("public/index.html", `
- 12|taxonomy|categories/cool/_index.md|Cascade Category|cat.png|categories|HTML-|
- 12|taxonomy|categories/catsect1|catsect1|cat.png|categories|HTML-|
- 12|taxonomy|categories/funny|funny|cat.png|categories|HTML-|
- 12|taxonomyTerm|categories/_index.md|My Categories|cat.png|categories|HTML-|
- 32|taxonomy|categories/sad/_index.md|Cascade Category|sad.png|categories|HTML-|
- 42|taxonomy|tags/blue|blue|home.png|tags|HTML-|
- 42|section|sect3|Cascade Home|home.png|sect3|HTML-|
- 42|taxonomyTerm|tags|Cascade Home|home.png|tags|HTML-|
- 42|page|bundle1/index.md|Cascade Home|home.png|page|HTML-|
- 42|page|p2.md|Cascade Home|home.png|page|HTML-|
- 42|page|sect2/p2.md|Cascade Home|home.png|sect2|HTML-|
- 42|page|sect3/p1.md|Cascade Home|home.png|sect3|HTML-|
- 42|taxonomy|tags/green|green|home.png|tags|HTML-|
- 42|home|_index.md|Home|home.png|page|HTML-|
- 42|page|p1.md|p1|home.png|page|HTML-|
- 42|section|sect1/_index.md|Sect1|sect1.png|stype|HTML-|
- 42|section|sect1/s1_2/_index.md|Sect1_2|sect1.png|stype|HTML-|
- 42|page|sect1/s1_2/p1.md|Sect1_2_p1|sect1.png|stype|HTML-|
- 42|page|sect1/s1_2/p2.md|Sect1_2_p2|sect1.png|stype|HTML-|
- 42|section|sect2/_index.md|Sect2|home.png|sect2|HTML-|
- 42|page|sect2/p1.md|Sect2_p1|home.png|sect2|HTML-|
- 52|page|sect4/p1.md|Cascade Home|home.png|sect4|RSS-|
- 52|section|sect4/_index.md|Sect4|home.png|sect4|RSS-|
+12|taxonomy|categories/cool/_index.md|Cascade Category|cat.png|categories|HTML-|
+12|taxonomy|categories/catsect1|catsect1|cat.png|categories|HTML-|
+12|taxonomy|categories/funny|funny|cat.png|categories|HTML-|
+12|taxonomyTerm|categories/_index.md|My Categories|cat.png|categories|HTML-|
+32|taxonomy|categories/sad/_index.md|Cascade Category|sad.png|categories|HTML-|
+42|taxonomy|tags/blue|blue|home.png|tags|HTML-|
+42|taxonomyTerm|tags|Cascade Home|home.png|tags|HTML-|
+42|section|sectnocontent|Cascade Home|home.png|sectnocontent|HTML-|
+42|section|sect3|Cascade Home|home.png|sect3|HTML-|
+42|page|bundle1/index.md|Cascade Home|home.png|page|HTML-|
+42|page|p2.md|Cascade Home|home.png|page|HTML-|
+42|page|sect2/p2.md|Cascade Home|home.png|sect2|HTML-|
+42|page|sect3/nofrontmatter.md|Cascade Home|home.png|sect3|HTML-|
+42|page|sect3/p1.md|Cascade Home|home.png|sect3|HTML-|
+42|page|sectnocontent/p1.md|Cascade Home|home.png|sectnocontent|HTML-|
+42|section|sectnofrontmatter/_index.md|Cascade Home|home.png|sectnofrontmatter|HTML-|
+42|taxonomy|tags/green|green|home.png|tags|HTML-|
+42|home|_index.md|Home|home.png|page|HTML-|
+42|page|p1.md|p1|home.png|page|HTML-|
+42|section|sect1/_index.md|Sect1|sect1.png|stype|HTML-|
+42|section|sect1/s1_2/_index.md|Sect1_2|sect1.png|stype|HTML-|
+42|page|sect1/s1_2/p1.md|Sect1_2_p1|sect1.png|stype|HTML-|
+42|page|sect1/s1_2/p2.md|Sect1_2_p2|sect1.png|stype|HTML-|
+42|section|sect2/_index.md|Sect2|home.png|sect2|HTML-|
+42|page|sect2/p1.md|Sect2_p1|home.png|sect2|HTML-|
+52|page|sect4/p1.md|Cascade Home|home.png|sect4|RSS-|
+52|section|sect4/_index.md|Sect4|home.png|sect4|RSS-|
`)
// Check that type set in cascade gets the correct layout.
title: P1
---
`
- b := newTestSitesBuilder(t).Running()
- b.WithTemplatesAdded("_default/single.html", `Banner: {{ .Params.banner }}|Layout: {{ .Layout }}|Type: {{ .Type }}|Content: {{ .Content }}`)
- b.WithContent("post/_index.md", `
+
+ indexContentNoCascade := `
+---
+title: Home
+---
+`
+
+ indexContentCascade := `
---
-title: Post
+title: Section
cascade:
banner: post.jpg
layout: postlayout
type: posttype
---
-`)
+`
- b.WithContent("post/dir/_index.md", `
----
-title: Dir
----
-`, "post/dir/p1.md", p1Content)
- b.Build(BuildCfg{})
+ layout := `Banner: {{ .Params.banner }}|Layout: {{ .Layout }}|Type: {{ .Type }}|Content: {{ .Content }}`
+
+ newSite := func(t *testing.T, cascade bool) *sitesBuilder {
+ b := newTestSitesBuilder(t).Running()
+ b.WithTemplates("_default/single.html", layout)
+ b.WithTemplates("_default/list.html", layout)
+ if cascade {
+ b.WithContent("post/_index.md", indexContentCascade)
+ } else {
+ b.WithContent("post/_index.md", indexContentNoCascade)
+ }
+ b.WithContent("post/dir/p1.md", p1Content)
- assert := func() {
- b.Helper()
+ return b
+ }
+
+ t.Run("Edit descendant", func(t *testing.T) {
+ t.Parallel()
+
+ b := newSite(t, true)
+ b.Build(BuildCfg{})
+
+ assert := func() {
+ b.Helper()
+ b.AssertFileContent("public/post/dir/p1/index.html",
+ `Banner: post.jpg|`,
+ `Layout: postlayout`,
+ `Type: posttype`,
+ )
+ }
+
+ assert()
+
+ b.EditFiles("content/post/dir/p1.md", p1Content+"\ncontent edit")
+ b.Build(BuildCfg{})
+
+ assert()
b.AssertFileContent("public/post/dir/p1/index.html",
- `Banner: post.jpg|`,
- `Layout: postlayout`,
- `Type: posttype`,
+ `content edit
+Banner: post.jpg`,
)
- }
+ })
+
+ t.Run("Edit ancestor", func(t *testing.T) {
+ t.Parallel()
- assert()
+ b := newSite(t, true)
+ b.Build(BuildCfg{})
- b.EditFiles("content/post/dir/p1.md", p1Content+"\ncontent edit")
- b.Build(BuildCfg{})
+ b.AssertFileContent("public/post/dir/p1/index.html", `Banner: post.jpg|Layout: postlayout|Type: posttype|Content:`)
- assert()
- b.AssertFileContent("public/post/dir/p1/index.html",
- `content edit`,
- )
+ b.EditFiles("content/post/_index.md", strings.Replace(indexContentCascade, "post.jpg", "edit.jpg", 1))
+
+ b.Build(BuildCfg{})
+
+ b.AssertFileContent("public/post/index.html", `Banner: edit.jpg|Layout: postlayout|Type: posttype|`)
+ b.AssertFileContent("public/post/dir/p1/index.html", `Banner: edit.jpg|Layout: postlayout|Type: posttype|`)
+ })
+
+ t.Run("Edit ancestor, add cascade", func(t *testing.T) {
+ t.Parallel()
+
+ b := newSite(t, true)
+ b.Build(BuildCfg{})
+
+ b.AssertFileContent("public/post/dir/p1/index.html", `Banner: post.jpg`)
+
+ b.EditFiles("content/post/_index.md", indexContentCascade)
+
+ b.Build(BuildCfg{})
+
+ b.AssertFileContent("public/post/index.html", `Banner: post.jpg|Layout: postlayout|Type: posttype|`)
+ b.AssertFileContent("public/post/dir/p1/index.html", `Banner: post.jpg|Layout: postlayout|`)
+ })
+
+ t.Run("Edit ancestor, remove cascade", func(t *testing.T) {
+ t.Parallel()
+
+ b := newSite(t, false)
+ b.Build(BuildCfg{})
+
+ b.AssertFileContent("public/post/dir/p1/index.html", `Banner: |Layout: |`)
+
+ b.EditFiles("content/post/_index.md", indexContentNoCascade)
+
+ b.Build(BuildCfg{})
+
+ b.AssertFileContent("public/post/index.html", `Banner: |Layout: |Type: post|`)
+ b.AssertFileContent("public/post/dir/p1/index.html", `Banner: |Layout: |`)
+ })
+
+ t.Run("Edit ancestor, content only", func(t *testing.T) {
+ t.Parallel()
+
+ b := newSite(t, true)
+ b.Build(BuildCfg{})
+
+ b.EditFiles("content/post/_index.md", indexContentCascade+"\ncontent edit")
+
+ counters := &testCounters{}
+ b.Build(BuildCfg{testCounters: counters})
+ // As we only changed the content, not the cascade front matter, make
+ // only the home page is re-rendered.
+ b.Assert(int(counters.contentRenderCounter), qt.Equals, 1)
+
+ b.AssertFileContent("public/post/index.html", `Banner: post.jpg|Layout: postlayout|Type: posttype|Content: <p>content edit</p>`)
+ b.AssertFileContent("public/post/dir/p1/index.html", `Banner: post.jpg|Layout: postlayout|`)
+ })
}
func newCascadeTestBuilder(t testing.TB, langs []string) *sitesBuilder {
}),
"sect2/p2.md", p(map[string]interface{}{}),
"sect3/p1.md", p(map[string]interface{}{}),
+
+ // No front matter, see #6855
+ "sect3/nofrontmatter.md", `**Hello**`,
+ "sectnocontent/p1.md", `**Hello**`,
+ "sectnofrontmatter/_index.md", `**Hello**`,
+
"sect4/_index.md", p(map[string]interface{}{
"title": "Sect4",
"cascade": map[string]interface{}{
--- /dev/null
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+ "fmt"
+ "path"
+ "path/filepath"
+ "strings"
+ "sync"
+
+ "github.com/gohugoio/hugo/resources/page"
+ "github.com/pkg/errors"
+
+ "github.com/gohugoio/hugo/hugofs/files"
+
+ "github.com/gohugoio/hugo/hugofs"
+
+ radix "github.com/armon/go-radix"
+)
+
+// We store the branch nodes in either the `sections` or `taxonomies` tree
+// with their path as a key; Unix style slashes, a leading slash but no
+// trailing slash.
+//
+// E.g. "/blog" or "/categories/funny"
+//
+// Pages that belongs to a section are stored in the `pages` tree below
+// the section name and a branch separator, e.g. "/blog__hb_". A page is
+// given a key using the path below the section and the base filename with no extension
+// with a leaf separator added.
+//
+// For bundled pages (/mybundle/index.md), we use the folder name.
+//
+// An exmple of a full page key would be "/blog__hb_/page1__hl_"
+//
+// Bundled resources are stored in the `resources` having their path prefixed
+// with the bundle they belong to, e.g.
+// "/blog__hb_/bundle__hl_data.json".
+//
+// The weighted taxonomy entries extracted from page front matter are stored in
+// the `taxonomyEntries` tree below /plural/term/page-key, e.g.
+// "/categories/funny/blog__hb_/bundle__hl_".
+const (
+ cmBranchSeparator = "__hb_"
+ cmLeafSeparator = "__hl_"
+)
+
+// Used to mark ambigous keys in reverse index lookups.
+var ambigousContentNode = &contentNode{}
+
+func newContentMap(cfg contentMapConfig) *contentMap {
+ m := &contentMap{
+ cfg: &cfg,
+ pages: &contentTree{Name: "pages", Tree: radix.New()},
+ sections: &contentTree{Name: "sections", Tree: radix.New()},
+ taxonomies: &contentTree{Name: "taxonomies", Tree: radix.New()},
+ taxonomyEntries: &contentTree{Name: "taxonomyEntries", Tree: radix.New()},
+ resources: &contentTree{Name: "resources", Tree: radix.New()},
+ }
+
+ m.pageTrees = []*contentTree{
+ m.pages, m.sections, m.taxonomies,
+ }
+
+ m.bundleTrees = []*contentTree{
+ m.pages, m.sections, m.taxonomies, m.resources,
+ }
+
+ m.branchTrees = []*contentTree{
+ m.sections, m.taxonomies,
+ }
+
+ addToReverseMap := func(k string, n *contentNode, m map[interface{}]*contentNode) {
+ k = strings.ToLower(k)
+ existing, found := m[k]
+ if found && existing != ambigousContentNode {
+ m[k] = ambigousContentNode
+ } else if !found {
+ m[k] = n
+ }
+ }
+
+ m.pageReverseIndex = &contentTreeReverseIndex{
+ t: []*contentTree{m.pages, m.sections, m.taxonomies},
+ initFn: func(t *contentTree, m map[interface{}]*contentNode) {
+ t.Walk(func(s string, v interface{}) bool {
+ n := v.(*contentNode)
+ if n.p != nil && !n.p.File().IsZero() {
+ meta := n.p.File().FileInfo().Meta()
+ if meta.Path() != meta.PathFile() {
+ // Keep track of the original mount source.
+ mountKey := filepath.ToSlash(filepath.Join(meta.Module(), meta.PathFile()))
+ addToReverseMap(mountKey, n, m)
+ }
+ }
+ k := strings.TrimSuffix(path.Base(s), cmLeafSeparator)
+ addToReverseMap(k, n, m)
+ return false
+ })
+ },
+ }
+
+ return m
+}
+
+type cmInsertKeyBuilder struct {
+ m *contentMap
+
+ err error
+
+ // Builder state
+ tree *contentTree
+ baseKey string // Section or page key
+ key string
+}
+
+func (b cmInsertKeyBuilder) ForPage(s string) *cmInsertKeyBuilder {
+ // TODO2 fmt.Println("ForPage:", s, "baseKey:", b.baseKey, "key:", b.key)
+ baseKey := b.baseKey
+ b.baseKey = s
+
+ if !strings.HasPrefix(s, "/") {
+ s = "/" + s
+ }
+
+ if baseKey != "/" {
+ // Don't repeat the section path in the key.
+ s = strings.TrimPrefix(s, baseKey)
+ }
+
+ switch b.tree {
+ case b.m.sections:
+ b.tree = b.m.pages
+ b.key = baseKey + cmBranchSeparator + s + cmLeafSeparator
+ case b.m.taxonomies:
+ b.key = path.Join(baseKey, s)
+ default:
+ panic("invalid state")
+ }
+
+ return &b
+}
+
+func (b cmInsertKeyBuilder) ForResource(s string) *cmInsertKeyBuilder {
+ // TODO2 fmt.Println("ForResource:", s, "baseKey:", b.baseKey, "key:", b.key)
+
+ s = strings.TrimPrefix(s, "/")
+ s = strings.TrimPrefix(s, strings.TrimPrefix(b.baseKey, "/")+"/")
+
+ switch b.tree {
+ case b.m.pages:
+ b.key = b.key + s
+ case b.m.sections, b.m.taxonomies:
+ b.key = b.key + cmLeafSeparator + s
+ default:
+ panic(fmt.Sprintf("invalid state: %#v", b.tree))
+ }
+ b.tree = b.m.resources
+ return &b
+}
+
+func (b *cmInsertKeyBuilder) Insert(n *contentNode) *cmInsertKeyBuilder {
+ if b.err == nil {
+ b.tree.Insert(cleanTreeKey(b.key), n)
+ }
+ return b
+}
+
+func (b *cmInsertKeyBuilder) DeleteAll() *cmInsertKeyBuilder {
+ if b.err == nil {
+ b.tree.DeletePrefix(cleanTreeKey(b.key))
+ }
+ return b
+}
+
+func (b *cmInsertKeyBuilder) WithFile(fi hugofs.FileMetaInfo) *cmInsertKeyBuilder {
+ b.newTopLevel()
+ m := b.m
+ meta := fi.Meta()
+ p := cleanTreeKey(meta.Path())
+ bundlePath := m.getBundleDir(meta)
+ isBundle := meta.Classifier().IsBundle()
+ if isBundle {
+ panic("not implemented")
+ }
+
+ p, k := b.getBundle(p)
+ if k == "" {
+ b.err = errors.Errorf("no bundle header found for %q", bundlePath)
+ return b
+ }
+
+ id := k + m.reduceKeyPart(p, fi.Meta().Path())
+ b.tree = b.m.resources
+ b.key = id
+ b.baseKey = p
+
+ return b
+}
+
+func (b *cmInsertKeyBuilder) WithSection(s string) *cmInsertKeyBuilder {
+ b.newTopLevel()
+ b.tree = b.m.sections
+ b.baseKey = s
+ b.key = s
+ // TODO2 fmt.Println("WithSection:", s, "baseKey:", b.baseKey, "key:", b.key)
+ return b
+}
+
+func (b *cmInsertKeyBuilder) WithTaxonomy(s string) *cmInsertKeyBuilder {
+ b.newTopLevel()
+ b.tree = b.m.taxonomies
+ b.baseKey = s
+ b.key = s
+ return b
+}
+
+// getBundle gets both the key to the section and the prefix to where to store
+// this page bundle and its resources.
+func (b *cmInsertKeyBuilder) getBundle(s string) (string, string) {
+ m := b.m
+ section, _ := m.getSection(s)
+
+ p := s
+ if section != "/" {
+ p = strings.TrimPrefix(s, section)
+ }
+
+ bundlePathParts := strings.Split(p, "/")[1:]
+ basePath := section + cmBranchSeparator
+
+ // Put it into an existing bundle if found.
+ for i := len(bundlePathParts) - 2; i >= 0; i-- {
+ bundlePath := path.Join(bundlePathParts[:i]...)
+ searchKey := basePath + "/" + bundlePath + cmLeafSeparator
+ if _, found := m.pages.Get(searchKey); found {
+ return section + "/" + bundlePath, searchKey
+ }
+ }
+
+ // Put it into the section bundle.
+ return section, section + cmLeafSeparator
+}
+
+func (b *cmInsertKeyBuilder) newTopLevel() {
+ b.key = ""
+}
+
+type contentBundleViewInfo struct {
+ name viewName
+ termKey string
+ termOrigin string
+ weight int
+ ref *contentNode
+}
+
+func (c *contentBundleViewInfo) kind() string {
+ if c.termKey != "" {
+ return page.KindTaxonomy
+ }
+ return page.KindTaxonomyTerm
+}
+
+func (c *contentBundleViewInfo) sections() []string {
+ if c.kind() == page.KindTaxonomyTerm {
+ return []string{c.name.plural}
+ }
+
+ return []string{c.name.plural, c.termKey}
+
+}
+
+func (c *contentBundleViewInfo) term() string {
+ if c.termOrigin != "" {
+ return c.termOrigin
+ }
+
+ return c.termKey
+}
+
+type contentMap struct {
+ cfg *contentMapConfig
+
+ // View of regular pages, sections, and taxonomies.
+ pageTrees contentTrees
+
+ // View of pages, sections, taxonomies, and resources.
+ bundleTrees contentTrees
+
+ // View of sections and taxonomies.
+ branchTrees contentTrees
+
+ // Stores page bundles keyed by its path's directory or the base filename,
+ // e.g. "blog/post.md" => "/blog/post", "blog/post/index.md" => "/blog/post"
+ // These are the "regular pages" and all of them are bundles.
+ pages *contentTree
+
+ // A reverse index used as a fallback in GetPage.
+ // There are currently two cases where this is used:
+ // 1. Short name lookups in ref/relRef, e.g. using only "mypage.md" without a path.
+ // 2. Links resolved from a remounted content directory. These are restricted to the same module.
+ // Both of the above cases can result in ambigous lookup errors.
+ pageReverseIndex *contentTreeReverseIndex
+
+ // Section nodes.
+ sections *contentTree
+
+ // Taxonomy nodes.
+ taxonomies *contentTree
+
+ // Pages in a taxonomy.
+ taxonomyEntries *contentTree
+
+ // Resources stored per bundle below a common prefix, e.g. "/blog/post__hb_".
+ resources *contentTree
+}
+
+func (m *contentMap) AddFiles(fis ...hugofs.FileMetaInfo) error {
+ for _, fi := range fis {
+ if err := m.addFile(fi); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (m *contentMap) AddFilesBundle(header hugofs.FileMetaInfo, resources ...hugofs.FileMetaInfo) error {
+ var (
+ meta = header.Meta()
+ classifier = meta.Classifier()
+ isBranch = classifier == files.ContentClassBranch
+ bundlePath = m.getBundleDir(meta)
+
+ n = m.newContentNodeFromFi(header)
+ b = m.newKeyBuilder()
+
+ section string
+ )
+
+ if isBranch {
+ // Either a section or a taxonomy node.
+ section = bundlePath
+ if tc := m.cfg.getTaxonomyConfig(section); !tc.IsZero() {
+ term := strings.TrimPrefix(strings.TrimPrefix(section, "/"+tc.plural), "/")
+
+ n.viewInfo = &contentBundleViewInfo{
+ name: tc,
+ termKey: term,
+ termOrigin: term,
+ }
+
+ n.viewInfo.ref = n
+ b.WithTaxonomy(section).Insert(n)
+ } else {
+ b.WithSection(section).Insert(n)
+ }
+ } else {
+ // A regular page. Attach it to its section.
+ section, _ = m.getOrCreateSection(n, bundlePath)
+ b = b.WithSection(section).ForPage(bundlePath).Insert(n)
+ }
+
+ if m.cfg.isRebuild {
+ // The resource owner will be either deleted or overwritten on rebuilds,
+ // but make sure we handle deletion of resources (images etc.) as well.
+ b.ForResource("").DeleteAll()
+ }
+
+ for _, r := range resources {
+ rb := b.ForResource(cleanTreeKey(r.Meta().Path()))
+ rb.Insert(&contentNode{fi: r})
+ }
+
+ return nil
+
+}
+
+func (m *contentMap) CreateMissingNodes() error {
+ // Create missing home and root sections
+ rootSections := make(map[string]interface{})
+ trackRootSection := func(s string, b *contentNode) {
+ parts := strings.Split(s, "/")
+ if len(parts) > 2 {
+ root := strings.TrimSuffix(parts[1], cmBranchSeparator)
+ if root != "" {
+ if _, found := rootSections[root]; !found {
+ rootSections[root] = b
+ }
+ }
+ }
+ }
+
+ m.sections.Walk(func(s string, v interface{}) bool {
+ n := v.(*contentNode)
+
+ if s == "/" {
+ return false
+ }
+
+ trackRootSection(s, n)
+ return false
+ })
+
+ m.pages.Walk(func(s string, v interface{}) bool {
+ trackRootSection(s, v.(*contentNode))
+ return false
+ })
+
+ if _, found := rootSections["/"]; !found {
+ rootSections["/"] = true
+ }
+
+ for sect, v := range rootSections {
+ var sectionPath string
+ if n, ok := v.(*contentNode); ok && n.path != "" {
+ sectionPath = n.path
+ firstSlash := strings.Index(sectionPath, "/")
+ if firstSlash != -1 {
+ sectionPath = sectionPath[:firstSlash]
+ }
+ }
+ sect = cleanTreeKey(sect)
+ _, found := m.sections.Get(sect)
+ if !found {
+ m.sections.Insert(sect, &contentNode{path: sectionPath})
+ }
+ }
+
+ for _, view := range m.cfg.taxonomyConfig {
+ s := cleanTreeKey(view.plural)
+ _, found := m.taxonomies.Get(s)
+ if !found {
+ b := &contentNode{
+ viewInfo: &contentBundleViewInfo{
+ name: view,
+ },
+ }
+ b.viewInfo.ref = b
+ m.taxonomies.Insert(s, b)
+ }
+ }
+
+ return nil
+
+}
+
+func (m *contentMap) getBundleDir(meta hugofs.FileMeta) string {
+ dir := cleanTreeKey(filepath.Dir(meta.Path()))
+
+ switch meta.Classifier() {
+ case files.ContentClassContent:
+ return path.Join(dir, meta.TranslationBaseName())
+ default:
+ return dir
+ }
+}
+
+func (m *contentMap) newContentNodeFromFi(fi hugofs.FileMetaInfo) *contentNode {
+ return &contentNode{
+ fi: fi,
+ path: strings.TrimPrefix(filepath.ToSlash(fi.Meta().Path()), "/"),
+ }
+}
+
+func (m *contentMap) getFirstSection(s string) (string, *contentNode) {
+ for {
+ k, v, found := m.sections.LongestPrefix(s)
+ if !found {
+ return "", nil
+ }
+ if strings.Count(k, "/") == 1 {
+ return k, v.(*contentNode)
+ }
+ s = path.Dir(s)
+ }
+}
+
+func (m *contentMap) newKeyBuilder() *cmInsertKeyBuilder {
+ return &cmInsertKeyBuilder{m: m}
+}
+
+func (m *contentMap) getOrCreateSection(n *contentNode, s string) (string, *contentNode) {
+ level := strings.Count(s, "/")
+ k, b := m.getSection(s)
+
+ mustCreate := false
+
+ if k == "" {
+ mustCreate = true
+ } else if level > 1 && k == "/" {
+ // We found the home section, but this page needs to be placed in
+ // the root, e.g. "/blog", section.
+ mustCreate = true
+ }
+
+ if mustCreate {
+ k = s[:strings.Index(s[1:], "/")+1]
+ if k == "" {
+ k = "/"
+ }
+
+ b = &contentNode{
+ path: n.rootSection(),
+ }
+
+ m.sections.Insert(k, b)
+ }
+
+ return k, b
+}
+
+func (m *contentMap) getPage(section, name string) *contentNode {
+ key := section + cmBranchSeparator + "/" + name + cmLeafSeparator
+ v, found := m.pages.Get(key)
+ if found {
+ return v.(*contentNode)
+ }
+ return nil
+}
+
+func (m *contentMap) getSection(s string) (string, *contentNode) {
+ k, v, found := m.sections.LongestPrefix(path.Dir(s))
+ if found {
+ return k, v.(*contentNode)
+ }
+ return "", nil
+}
+
+func (m *contentMap) getTaxonomyParent(s string) (string, *contentNode) {
+ s = path.Dir(s)
+ if s == "/" {
+ v, found := m.sections.Get(s)
+ if found {
+ return s, v.(*contentNode)
+ }
+ return "", nil
+ }
+
+ for _, tree := range []*contentTree{m.taxonomies, m.sections} {
+ k, v, found := tree.LongestPrefix(s)
+ if found {
+ return k, v.(*contentNode)
+ }
+ }
+ return "", nil
+}
+
+func (m *contentMap) addFile(fi hugofs.FileMetaInfo) error {
+ b := m.newKeyBuilder()
+ return b.WithFile(fi).Insert(m.newContentNodeFromFi(fi)).err
+}
+
+func cleanTreeKey(k string) string {
+ k = "/" + strings.ToLower(strings.Trim(path.Clean(filepath.ToSlash(k)), "./"))
+ return k
+}
+
+func (m *contentMap) onSameLevel(s1, s2 string) bool {
+ return strings.Count(s1, "/") == strings.Count(s2, "/")
+}
+
+func (m *contentMap) deleteBundleMatching(matches func(b *contentNode) bool) {
+ // Check sections first
+ s := m.sections.getMatch(matches)
+ if s != "" {
+ m.deleteSectionByPath(s)
+ return
+ }
+
+ s = m.pages.getMatch(matches)
+ if s != "" {
+ m.deletePage(s)
+ return
+ }
+
+ s = m.resources.getMatch(matches)
+ if s != "" {
+ m.resources.Delete(s)
+ }
+
+}
+
+// Deletes any empty root section that's not backed by a content file.
+func (m *contentMap) deleteOrphanSections() {
+
+ m.sections.Walk(func(s string, v interface{}) bool {
+ n := v.(*contentNode)
+
+ if n.fi != nil {
+ // Section may be empty, but is backed by a content file.
+ return false
+ }
+
+ if s == "/" || strings.Count(s, "/") > 1 {
+ return false
+ }
+
+ prefixBundle := s + cmBranchSeparator
+
+ if !(m.sections.hasPrefix(s+"/") || m.pages.hasPrefix(prefixBundle) || m.resources.hasPrefix(prefixBundle)) {
+ m.sections.Delete(s)
+ }
+
+ return false
+ })
+}
+
+func (m *contentMap) deletePage(s string) {
+ m.pages.DeletePrefix(s)
+ m.resources.DeletePrefix(s)
+}
+
+func (m *contentMap) deleteSectionByPath(s string) {
+ m.sections.Delete(s)
+ m.sections.DeletePrefix(s + "/")
+ m.pages.DeletePrefix(s + cmBranchSeparator)
+ m.pages.DeletePrefix(s + "/")
+ m.resources.DeletePrefix(s + cmBranchSeparator)
+ m.resources.DeletePrefix(s + cmLeafSeparator)
+ m.resources.DeletePrefix(s + "/")
+}
+
+func (m *contentMap) deletePageByPath(s string) {
+ m.pages.Walk(func(s string, v interface{}) bool {
+ fmt.Println("S", s)
+
+ return false
+ })
+}
+
+func (m *contentMap) deleteTaxonomy(s string) {
+ m.taxonomies.Delete(s)
+ m.taxonomies.DeletePrefix(s + "/")
+}
+
+func (m *contentMap) reduceKeyPart(dir, filename string) string {
+ dir, filename = filepath.ToSlash(dir), filepath.ToSlash(filename)
+ dir, filename = strings.TrimPrefix(dir, "/"), strings.TrimPrefix(filename, "/")
+
+ return strings.TrimPrefix(strings.TrimPrefix(filename, dir), "/")
+}
+
+func (m *contentMap) splitKey(k string) []string {
+ if k == "" || k == "/" {
+ return nil
+ }
+
+ return strings.Split(k, "/")[1:]
+
+}
+
+func (m *contentMap) testDump() string {
+ var sb strings.Builder
+
+ for i, r := range []*contentTree{m.pages, m.sections, m.resources} {
+ sb.WriteString(fmt.Sprintf("Tree %d:\n", i))
+ r.Walk(func(s string, v interface{}) bool {
+ sb.WriteString("\t" + s + "\n")
+ return false
+ })
+ }
+
+ for i, r := range []*contentTree{m.pages, m.sections} {
+
+ r.Walk(func(s string, v interface{}) bool {
+ c := v.(*contentNode)
+ cpToString := func(c *contentNode) string {
+ var sb strings.Builder
+ if c.p != nil {
+ sb.WriteString("|p:" + c.p.Title())
+ }
+ if c.fi != nil {
+ sb.WriteString("|f:" + filepath.ToSlash(c.fi.Meta().Path()))
+ }
+ return sb.String()
+ }
+ sb.WriteString(path.Join(m.cfg.lang, r.Name) + s + cpToString(c) + "\n")
+
+ resourcesPrefix := s
+
+ if i == 1 {
+ resourcesPrefix += cmLeafSeparator
+
+ m.pages.WalkPrefix(s+cmBranchSeparator, func(s string, v interface{}) bool {
+ sb.WriteString("\t - P: " + filepath.ToSlash((v.(*contentNode).fi.(hugofs.FileMetaInfo)).Meta().Filename()) + "\n")
+ return false
+ })
+ }
+
+ m.resources.WalkPrefix(resourcesPrefix, func(s string, v interface{}) bool {
+ sb.WriteString("\t - R: " + filepath.ToSlash((v.(*contentNode).fi.(hugofs.FileMetaInfo)).Meta().Filename()) + "\n")
+ return false
+
+ })
+
+ return false
+ })
+ }
+
+ return sb.String()
+
+}
+
+type contentMapConfig struct {
+ lang string
+ taxonomyConfig []viewName
+ taxonomyDisabled bool
+ taxonomyTermDisabled bool
+ pageDisabled bool
+ isRebuild bool
+}
+
+func (cfg contentMapConfig) getTaxonomyConfig(s string) (v viewName) {
+ s = strings.TrimPrefix(s, "/")
+ if s == "" {
+ return
+ }
+ for _, n := range cfg.taxonomyConfig {
+ if strings.HasPrefix(s, n.plural) {
+ return n
+ }
+ }
+
+ return
+}
+
+type contentNode struct {
+ p *pageState
+
+ // Set for taxonomy nodes.
+ viewInfo *contentBundleViewInfo
+
+ // Set if source is a file.
+ // We will soon get other sources.
+ fi hugofs.FileMetaInfo
+
+ // The source path. Unix slashes. No leading slash.
+ path string
+}
+
+func (b *contentNode) rootSection() string {
+ if b.path == "" {
+ return ""
+ }
+ firstSlash := strings.Index(b.path, "/")
+ if firstSlash == -1 {
+ return b.path
+ }
+ return b.path[:firstSlash]
+
+}
+
+type contentTree struct {
+ Name string
+ *radix.Tree
+}
+
+type contentTrees []*contentTree
+
+func (t contentTrees) DeletePrefix(prefix string) int {
+ var count int
+ for _, tree := range t {
+ tree.Walk(func(s string, v interface{}) bool {
+ return false
+ })
+ count += tree.DeletePrefix(prefix)
+ }
+ return count
+}
+
+type contentTreeNodeCallback func(s string, n *contentNode) bool
+
+var (
+ contentTreeNoListFilter = func(s string, n *contentNode) bool {
+ if n.p == nil {
+ return true
+ }
+ return n.p.m.noList()
+ }
+
+ contentTreeNoRenderFilter = func(s string, n *contentNode) bool {
+ if n.p == nil {
+ return true
+ }
+ return n.p.m.noRender()
+ }
+)
+
+func (c *contentTree) WalkPrefixListable(prefix string, fn contentTreeNodeCallback) {
+ c.WalkPrefixFilter(prefix, contentTreeNoListFilter, fn)
+}
+
+func (c *contentTree) WalkPrefixFilter(prefix string, filter, walkFn contentTreeNodeCallback) {
+ c.WalkPrefix(prefix, func(s string, v interface{}) bool {
+ n := v.(*contentNode)
+ if filter(s, n) {
+ return false
+ }
+ return walkFn(s, n)
+ })
+}
+
+func (c *contentTree) WalkListable(fn contentTreeNodeCallback) {
+ c.WalkFilter(contentTreeNoListFilter, fn)
+}
+
+func (c *contentTree) WalkFilter(filter, walkFn contentTreeNodeCallback) {
+ c.Walk(func(s string, v interface{}) bool {
+ n := v.(*contentNode)
+ if filter(s, n) {
+ return false
+ }
+ return walkFn(s, n)
+ })
+}
+
+func (c contentTrees) WalkListable(fn contentTreeNodeCallback) {
+ for _, tree := range c {
+ tree.WalkListable(fn)
+ }
+}
+
+func (c contentTrees) WalkRenderable(fn contentTreeNodeCallback) {
+ for _, tree := range c {
+ tree.WalkFilter(contentTreeNoRenderFilter, fn)
+ }
+}
+
+func (c contentTrees) Walk(fn contentTreeNodeCallback) {
+ for _, tree := range c {
+ tree.Walk(func(s string, v interface{}) bool {
+ n := v.(*contentNode)
+ return fn(s, n)
+ })
+ }
+}
+
+func (c contentTrees) WalkPrefix(prefix string, fn contentTreeNodeCallback) {
+ for _, tree := range c {
+ tree.WalkPrefix(prefix, func(s string, v interface{}) bool {
+ n := v.(*contentNode)
+ return fn(s, n)
+ })
+ }
+}
+
+func (c *contentTree) getMatch(matches func(b *contentNode) bool) string {
+ var match string
+ c.Walk(func(s string, v interface{}) bool {
+ n, ok := v.(*contentNode)
+ if !ok {
+ return false
+ }
+
+ if matches(n) {
+ match = s
+ return true
+ }
+
+ return false
+ })
+
+ return match
+}
+
+func (c *contentTree) hasPrefix(s string) bool {
+ var t bool
+ c.Tree.WalkPrefix(s, func(s string, v interface{}) bool {
+ t = true
+ return true
+ })
+ return t
+}
+
+func (c *contentTree) printKeys() {
+ c.Walk(func(s string, v interface{}) bool {
+ fmt.Println(s)
+ return false
+ })
+}
+
+func (c *contentTree) printKeysPrefix(prefix string) {
+ c.WalkPrefix(prefix, func(s string, v interface{}) bool {
+ fmt.Println(s)
+ return false
+ })
+}
+
+// contentTreeRef points to a node in the given tree.
+type contentTreeRef struct {
+ m *pageMap
+ t *contentTree
+ n *contentNode
+ key string
+}
+
+func (c *contentTreeRef) getCurrentSection() (string, *contentNode) {
+ if c.isSection() {
+ return c.key, c.n
+ }
+ return c.getSection()
+}
+
+func (c *contentTreeRef) isSection() bool {
+ return c.t == c.m.sections
+}
+
+func (c *contentTreeRef) getSection() (string, *contentNode) {
+ return c.m.getSection(c.key)
+}
+
+func (c *contentTreeRef) collectPages() page.Pages {
+ var pas page.Pages
+ c.m.collectPages(c.key+cmBranchSeparator, func(c *contentNode) {
+ pas = append(pas, c.p)
+ })
+ page.SortByDefault(pas)
+
+ return pas
+}
+
+func (c *contentTreeRef) collectPagesAndSections() page.Pages {
+ var pas page.Pages
+ c.m.collectPagesAndSections(c.key, func(c *contentNode) {
+ pas = append(pas, c.p)
+ })
+ page.SortByDefault(pas)
+
+ return pas
+}
+
+func (c *contentTreeRef) collectSections() page.Pages {
+ var pas page.Pages
+ c.m.collectSections(c.key, func(c *contentNode) {
+ pas = append(pas, c.p)
+ })
+ page.SortByDefault(pas)
+
+ return pas
+}
+
+type contentTreeReverseIndex struct {
+ t []*contentTree
+ m map[interface{}]*contentNode
+
+ init sync.Once
+ initFn func(*contentTree, map[interface{}]*contentNode)
+}
+
+func (c *contentTreeReverseIndex) Get(key interface{}) *contentNode {
+ c.init.Do(func() {
+ c.m = make(map[interface{}]*contentNode)
+ for _, tree := range c.t {
+ c.initFn(tree, c.m)
+ }
+ })
+ return c.m[key]
+}
--- /dev/null
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+ "context"
+ "fmt"
+ "path"
+ "path/filepath"
+ "strings"
+ "sync"
+
+ "github.com/gohugoio/hugo/common/maps"
+
+ "github.com/gohugoio/hugo/common/types"
+ "github.com/gohugoio/hugo/resources"
+
+ "github.com/gohugoio/hugo/common/hugio"
+ "github.com/gohugoio/hugo/hugofs"
+ "github.com/gohugoio/hugo/hugofs/files"
+ "github.com/gohugoio/hugo/parser/pageparser"
+ "github.com/gohugoio/hugo/resources/page"
+ "github.com/gohugoio/hugo/resources/resource"
+ "github.com/spf13/cast"
+
+ "github.com/gohugoio/hugo/common/para"
+ "github.com/pkg/errors"
+)
+
+func newPageMaps(h *HugoSites) *pageMaps {
+ mps := make([]*pageMap, len(h.Sites))
+ for i, s := range h.Sites {
+ mps[i] = s.pageMap
+ }
+ return &pageMaps{
+ workers: para.New(h.numWorkers),
+ pmaps: mps,
+ }
+
+}
+
+type pageMap struct {
+ s *Site
+ *contentMap
+}
+
+func (m *pageMap) Len() int {
+ l := 0
+ for _, t := range m.contentMap.pageTrees {
+ l += t.Len()
+ }
+ return l
+}
+
+func (m *pageMap) createMissingTaxonomyNodes() error {
+ if m.cfg.taxonomyDisabled {
+ return nil
+ }
+ m.taxonomyEntries.Walk(func(s string, v interface{}) bool {
+ n := v.(*contentNode)
+ vi := n.viewInfo
+ k := cleanTreeKey(vi.name.plural + "/" + vi.termKey)
+
+ if _, found := m.taxonomies.Get(k); !found {
+ vic := &contentBundleViewInfo{
+ name: vi.name,
+ termKey: vi.termKey,
+ termOrigin: vi.termOrigin,
+ }
+ m.taxonomies.Insert(k, &contentNode{viewInfo: vic})
+ }
+ return false
+ })
+
+ return nil
+}
+
+func (m *pageMap) newPageFromContentNode(n *contentNode, parentBucket *pagesMapBucket, owner *pageState) (*pageState, error) {
+ if n.fi == nil {
+ panic("FileInfo must (currently) be set")
+ }
+
+ f, err := newFileInfo(m.s.SourceSpec, n.fi)
+ if err != nil {
+ return nil, err
+ }
+
+ meta := n.fi.Meta()
+ content := func() (hugio.ReadSeekCloser, error) {
+ return meta.Open()
+ }
+
+ bundled := owner != nil
+ s := m.s
+
+ sections := s.sectionsFromFile(f)
+
+ kind := s.kindFromFileInfoOrSections(f, sections)
+ if kind == page.KindTaxonomy {
+ s.PathSpec.MakePathsSanitized(sections)
+ }
+
+ metaProvider := &pageMeta{kind: kind, sections: sections, bundled: bundled, s: s, f: f}
+
+ ps, err := newPageBase(metaProvider)
+ if err != nil {
+ return nil, err
+ }
+
+ if n.fi.Meta().GetBool(walkIsRootFileMetaKey) {
+ // Make sure that the bundle/section we start walking from is always
+ // rendered.
+ // This is only relevant in server fast render mode.
+ ps.forceRender = true
+ }
+
+ n.p = ps
+ if ps.IsNode() {
+ ps.bucket = newPageBucket(ps)
+ }
+
+ gi, err := s.h.gitInfoForPage(ps)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to load Git data")
+ }
+ ps.gitInfo = gi
+
+ r, err := content()
+ if err != nil {
+ return nil, err
+ }
+ defer r.Close()
+
+ parseResult, err := pageparser.Parse(
+ r,
+ pageparser.Config{EnableEmoji: s.siteCfg.enableEmoji},
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ ps.pageContent = pageContent{
+ source: rawPageContent{
+ parsed: parseResult,
+ posMainContent: -1,
+ posSummaryEnd: -1,
+ posBodyStart: -1,
+ },
+ }
+
+ ps.shortcodeState = newShortcodeHandler(ps, ps.s, nil)
+
+ if err := ps.mapContent(parentBucket, metaProvider); err != nil {
+ return nil, ps.wrapError(err)
+ }
+
+ if err := metaProvider.applyDefaultValues(n); err != nil {
+ return nil, err
+ }
+
+ ps.init.Add(func() (interface{}, error) {
+ pp, err := newPagePaths(s, ps, metaProvider)
+ if err != nil {
+ return nil, err
+ }
+
+ outputFormatsForPage := ps.m.outputFormats()
+
+ if !ps.m.noRender() {
+ // Prepare output formats for all sites.
+ ps.pageOutputs = make([]*pageOutput, len(ps.s.h.renderFormats))
+ created := make(map[string]*pageOutput)
+
+ for i, f := range ps.s.h.renderFormats {
+ if po, found := created[f.Name]; found {
+ ps.pageOutputs[i] = po
+ continue
+ }
+
+ _, render := outputFormatsForPage.GetByName(f.Name)
+ po := newPageOutput(ps, pp, f, render)
+
+ // Create a content provider for the first,
+ // we may be able to reuse it.
+ if i == 0 {
+ contentProvider, err := newPageContentOutput(ps, po)
+ if err != nil {
+ return nil, err
+ }
+ po.initContentProvider(contentProvider)
+ }
+
+ ps.pageOutputs[i] = po
+ created[f.Name] = po
+ }
+ } else if ps.m.buildConfig.PublishResources {
+ // We need one output format for potential resources to publish.
+ po := newPageOutput(ps, pp, outputFormatsForPage[0], false)
+ contentProvider, err := newPageContentOutput(ps, po)
+ if err != nil {
+ return nil, err
+ }
+ po.initContentProvider(contentProvider)
+ ps.pageOutputs = []*pageOutput{po}
+ }
+
+ if err := ps.initCommonProviders(pp); err != nil {
+ return nil, err
+ }
+
+ return nil, nil
+ })
+
+ ps.parent = owner
+
+ return ps, nil
+}
+
+func (m *pageMap) newResource(fim hugofs.FileMetaInfo, owner *pageState) (resource.Resource, error) {
+
+ if owner == nil {
+ panic("owner is nil")
+ }
+ // TODO(bep) consolidate with multihost logic + clean up
+ outputFormats := owner.m.outputFormats()
+ seen := make(map[string]bool)
+ var targetBasePaths []string
+ // Make sure bundled resources are published to all of the ouptput formats'
+ // sub paths.
+ for _, f := range outputFormats {
+ p := f.Path
+ if seen[p] {
+ continue
+ }
+ seen[p] = true
+ targetBasePaths = append(targetBasePaths, p)
+
+ }
+
+ meta := fim.Meta()
+ r := func() (hugio.ReadSeekCloser, error) {
+ return meta.Open()
+ }
+
+ target := strings.TrimPrefix(meta.Path(), owner.File().Dir())
+
+ return owner.s.ResourceSpec.New(
+ resources.ResourceSourceDescriptor{
+ TargetPaths: owner.getTargetPaths,
+ OpenReadSeekCloser: r,
+ FileInfo: fim,
+ RelTargetFilename: target,
+ TargetBasePaths: targetBasePaths,
+ })
+}
+
+func (m *pageMap) createSiteTaxonomies() error {
+ m.s.taxonomies = make(TaxonomyList)
+ m.taxonomies.Walk(func(s string, v interface{}) bool {
+ n := v.(*contentNode)
+ t := n.viewInfo
+
+ viewName := t.name
+
+ if t.termKey == "" {
+ m.s.taxonomies[viewName.plural] = make(Taxonomy)
+ } else {
+ taxonomy := m.s.taxonomies[viewName.plural]
+ m.taxonomyEntries.WalkPrefix(s+"/", func(ss string, v interface{}) bool {
+ b2 := v.(*contentNode)
+ info := b2.viewInfo
+ taxonomy.add(info.termKey, page.NewWeightedPage(info.weight, info.ref.p, n.p))
+
+ return false
+ })
+ }
+
+ return false
+ })
+
+ for _, taxonomy := range m.s.taxonomies {
+ for _, v := range taxonomy {
+ v.Sort()
+ }
+ }
+
+ return nil
+}
+
+func (m *pageMap) createListAllPages() page.Pages {
+ pages := make(page.Pages, 0)
+
+ m.contentMap.pageTrees.Walk(func(s string, n *contentNode) bool {
+ if n.p == nil {
+ panic(fmt.Sprintf("BUG: page not set for %q", s))
+ }
+ if contentTreeNoListFilter(s, n) {
+ return false
+ }
+ pages = append(pages, n.p)
+ return false
+ })
+
+ page.SortByDefault(pages)
+ return pages
+}
+
+func (m *pageMap) assemblePages() error {
+ m.taxonomyEntries.DeletePrefix("/")
+
+ if err := m.assembleSections(); err != nil {
+ return err
+ }
+
+ var err error
+
+ if err != nil {
+ return err
+ }
+
+ m.pages.Walk(func(s string, v interface{}) bool {
+ n := v.(*contentNode)
+
+ var shouldBuild bool
+
+ defer func() {
+ // Make sure we always rebuild the view cache.
+ if shouldBuild && err == nil && n.p != nil {
+ m.attachPageToViews(s, n)
+ }
+ }()
+
+ if n.p != nil {
+ // A rebuild
+ shouldBuild = true
+ return false
+ }
+
+ var parent *contentNode
+ var parentBucket *pagesMapBucket
+
+ _, parent = m.getSection(s)
+ if parent == nil {
+ panic(fmt.Sprintf("BUG: parent not set for %q", s))
+ }
+ parentBucket = parent.p.bucket
+
+ n.p, err = m.newPageFromContentNode(n, parentBucket, nil)
+ if err != nil {
+ return true
+ }
+
+ shouldBuild = !(n.p.Kind() == page.KindPage && m.cfg.pageDisabled) && m.s.shouldBuild(n.p)
+ if !shouldBuild {
+ m.deletePage(s)
+ return false
+ }
+
+ n.p.treeRef = &contentTreeRef{
+ m: m,
+ t: m.pages,
+ n: n,
+ key: s,
+ }
+
+ if err = m.assembleResources(s, n.p, parentBucket); err != nil {
+ return true
+ }
+
+ return false
+ })
+
+ m.deleteOrphanSections()
+
+ return err
+}
+
+func (m *pageMap) assembleResources(s string, p *pageState, parentBucket *pagesMapBucket) error {
+ var err error
+
+ m.resources.WalkPrefix(s, func(s string, v interface{}) bool {
+ n := v.(*contentNode)
+ meta := n.fi.Meta()
+ classifier := meta.Classifier()
+ var r resource.Resource
+ switch classifier {
+ case files.ContentClassContent:
+ var rp *pageState
+ rp, err = m.newPageFromContentNode(n, parentBucket, p)
+ if err != nil {
+ return true
+ }
+ rp.m.resourcePath = filepath.ToSlash(strings.TrimPrefix(rp.Path(), p.File().Dir()))
+ r = rp
+
+ case files.ContentClassFile:
+ r, err = m.newResource(n.fi, p)
+ if err != nil {
+ return true
+ }
+ default:
+ panic(fmt.Sprintf("invalid classifier: %q", classifier))
+ }
+
+ p.resources = append(p.resources, r)
+ return false
+ })
+
+ return err
+}
+
+func (m *pageMap) assembleSections() error {
+
+ var sectionsToDelete []string
+ var err error
+
+ m.sections.Walk(func(s string, v interface{}) bool {
+ n := v.(*contentNode)
+
+ var shouldBuild bool
+
+ defer func() {
+ // Make sure we always rebuild the view cache.
+ if shouldBuild && err == nil && n.p != nil {
+ m.attachPageToViews(s, n)
+ if n.p.IsHome() {
+ m.s.home = n.p
+ }
+ }
+ }()
+
+ sections := m.splitKey(s)
+
+ if n.p != nil {
+ if n.p.IsHome() {
+ m.s.home = n.p
+ }
+ shouldBuild = true
+ return false
+ }
+
+ var parent *contentNode
+ var parentBucket *pagesMapBucket
+
+ if s != "/" {
+ _, parent = m.getSection(s)
+ if parent == nil || parent.p == nil {
+ panic(fmt.Sprintf("BUG: parent not set for %q", s))
+ }
+ }
+
+ if parent != nil {
+ parentBucket = parent.p.bucket
+ }
+
+ kind := page.KindSection
+ if s == "/" {
+ kind = page.KindHome
+ }
+
+ if n.fi != nil {
+ n.p, err = m.newPageFromContentNode(n, parentBucket, nil)
+ if err != nil {
+ return true
+ }
+ } else {
+ n.p = m.s.newPage(n, parentBucket, kind, "", sections...)
+ }
+
+ shouldBuild = m.s.shouldBuild(n.p)
+ if !shouldBuild {
+ sectionsToDelete = append(sectionsToDelete, s)
+ return false
+ }
+
+ n.p.treeRef = &contentTreeRef{
+ m: m,
+ t: m.sections,
+ n: n,
+ key: s,
+ }
+
+ if err = m.assembleResources(s+cmLeafSeparator, n.p, parentBucket); err != nil {
+ return true
+ }
+
+ return false
+ })
+
+ for _, s := range sectionsToDelete {
+ m.deleteSectionByPath(s)
+ }
+
+ return err
+}
+
+func (m *pageMap) assembleTaxonomies() error {
+
+ var taxonomiesToDelete []string
+ var err error
+
+ m.taxonomies.Walk(func(s string, v interface{}) bool {
+ n := v.(*contentNode)
+
+ if n.p != nil {
+ return false
+ }
+
+ kind := n.viewInfo.kind()
+ sections := n.viewInfo.sections()
+
+ _, parent := m.getTaxonomyParent(s)
+ if parent == nil || parent.p == nil {
+ panic(fmt.Sprintf("BUG: parent not set for %q", s))
+ }
+ parentBucket := parent.p.bucket
+
+ if n.fi != nil {
+ n.p, err = m.newPageFromContentNode(n, parent.p.bucket, nil)
+ if err != nil {
+ return true
+ }
+ } else {
+ title := ""
+ if kind == page.KindTaxonomy {
+ title = n.viewInfo.term()
+ }
+ n.p = m.s.newPage(n, parent.p.bucket, kind, title, sections...)
+ }
+
+ if !m.s.shouldBuild(n.p) {
+ taxonomiesToDelete = append(taxonomiesToDelete, s)
+ return false
+ }
+
+ n.p.treeRef = &contentTreeRef{
+ m: m,
+ t: m.taxonomies,
+ n: n,
+ key: s,
+ }
+
+ if err = m.assembleResources(s+cmLeafSeparator, n.p, parentBucket); err != nil {
+ return true
+ }
+
+ return false
+ })
+
+ for _, s := range taxonomiesToDelete {
+ m.deleteTaxonomy(s)
+ }
+
+ return err
+
+}
+
+func (m *pageMap) attachPageToViews(s string, b *contentNode) {
+ if m.cfg.taxonomyDisabled {
+ return
+ }
+
+ for _, viewName := range m.cfg.taxonomyConfig {
+ vals := types.ToStringSlicePreserveString(getParam(b.p, viewName.plural, false))
+ if vals == nil {
+ continue
+ }
+
+ w := getParamToLower(b.p, viewName.plural+"_weight")
+ weight, err := cast.ToIntE(w)
+ if err != nil {
+ m.s.Log.ERROR.Printf("Unable to convert taxonomy weight %#v to int for %q", w, b.p.Path())
+ // weight will equal zero, so let the flow continue
+ }
+
+ for _, v := range vals {
+ termKey := m.s.getTaxonomyKey(v)
+
+ bv := &contentNode{
+ viewInfo: &contentBundleViewInfo{
+ name: viewName,
+ termKey: termKey,
+ termOrigin: v,
+ weight: weight,
+ ref: b,
+ },
+ }
+
+ if s == "/" {
+ // To avoid getting an empty key.
+ s = "home"
+ }
+ key := cleanTreeKey(path.Join(viewName.plural, termKey, s))
+ m.taxonomyEntries.Insert(key, bv)
+ }
+ }
+}
+
+func (m *pageMap) collectPages(prefix string, fn func(c *contentNode)) error {
+ m.pages.WalkPrefixListable(prefix, func(s string, n *contentNode) bool {
+ fn(n)
+ return false
+ })
+ return nil
+}
+
+func (m *pageMap) collectPagesAndSections(prefix string, fn func(c *contentNode)) error {
+ if err := m.collectSections(prefix, fn); err != nil {
+ return err
+ }
+
+ if err := m.collectPages(prefix+cmBranchSeparator, fn); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (m *pageMap) collectSections(prefix string, fn func(c *contentNode)) error {
+ var level int
+ isHome := prefix == "/"
+
+ if !isHome {
+ level = strings.Count(prefix, "/")
+ }
+
+ return m.collectSectionsFn(prefix, func(s string, c *contentNode) bool {
+ if s == prefix {
+ return false
+ }
+
+ if (strings.Count(s, "/") - level) != 1 {
+ return false
+ }
+
+ fn(c)
+
+ return false
+ })
+}
+
+func (m *pageMap) collectSectionsFn(prefix string, fn func(s string, c *contentNode) bool) error {
+ if !strings.HasSuffix(prefix, "/") {
+ prefix += "/"
+ }
+
+ m.sections.WalkPrefixListable(prefix, func(s string, n *contentNode) bool {
+ return fn(s, n)
+ })
+
+ return nil
+}
+
+func (m *pageMap) collectSectionsRecursiveIncludingSelf(prefix string, fn func(c *contentNode)) error {
+ return m.collectSectionsFn(prefix, func(s string, c *contentNode) bool {
+ fn(c)
+ return false
+ })
+}
+
+func (m *pageMap) collectTaxonomies(prefix string, fn func(c *contentNode)) error {
+ m.taxonomies.WalkPrefixListable(prefix, func(s string, n *contentNode) bool {
+ fn(n)
+ return false
+ })
+ return nil
+}
+
+// withEveryBundlePage applies fn to every Page, including those bundled inside
+// leaf bundles.
+func (m *pageMap) withEveryBundlePage(fn func(p *pageState) bool) {
+ m.bundleTrees.Walk(func(s string, n *contentNode) bool {
+ if n.p != nil {
+ return fn(n.p)
+ }
+ return false
+ })
+}
+
+type pageMaps struct {
+ workers *para.Workers
+ pmaps []*pageMap
+}
+
+// deleteSection deletes the entire section from s.
+func (m *pageMaps) deleteSection(s string) {
+ m.withMaps(func(pm *pageMap) error {
+ pm.deleteSectionByPath(s)
+ return nil
+ })
+}
+
+func (m *pageMaps) AssemblePages() error {
+ return m.withMaps(func(pm *pageMap) error {
+ if err := pm.CreateMissingNodes(); err != nil {
+ return err
+ }
+
+ if err := pm.assemblePages(); err != nil {
+ return err
+ }
+
+ if err := pm.createMissingTaxonomyNodes(); err != nil {
+ return err
+ }
+
+ // Handle any new sections created in the step above.
+ if err := pm.assembleSections(); err != nil {
+ return err
+ }
+
+ if err := pm.assembleTaxonomies(); err != nil {
+ return err
+ }
+
+ if err := pm.createSiteTaxonomies(); err != nil {
+ return err
+ }
+
+ a := (§ionWalker{m: pm.contentMap}).applyAggregates()
+ _, mainSectionsSet := pm.s.s.Info.Params()["mainsections"]
+ if !mainSectionsSet && a.mainSection != "" {
+ mainSections := []string{a.mainSection}
+ pm.s.s.Info.Params()["mainSections"] = mainSections
+ pm.s.s.Info.Params()["mainsections"] = mainSections
+ }
+
+ pm.s.lastmod = a.datesAll.Lastmod()
+ if resource.IsZeroDates(pm.s.home) {
+ pm.s.home.m.Dates = a.datesAll
+ }
+
+ return nil
+ })
+}
+
+func (m *pageMaps) walkBundles(fn func(n *contentNode) bool) {
+ _ = m.withMaps(func(pm *pageMap) error {
+ pm.bundleTrees.Walk(func(s string, n *contentNode) bool {
+ return fn(n)
+ })
+ return nil
+ })
+}
+
+func (m *pageMaps) walkBranchesPrefix(prefix string, fn func(s string, n *contentNode) bool) {
+ _ = m.withMaps(func(pm *pageMap) error {
+ pm.branchTrees.WalkPrefix(prefix, func(s string, n *contentNode) bool {
+ return fn(s, n)
+ })
+ return nil
+ })
+}
+
+func (m *pageMaps) withMaps(fn func(pm *pageMap) error) error {
+ g, _ := m.workers.Start(context.Background())
+ for _, pm := range m.pmaps {
+ pm := pm
+ g.Run(func() error {
+ return fn(pm)
+ })
+ }
+ return g.Wait()
+}
+
+type pagesMapBucket struct {
+ // Cascading front matter.
+ cascade maps.Params
+
+ owner *pageState // The branch node
+
+ pagesInit sync.Once
+ pages page.Pages
+
+ pagesAndSectionsInit sync.Once
+ pagesAndSections page.Pages
+
+ sectionsInit sync.Once
+ sections page.Pages
+}
+
+func (b *pagesMapBucket) getPages() page.Pages {
+ b.pagesInit.Do(func() {
+ b.pages = b.owner.treeRef.collectPages()
+ page.SortByDefault(b.pages)
+ })
+ return b.pages
+}
+
+func (b *pagesMapBucket) getPagesAndSections() page.Pages {
+ b.pagesAndSectionsInit.Do(func() {
+ b.pagesAndSections = b.owner.treeRef.collectPagesAndSections()
+ })
+ return b.pagesAndSections
+}
+
+func (b *pagesMapBucket) getSections() page.Pages {
+ b.sectionsInit.Do(func() {
+ b.sections = b.owner.treeRef.collectSections()
+ })
+
+ return b.sections
+}
+
+func (b *pagesMapBucket) getTaxonomies() page.Pages {
+ b.sectionsInit.Do(func() {
+ var pas page.Pages
+ ref := b.owner.treeRef
+ ref.m.collectTaxonomies(ref.key+"/", func(c *contentNode) {
+ pas = append(pas, c.p)
+ })
+ page.SortByDefault(pas)
+ b.sections = pas
+ })
+
+ return b.sections
+}
+
+type sectionAggregate struct {
+ datesAll resource.Dates
+ datesSection resource.Dates
+ pageCount int
+ mainSection string
+ mainSectionPageCount int
+}
+
+type sectionAggregateHandler struct {
+ sectionAggregate
+ sectionPageCount int
+
+ // Section
+ b *contentNode
+ s string
+}
+
+func (h *sectionAggregateHandler) isRootSection() bool {
+ return h.s != "/" && strings.Count(h.s, "/") == 1
+}
+
+func (h *sectionAggregateHandler) handleNested(v sectionWalkHandler) error {
+ nested := v.(*sectionAggregateHandler)
+ h.sectionPageCount += nested.pageCount
+ h.pageCount += h.sectionPageCount
+ h.datesAll.UpdateDateAndLastmodIfAfter(nested.datesAll)
+ h.datesSection.UpdateDateAndLastmodIfAfter(nested.datesAll)
+ return nil
+}
+
+func (h *sectionAggregateHandler) handlePage(s string, n *contentNode) error {
+ h.sectionPageCount++
+
+ var d resource.Dated
+ if n.p != nil {
+ d = n.p
+ } else if n.viewInfo != nil && n.viewInfo.ref != nil {
+ d = n.viewInfo.ref.p
+ } else {
+ return nil
+ }
+
+ h.datesAll.UpdateDateAndLastmodIfAfter(d)
+ h.datesSection.UpdateDateAndLastmodIfAfter(d)
+ return nil
+}
+
+func (h *sectionAggregateHandler) handleSectionPost() error {
+ if h.sectionPageCount > h.mainSectionPageCount && h.isRootSection() {
+ h.mainSectionPageCount = h.sectionPageCount
+ h.mainSection = strings.TrimPrefix(h.s, "/")
+ }
+
+ if resource.IsZeroDates(h.b.p) {
+ h.b.p.m.Dates = h.datesSection
+ }
+
+ h.datesSection = resource.Dates{}
+
+ return nil
+}
+
+func (h *sectionAggregateHandler) handleSectionPre(s string, b *contentNode) error {
+ h.s = s
+ h.b = b
+ h.sectionPageCount = 0
+ h.datesAll.UpdateDateAndLastmodIfAfter(b.p)
+ return nil
+}
+
+type sectionWalkHandler interface {
+ handleNested(v sectionWalkHandler) error
+ handlePage(s string, b *contentNode) error
+ handleSectionPost() error
+ handleSectionPre(s string, b *contentNode) error
+}
+
+type sectionWalker struct {
+ err error
+ m *contentMap
+}
+
+func (w *sectionWalker) applyAggregates() *sectionAggregateHandler {
+ return w.walkLevel("/", func() sectionWalkHandler {
+ return §ionAggregateHandler{}
+ }).(*sectionAggregateHandler)
+
+}
+
+func (w *sectionWalker) walkLevel(prefix string, createVisitor func() sectionWalkHandler) sectionWalkHandler {
+
+ level := strings.Count(prefix, "/")
+ visitor := createVisitor()
+
+ w.m.taxonomies.WalkPrefix(prefix, func(s string, v interface{}) bool {
+ currentLevel := strings.Count(s, "/")
+ if currentLevel > level {
+ return false
+ }
+
+ n := v.(*contentNode)
+
+ if w.err = visitor.handleSectionPre(s, n); w.err != nil {
+ return true
+ }
+
+ if currentLevel == 1 {
+ nested := w.walkLevel(s+"/", createVisitor)
+ if w.err = visitor.handleNested(nested); w.err != nil {
+ return true
+ }
+ } else {
+ w.m.taxonomyEntries.WalkPrefix(s, func(ss string, v interface{}) bool {
+ n := v.(*contentNode)
+ w.err = visitor.handlePage(ss, n)
+ return w.err != nil
+ })
+ }
+
+ w.err = visitor.handleSectionPost()
+
+ return w.err != nil
+ })
+
+ w.m.sections.WalkPrefix(prefix, func(s string, v interface{}) bool {
+ currentLevel := strings.Count(s, "/")
+ if currentLevel > level {
+ return false
+ }
+
+ n := v.(*contentNode)
+
+ if w.err = visitor.handleSectionPre(s, n); w.err != nil {
+ return true
+ }
+
+ w.m.pages.WalkPrefix(s+cmBranchSeparator, func(s string, v interface{}) bool {
+ w.err = visitor.handlePage(s, v.(*contentNode))
+ return w.err != nil
+ })
+
+ if w.err != nil {
+ return true
+ }
+
+ if s != "/" {
+ nested := w.walkLevel(s+"/", createVisitor)
+ if w.err = visitor.handleNested(nested); w.err != nil {
+ return true
+ }
+ }
+
+ w.err = visitor.handleSectionPost()
+
+ return w.err != nil
+ })
+
+ return visitor
+
+}
+
+type viewName struct {
+ singular string // e.g. "category"
+ plural string // e.g. "categories"
+}
+
+func (v viewName) IsZero() bool {
+ return v.singular == ""
+}
--- /dev/null
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+ "fmt"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/gohugoio/hugo/helpers"
+
+ "github.com/gohugoio/hugo/htesting/hqt"
+
+ "github.com/gohugoio/hugo/hugofs/files"
+
+ "github.com/gohugoio/hugo/hugofs"
+ "github.com/spf13/afero"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func BenchmarkContentMap(b *testing.B) {
+ writeFile := func(c *qt.C, fs afero.Fs, filename, content string) hugofs.FileMetaInfo {
+ c.Helper()
+ filename = filepath.FromSlash(filename)
+ c.Assert(fs.MkdirAll(filepath.Dir(filename), 0777), qt.IsNil)
+ c.Assert(afero.WriteFile(fs, filename, []byte(content), 0777), qt.IsNil)
+
+ fi, err := fs.Stat(filename)
+ c.Assert(err, qt.IsNil)
+
+ mfi := fi.(hugofs.FileMetaInfo)
+ return mfi
+
+ }
+
+ createFs := func(fs afero.Fs, lang string) afero.Fs {
+ return hugofs.NewBaseFileDecorator(fs,
+ func(fi hugofs.FileMetaInfo) {
+ meta := fi.Meta()
+ // We have a more elaborate filesystem setup in the
+ // real flow, so simulate this here.
+ meta["lang"] = lang
+ meta["path"] = meta.Filename()
+ meta["classifier"] = files.ClassifyContentFile(fi.Name())
+
+ })
+ }
+
+ b.Run("CreateMissingNodes", func(b *testing.B) {
+ c := qt.New(b)
+ b.StopTimer()
+ mps := make([]*contentMap, b.N)
+ for i := 0; i < b.N; i++ {
+ m := newContentMap(contentMapConfig{lang: "en"})
+ mps[i] = m
+ memfs := afero.NewMemMapFs()
+ fs := createFs(memfs, "en")
+ for i := 1; i <= 20; i++ {
+ c.Assert(m.AddFilesBundle(writeFile(c, fs, fmt.Sprintf("sect%d/a/index.md", i), "page")), qt.IsNil)
+ c.Assert(m.AddFilesBundle(writeFile(c, fs, fmt.Sprintf("sect2%d/%sindex.md", i, strings.Repeat("b/", i)), "page")), qt.IsNil)
+ }
+
+ }
+
+ b.StartTimer()
+
+ for i := 0; i < b.N; i++ {
+ m := mps[i]
+ c.Assert(m.CreateMissingNodes(), qt.IsNil)
+
+ b.StopTimer()
+ m.pages.DeletePrefix("/")
+ m.sections.DeletePrefix("/")
+ b.StartTimer()
+ }
+ })
+
+}
+
+func TestContentMap(t *testing.T) {
+ c := qt.New(t)
+
+ writeFile := func(c *qt.C, fs afero.Fs, filename, content string) hugofs.FileMetaInfo {
+ c.Helper()
+ filename = filepath.FromSlash(filename)
+ c.Assert(fs.MkdirAll(filepath.Dir(filename), 0777), qt.IsNil)
+ c.Assert(afero.WriteFile(fs, filename, []byte(content), 0777), qt.IsNil)
+
+ fi, err := fs.Stat(filename)
+ c.Assert(err, qt.IsNil)
+
+ mfi := fi.(hugofs.FileMetaInfo)
+ return mfi
+
+ }
+
+ createFs := func(fs afero.Fs, lang string) afero.Fs {
+ return hugofs.NewBaseFileDecorator(fs,
+ func(fi hugofs.FileMetaInfo) {
+ meta := fi.Meta()
+ // We have a more elaborate filesystem setup in the
+ // real flow, so simulate this here.
+ meta["lang"] = lang
+ meta["path"] = meta.Filename()
+ meta["classifier"] = files.ClassifyContentFile(fi.Name())
+ meta["translationBaseName"] = helpers.Filename(fi.Name())
+
+ })
+ }
+
+ c.Run("AddFiles", func(c *qt.C) {
+
+ memfs := afero.NewMemMapFs()
+
+ fsl := func(lang string) afero.Fs {
+ return createFs(memfs, lang)
+ }
+
+ fs := fsl("en")
+
+ header := writeFile(c, fs, "blog/a/index.md", "page")
+
+ c.Assert(header.Meta().Lang(), qt.Equals, "en")
+
+ resources := []hugofs.FileMetaInfo{
+ writeFile(c, fs, "blog/a/b/data.json", "data"),
+ writeFile(c, fs, "blog/a/logo.png", "image"),
+ }
+
+ m := newContentMap(contentMapConfig{lang: "en"})
+
+ c.Assert(m.AddFilesBundle(header, resources...), qt.IsNil)
+
+ c.Assert(m.AddFilesBundle(writeFile(c, fs, "blog/b/c/index.md", "page")), qt.IsNil)
+
+ c.Assert(m.AddFilesBundle(
+ writeFile(c, fs, "blog/_index.md", "section page"),
+ writeFile(c, fs, "blog/sectiondata.json", "section resource"),
+ ), qt.IsNil)
+
+ got := m.testDump()
+
+ expect := `
+ Tree 0:
+ /blog__hb_/a__hl_
+ /blog__hb_/b/c__hl_
+ Tree 1:
+ /blog
+ Tree 2:
+ /blog__hb_/a__hl_b/data.json
+ /blog__hb_/a__hl_logo.png
+ /blog__hl_sectiondata.json
+ en/pages/blog__hb_/a__hl_|f:blog/a/index.md
+ - R: blog/a/b/data.json
+ - R: blog/a/logo.png
+ en/pages/blog__hb_/b/c__hl_|f:blog/b/c/index.md
+ en/sections/blog|f:blog/_index.md
+ - P: blog/a/index.md
+ - P: blog/b/c/index.md
+ - R: blog/sectiondata.json
+
+`
+
+ c.Assert(got, hqt.IsSameString, expect, qt.Commentf(got))
+
+ // Add a data file to the section bundle
+ c.Assert(m.AddFiles(
+ writeFile(c, fs, "blog/sectiondata2.json", "section resource"),
+ ), qt.IsNil)
+
+ // And then one to the leaf bundles
+ c.Assert(m.AddFiles(
+ writeFile(c, fs, "blog/a/b/data2.json", "data2"),
+ ), qt.IsNil)
+
+ c.Assert(m.AddFiles(
+ writeFile(c, fs, "blog/b/c/d/data3.json", "data3"),
+ ), qt.IsNil)
+
+ got = m.testDump()
+
+ expect = `
+ Tree 0:
+ /blog__hb_/a__hl_
+ /blog__hb_/b/c__hl_
+ Tree 1:
+ /blog
+ Tree 2:
+ /blog__hb_/a__hl_b/data.json
+ /blog__hb_/a__hl_b/data2.json
+ /blog__hb_/a__hl_logo.png
+ /blog__hb_/b/c__hl_d/data3.json
+ /blog__hl_sectiondata.json
+ /blog__hl_sectiondata2.json
+ en/pages/blog__hb_/a__hl_|f:blog/a/index.md
+ - R: blog/a/b/data.json
+ - R: blog/a/b/data2.json
+ - R: blog/a/logo.png
+ en/pages/blog__hb_/b/c__hl_|f:blog/b/c/index.md
+ - R: blog/b/c/d/data3.json
+ en/sections/blog|f:blog/_index.md
+ - P: blog/a/index.md
+ - P: blog/b/c/index.md
+ - R: blog/sectiondata.json
+ - R: blog/sectiondata2.json
+
+`
+
+ c.Assert(got, hqt.IsSameString, expect, qt.Commentf(got))
+
+ // Add a regular page (i.e. not a bundle)
+ c.Assert(m.AddFilesBundle(writeFile(c, fs, "blog/b.md", "page")), qt.IsNil)
+
+ c.Assert(m.testDump(), hqt.IsSameString, `
+ Tree 0:
+ /blog__hb_/a__hl_
+ /blog__hb_/b/c__hl_
+ /blog__hb_/b__hl_
+ Tree 1:
+ /blog
+ Tree 2:
+ /blog__hb_/a__hl_b/data.json
+ /blog__hb_/a__hl_b/data2.json
+ /blog__hb_/a__hl_logo.png
+ /blog__hb_/b/c__hl_d/data3.json
+ /blog__hl_sectiondata.json
+ /blog__hl_sectiondata2.json
+ en/pages/blog__hb_/a__hl_|f:blog/a/index.md
+ - R: blog/a/b/data.json
+ - R: blog/a/b/data2.json
+ - R: blog/a/logo.png
+ en/pages/blog__hb_/b/c__hl_|f:blog/b/c/index.md
+ - R: blog/b/c/d/data3.json
+ en/pages/blog__hb_/b__hl_|f:blog/b.md
+ en/sections/blog|f:blog/_index.md
+ - P: blog/a/index.md
+ - P: blog/b/c/index.md
+ - P: blog/b.md
+ - R: blog/sectiondata.json
+ - R: blog/sectiondata2.json
+
+
+ `, qt.Commentf(m.testDump()))
+
+ })
+
+ c.Run("CreateMissingNodes", func(c *qt.C) {
+
+ memfs := afero.NewMemMapFs()
+
+ fsl := func(lang string) afero.Fs {
+ return createFs(memfs, lang)
+ }
+
+ fs := fsl("en")
+
+ m := newContentMap(contentMapConfig{lang: "en"})
+
+ c.Assert(m.AddFilesBundle(writeFile(c, fs, "blog/page.md", "page")), qt.IsNil)
+ c.Assert(m.AddFilesBundle(writeFile(c, fs, "blog/a/index.md", "page")), qt.IsNil)
+ c.Assert(m.AddFilesBundle(writeFile(c, fs, "bundle/index.md", "page")), qt.IsNil)
+
+ c.Assert(m.CreateMissingNodes(), qt.IsNil)
+
+ got := m.testDump()
+
+ c.Assert(got, hqt.IsSameString, `
+
+ Tree 0:
+ /__hb_/bundle__hl_
+ /blog__hb_/a__hl_
+ /blog__hb_/page__hl_
+ Tree 1:
+ /
+ /blog
+ Tree 2:
+ en/pages/__hb_/bundle__hl_|f:bundle/index.md
+ en/pages/blog__hb_/a__hl_|f:blog/a/index.md
+ en/pages/blog__hb_/page__hl_|f:blog/page.md
+ en/sections/
+ - P: bundle/index.md
+ en/sections/blog
+ - P: blog/a/index.md
+ - P: blog/page.md
+
+ `, qt.Commentf(got))
+
+ })
+
+ c.Run("cleanKey", func(c *qt.C) {
+ for _, test := range []struct {
+ in string
+ expected string
+ }{
+ {"/a/b/", "/a/b"},
+ {filepath.FromSlash("/a/b/"), "/a/b"},
+ {"/a//b/", "/a/b"},
+ } {
+
+ c.Assert(cleanTreeKey(test.in), qt.Equals, test.expected)
+
+ }
+ })
+}
+
+func TestContentMapSite(t *testing.T) {
+
+ b := newTestSitesBuilder(t)
+
+ pageTempl := `
+---
+title: "Page %d"
+date: "2019-06-0%d"
+lastMod: "2019-06-0%d"
+categories: ["funny"]
+---
+
+Page content.
+`
+ createPage := func(i int) string {
+ return fmt.Sprintf(pageTempl, i, i, i+1)
+ }
+
+ draftTemplate := `---
+title: "Draft"
+draft: true
+---
+
+`
+
+ b.WithContent("_index.md", `
+---
+title: "Hugo Home"
+cascade:
+ description: "Common Description"
+
+---
+
+Home Content.
+`)
+
+ b.WithContent("blog/page1.md", createPage(1))
+ b.WithContent("blog/page2.md", createPage(2))
+ b.WithContent("blog/page3.md", createPage(3))
+ b.WithContent("blog/bundle/index.md", createPage(12))
+ b.WithContent("blog/bundle/data.json", "data")
+ b.WithContent("blog/bundle/page.md", createPage(99))
+ b.WithContent("blog/subsection/_index.md", createPage(3))
+ b.WithContent("blog/subsection/subdata.json", "data")
+ b.WithContent("blog/subsection/page4.md", createPage(8))
+ b.WithContent("blog/subsection/page5.md", createPage(10))
+ b.WithContent("blog/subsection/draft/index.md", draftTemplate)
+ b.WithContent("blog/subsection/draft/data.json", "data")
+ b.WithContent("blog/draftsection/_index.md", draftTemplate)
+ b.WithContent("blog/draftsection/page/index.md", createPage(12))
+ b.WithContent("blog/draftsection/page/folder/data.json", "data")
+ b.WithContent("blog/draftsection/sub/_index.md", createPage(12))
+ b.WithContent("blog/draftsection/sub/page.md", createPage(13))
+ b.WithContent("docs/page6.md", createPage(11))
+ b.WithContent("tags/_index.md", createPage(32))
+
+ b.WithTemplatesAdded("layouts/index.html", `
+Num Regular: {{ len .Site.RegularPages }}
+Main Sections: {{ .Site.Params.mainSections }}
+Pag Num Pages: {{ len .Paginator.Pages }}
+{{ $home := .Site.Home }}
+{{ $blog := .Site.GetPage "blog" }}
+{{ $categories := .Site.GetPage "categories" }}
+{{ $funny := .Site.GetPage "categories/funny" }}
+{{ $blogSub := .Site.GetPage "blog/subsection" }}
+{{ $page := .Site.GetPage "blog/page1" }}
+{{ $page2 := .Site.GetPage "blog/page2" }}
+{{ $page4 := .Site.GetPage "blog/subsection/page4" }}
+{{ $bundle := .Site.GetPage "blog/bundle" }}
+Home: {{ template "print-page" $home }}
+Blog Section: {{ template "print-page" $blog }}
+Blog Sub Section: {{ template "print-page" $blogSub }}
+Page: {{ template "print-page" $page }}
+Bundle: {{ template "print-page" $bundle }}
+IsDescendant: true: {{ $page.IsDescendant $blog }} true: {{ $blogSub.IsDescendant $blog }} true: {{ $blog.IsDescendant $home }} false: {{ $home.IsDescendant $blog }}
+IsAncestor: true: {{ $blog.IsAncestor $page }} true: {{ $home.IsAncestor $blog }} true: {{ $blog.IsAncestor $blogSub }} true: {{ $home.IsAncestor $page }} false: {{ $page.IsAncestor $blog }} false: {{ $blog.IsAncestor $home }} false: {{ $blogSub.IsAncestor $blog }}
+FirstSection: {{ $blogSub.FirstSection.RelPermalink }} {{ $blog.FirstSection.RelPermalink }} {{ $home.FirstSection.RelPermalink }} {{ $page.FirstSection.RelPermalink }}
+InSection: true: {{ $page.InSection $blog }} false: {{ $page.InSection $blogSub }}
+Next: {{ $page2.Next.RelPermalink }}
+NextInSection: {{ $page2.NextInSection.RelPermalink }}
+Pages: {{ range $blog.Pages }}{{ .RelPermalink }}|{{ end }}
+Sections: {{ range $home.Sections }}{{ .RelPermalink }}|{{ end }}
+Categories: {{ range .Site.Taxonomies.categories }}{{ .Page.RelPermalink }}; {{ .Page.Title }}; {{ .Count }}|{{ end }}
+Category Terms: {{ $categories.Kind}}: {{ range $categories.Data.Terms.Alphabetical }}{{ .Page.RelPermalink }}; {{ .Page.Title }}; {{ .Count }}|{{ end }}
+Category Funny: {{ $funny.Kind}}; {{ $funny.Data.Term }}: {{ range $funny.Pages }}{{ .RelPermalink }};|{{ end }}
+Pag Num Pages: {{ len .Paginator.Pages }}
+Pag Blog Num Pages: {{ len $blog.Paginator.Pages }}
+Blog Num RegularPages: {{ len $blog.RegularPages }}
+Blog Num Pages: {{ len $blog.Pages }}
+
+Draft1: {{ if (.Site.GetPage "blog/subsection/draft") }}FOUND{{ end }}|
+Draft2: {{ if (.Site.GetPage "blog/draftsection") }}FOUND{{ end }}|
+Draft3: {{ if (.Site.GetPage "blog/draftsection/page") }}FOUND{{ end }}|
+Draft4: {{ if (.Site.GetPage "blog/draftsection/sub") }}FOUND{{ end }}|
+Draft5: {{ if (.Site.GetPage "blog/draftsection/sub/page") }}FOUND{{ end }}|
+
+{{ define "print-page" }}{{ .Title }}|{{ .RelPermalink }}|{{ .Date.Format "2006-01-02" }}|Current Section: {{ .CurrentSection.SectionsPath }}|Resources: {{ range .Resources }}{{ .ResourceType }}: {{ .RelPermalink }}|{{ end }}{{ end }}
+`)
+
+ b.Build(BuildCfg{})
+
+ b.AssertFileContent("public/index.html",
+
+ `
+ Num Regular: 7
+ Main Sections: [blog]
+ Pag Num Pages: 7
+
+ Home: Hugo Home|/|2019-06-08|Current Section: |Resources:
+ Blog Section: Blogs|/blog/|2019-06-08|Current Section: blog|Resources:
+ Blog Sub Section: Page 3|/blog/subsection/|2019-06-03|Current Section: blog/subsection|Resources: json: /blog/subsection/subdata.json|
+ Page: Page 1|/blog/page1/|2019-06-01|Current Section: blog|Resources:
+ Bundle: Page 12|/blog/bundle/|0001-01-01|Current Section: blog|Resources: json: /blog/bundle/data.json|page: |
+ IsDescendant: true: true true: true true: true false: false
+ IsAncestor: true: true true: true true: true true: true false: false false: false false: false
+ FirstSection: /blog/ /blog/ / /blog/
+ InSection: true: true false: false
+ Next: /blog/page3/
+ NextInSection: /blog/page3/
+ Pages: /blog/page3/|/blog/subsection/|/blog/page2/|/blog/page1/|/blog/bundle/|
+ Sections: /blog/|/docs/|
+ Categories: /categories/funny/; funny; 9|
+ Category Terms: taxonomyTerm: /categories/funny/; funny; 9|
+ Category Funny: taxonomy; funny: /blog/subsection/page4/;|/blog/page3/;|/blog/subsection/;|/blog/page2/;|/blog/page1/;|/blog/subsection/page5/;|/docs/page6/;|/blog/bundle/;|;|
+ Pag Num Pages: 7
+ Pag Blog Num Pages: 4
+ Blog Num RegularPages: 4
+ Blog Num Pages: 5
+
+ Draft1: |
+ Draft2: |
+ Draft3: |
+ Draft4: |
+ Draft5: |
+
+`)
+}
package hugolib
import (
- "strings"
"testing"
"fmt"
qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/resources/page"
-
- "github.com/gohugoio/hugo/helpers"
)
-func TestDisableKindsNoneDisabled(t *testing.T) {
- t.Parallel()
- doTestDisableKinds(t)
-}
-
-func TestDisableKindsSomeDisabled(t *testing.T) {
- t.Parallel()
- doTestDisableKinds(t, page.KindSection, kind404)
-}
+func TestDisable(t *testing.T) {
+ c := qt.New(t)
-func TestDisableKindsOneDisabled(t *testing.T) {
- t.Parallel()
- for _, kind := range allKinds {
- if kind == page.KindPage {
- // Turning off regular page generation have some side-effects
- // not handled by the assertions below (no sections), so
- // skip that for now.
- continue
- }
- doTestDisableKinds(t, kind)
- }
-}
-
-func TestDisableKindsAllDisabled(t *testing.T) {
- t.Parallel()
- doTestDisableKinds(t, allKinds...)
-}
-
-func doTestDisableKinds(t *testing.T, disabled ...string) {
- siteConfigTemplate := `
+ newSitesBuilder := func(c *qt.C, disableKind string) *sitesBuilder {
+ config := fmt.Sprintf(`
baseURL = "http://example.com/blog"
enableRobotsTXT = true
-disableKinds = %s
-
-paginate = 1
-defaultContentLanguage = "en"
-
-[Taxonomies]
-tag = "tags"
-category = "categories"
-`
-
- pageTemplate := `---
-title: "%s"
-tags:
-%s
-categories:
-- Hugo
----
-# Doc
-`
+disableKinds = [%q]
+`, disableKind)
- disabledStr := "[]"
-
- if len(disabled) > 0 {
- disabledStr = strings.Replace(fmt.Sprintf("%#v", disabled), "[]string{", "[", -1)
- disabledStr = strings.Replace(disabledStr, "}", "]", -1)
- }
+ b := newTestSitesBuilder(c)
+ b.WithConfigFile("toml", config).WithContent("sect/page.md", `
+---
+title: Page
+categories: ["mycat"]
+tags: ["mytag"]
+---
- siteConfig := fmt.Sprintf(siteConfigTemplate, disabledStr)
+`, "sect/no-list.md", `
+---
+title: No List
+_build:
+ list: false
+---
- b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig)
+`, "sect/no-render.md", `
+---
+title: No List
+_build:
+ render: false
+---
+`, "sect/no-publishresources/index.md", `
+---
+title: No Publish Resources
+_build:
+ publishResources: false
+---
- b.WithTemplates(
- "index.html", "Home|{{ .Title }}|{{ .Content }}",
- "_default/single.html", "Single|{{ .Title }}|{{ .Content }}",
- "_default/list.html", "List|{{ .Title }}|{{ .Content }}",
- "_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}",
- "layouts/404.html", "Page Not Found",
- )
+`, "sect/headlessbundle/index.md", `
+---
+title: Headless
+headless: true
+---
- b.WithContent(
- "sect/p1.md", fmt.Sprintf(pageTemplate, "P1", "- tag1"),
- "categories/_index.md", newTestPage("Category Terms", "2017-01-01", 10),
- "tags/tag1/_index.md", newTestPage("Tag1 List", "2017-01-01", 10),
- )
+`)
+ b.WithSourceFile("content/sect/headlessbundle/data.json", "DATA")
+ b.WithSourceFile("content/sect/no-publishresources/data.json", "DATA")
- b.Build(BuildCfg{})
- h := b.H
+ return b
- assertDisabledKinds(b, h.Sites[0], disabled...)
+ }
-}
+ getPage := func(b *sitesBuilder, ref string) page.Page {
+ b.Helper()
+ p, err := b.H.Sites[0].getPageNew(nil, ref)
+ b.Assert(err, qt.IsNil)
+ return p
+ }
-func assertDisabledKinds(b *sitesBuilder, s *Site, disabled ...string) {
- assertDisabledKind(b,
- func(isDisabled bool) bool {
- if isDisabled {
- return len(s.RegularPages()) == 0
- }
- return len(s.RegularPages()) > 0
- }, disabled, page.KindPage, "public/sect/p1/index.html", "Single|P1")
- assertDisabledKind(b,
- func(isDisabled bool) bool {
- p := s.getPage(page.KindHome)
- if isDisabled {
- return p == nil
- }
- return p != nil
- }, disabled, page.KindHome, "public/index.html", "Home")
- assertDisabledKind(b,
- func(isDisabled bool) bool {
- p := s.getPage(page.KindSection, "sect")
- if isDisabled {
- return p == nil
- }
- return p != nil
- }, disabled, page.KindSection, "public/sect/index.html", "Sects")
- assertDisabledKind(b,
- func(isDisabled bool) bool {
- p := s.getPage(page.KindTaxonomy, "tags", "tag1")
-
- if isDisabled {
- return p == nil
- }
- return p != nil
-
- }, disabled, page.KindTaxonomy, "public/tags/tag1/index.html", "Tag1")
- assertDisabledKind(b,
- func(isDisabled bool) bool {
- p := s.getPage(page.KindTaxonomyTerm, "tags")
- if isDisabled {
- return p == nil
+ getPageInSitePages := func(b *sitesBuilder, ref string) page.Page {
+ b.Helper()
+ for _, pages := range []page.Pages{b.H.Sites[0].Pages(), b.H.Sites[0].RegularPages()} {
+ for _, p := range pages {
+ if ref == p.(*pageState).sourceRef() {
+ return p
+ }
}
- return p != nil
-
- }, disabled, page.KindTaxonomyTerm, "public/tags/index.html", "Tags")
- assertDisabledKind(b,
- func(isDisabled bool) bool {
- p := s.getPage(page.KindTaxonomyTerm, "categories")
+ }
+ return nil
+ }
- if isDisabled {
- return p == nil
+ getPageInPagePages := func(p page.Page, ref string) page.Page {
+ for _, pages := range []page.Pages{p.Pages(), p.RegularPages(), p.Sections()} {
+ for _, p := range pages {
+ if ref == p.(*pageState).sourceRef() {
+ return p
+ }
}
- return p != nil
-
- }, disabled, page.KindTaxonomyTerm, "public/categories/index.html", "Category Terms")
- assertDisabledKind(b,
- func(isDisabled bool) bool {
- p := s.getPage(page.KindTaxonomy, "categories", "hugo")
- if isDisabled {
- return p == nil
- }
- return p != nil
-
- }, disabled, page.KindTaxonomy, "public/categories/hugo/index.html", "Hugo")
- // The below have no page in any collection.
- assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kindRSS, "public/index.xml", "<link>")
- assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kindSitemap, "public/sitemap.xml", "sitemap")
- assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kindRobotsTXT, "public/robots.txt", "User-agent")
- assertDisabledKind(b, func(isDisabled bool) bool { return true }, disabled, kind404, "public/404.html", "Page Not Found")
-}
-
-func assertDisabledKind(b *sitesBuilder, kindAssert func(bool) bool, disabled []string, kind, path, matcher string) {
- isDisabled := stringSliceContains(kind, disabled...)
- b.Assert(kindAssert(isDisabled), qt.Equals, true)
-
- if kind == kindRSS && !isDisabled {
- // If the home page is also disabled, there is not RSS to look for.
- if stringSliceContains(page.KindHome, disabled...) {
- isDisabled = true
}
+ return nil
}
- if isDisabled {
- // Path should not exist
- fileExists, err := helpers.Exists(path, b.Fs.Destination)
- b.Assert(err, qt.IsNil)
- b.Assert(fileExists, qt.Equals, false)
-
- } else {
- b.AssertFileContent(path, matcher)
- }
+ disableKind := page.KindPage
+ c.Run("Disable "+disableKind, func(c *qt.C) {
+ b := newSitesBuilder(c, disableKind)
+ b.Build(BuildCfg{})
+ s := b.H.Sites[0]
+ b.Assert(getPage(b, "/sect/page.md"), qt.IsNil)
+ b.Assert(b.CheckExists("public/sect/page/index.html"), qt.Equals, false)
+ b.Assert(getPageInSitePages(b, "/sect/page.md"), qt.IsNil)
+ b.Assert(getPageInPagePages(getPage(b, "/"), "/sect/page.md"), qt.IsNil)
+
+ // Also check the side effects
+ b.Assert(b.CheckExists("public/categories/mycat/index.html"), qt.Equals, false)
+ b.Assert(len(s.Taxonomies()["categories"]), qt.Equals, 0)
+ })
+
+ disableKind = page.KindTaxonomy
+ c.Run("Disable "+disableKind, func(c *qt.C) {
+ b := newSitesBuilder(c, disableKind)
+ b.Build(BuildCfg{})
+ s := b.H.Sites[0]
+ b.Assert(b.CheckExists("public/categories/index.html"), qt.Equals, true)
+ b.Assert(b.CheckExists("public/categories/mycat/index.html"), qt.Equals, false)
+ b.Assert(len(s.Taxonomies()["categories"]), qt.Equals, 0)
+ b.Assert(getPage(b, "/categories"), qt.Not(qt.IsNil))
+ b.Assert(getPage(b, "/categories/mycat"), qt.IsNil)
+ })
+
+ disableKind = page.KindTaxonomyTerm
+ c.Run("Disable "+disableKind, func(c *qt.C) {
+ b := newSitesBuilder(c, disableKind)
+ b.Build(BuildCfg{})
+ s := b.H.Sites[0]
+ b.Assert(b.CheckExists("public/categories/mycat/index.html"), qt.Equals, true)
+ b.Assert(b.CheckExists("public/categories/index.html"), qt.Equals, false)
+ b.Assert(len(s.Taxonomies()["categories"]), qt.Equals, 1)
+ b.Assert(getPage(b, "/categories/mycat"), qt.Not(qt.IsNil))
+ categories := getPage(b, "/categories")
+ b.Assert(categories, qt.Not(qt.IsNil))
+ b.Assert(categories.RelPermalink(), qt.Equals, "")
+ b.Assert(getPageInSitePages(b, "/categories"), qt.IsNil)
+ b.Assert(getPageInPagePages(getPage(b, "/"), "/categories"), qt.IsNil)
+ })
+
+ disableKind = page.KindHome
+ c.Run("Disable "+disableKind, func(c *qt.C) {
+ b := newSitesBuilder(c, disableKind)
+ b.Build(BuildCfg{})
+ b.Assert(b.CheckExists("public/index.html"), qt.Equals, false)
+ home := getPage(b, "/")
+ b.Assert(home, qt.Not(qt.IsNil))
+ b.Assert(home.RelPermalink(), qt.Equals, "")
+ b.Assert(getPageInSitePages(b, "/"), qt.IsNil)
+ b.Assert(getPageInPagePages(home, "/"), qt.IsNil)
+ b.Assert(getPage(b, "/sect/page.md"), qt.Not(qt.IsNil))
+ })
+
+ disableKind = page.KindSection
+ c.Run("Disable "+disableKind, func(c *qt.C) {
+ b := newSitesBuilder(c, disableKind)
+ b.Build(BuildCfg{})
+ b.Assert(b.CheckExists("public/sect/index.html"), qt.Equals, false)
+ sect := getPage(b, "/sect")
+ b.Assert(sect, qt.Not(qt.IsNil))
+ b.Assert(sect.RelPermalink(), qt.Equals, "")
+ b.Assert(getPageInSitePages(b, "/sect"), qt.IsNil)
+ home := getPage(b, "/")
+ b.Assert(getPageInPagePages(home, "/sect"), qt.IsNil)
+ b.Assert(home.OutputFormats(), qt.HasLen, 2)
+ page := getPage(b, "/sect/page.md")
+ b.Assert(page, qt.Not(qt.IsNil))
+ b.Assert(page.CurrentSection(), qt.Equals, sect)
+ b.Assert(getPageInPagePages(sect, "/sect/page.md"), qt.Not(qt.IsNil))
+ b.AssertFileContent("public/sitemap.xml", "sitemap")
+ b.AssertFileContent("public/index.xml", "rss")
+
+ })
+
+ disableKind = kindRSS
+ c.Run("Disable "+disableKind, func(c *qt.C) {
+ b := newSitesBuilder(c, disableKind)
+ b.Build(BuildCfg{})
+ b.Assert(b.CheckExists("public/index.xml"), qt.Equals, false)
+ home := getPage(b, "/")
+ b.Assert(home.OutputFormats(), qt.HasLen, 1)
+ })
+
+ disableKind = kindSitemap
+ c.Run("Disable "+disableKind, func(c *qt.C) {
+ b := newSitesBuilder(c, disableKind)
+ b.Build(BuildCfg{})
+ b.Assert(b.CheckExists("public/sitemap.xml"), qt.Equals, false)
+ })
+
+ disableKind = kind404
+ c.Run("Disable "+disableKind, func(c *qt.C) {
+ b := newSitesBuilder(c, disableKind)
+ b.Build(BuildCfg{})
+ b.Assert(b.CheckExists("public/404.html"), qt.Equals, false)
+ })
+
+ disableKind = kindRobotsTXT
+ c.Run("Disable "+disableKind, func(c *qt.C) {
+ b := newSitesBuilder(c, disableKind)
+ b.WithTemplatesAdded("robots.txt", "myrobots")
+ b.Build(BuildCfg{})
+ b.Assert(b.CheckExists("public/robots.txt"), qt.Equals, false)
+ })
+
+ c.Run("Headless bundle", func(c *qt.C) {
+ b := newSitesBuilder(c, disableKind)
+ b.Build(BuildCfg{})
+ b.Assert(b.CheckExists("public/sect/headlessbundle/index.html"), qt.Equals, false)
+ b.Assert(b.CheckExists("public/sect/headlessbundle/data.json"), qt.Equals, true)
+ bundle := getPage(b, "/sect/headlessbundle/index.md")
+ b.Assert(bundle, qt.Not(qt.IsNil))
+ b.Assert(bundle.RelPermalink(), qt.Equals, "")
+ resource := bundle.Resources()[0]
+ b.Assert(resource.RelPermalink(), qt.Equals, "/blog/sect/headlessbundle/data.json")
+ b.Assert(bundle.OutputFormats(), qt.HasLen, 0)
+ b.Assert(bundle.AlternativeOutputFormats(), qt.HasLen, 0)
+ })
+
+ c.Run("Build config, no list", func(c *qt.C) {
+ b := newSitesBuilder(c, disableKind)
+ b.Build(BuildCfg{})
+ ref := "/sect/no-list.md"
+ b.Assert(b.CheckExists("public/sect/no-list/index.html"), qt.Equals, true)
+ p := getPage(b, ref)
+ b.Assert(p, qt.Not(qt.IsNil))
+ b.Assert(p.RelPermalink(), qt.Equals, "/blog/sect/no-list/")
+ b.Assert(getPageInSitePages(b, ref), qt.IsNil)
+ sect := getPage(b, "/sect")
+ b.Assert(getPageInPagePages(sect, ref), qt.IsNil)
+
+ })
+
+ c.Run("Build config, no render", func(c *qt.C) {
+ b := newSitesBuilder(c, disableKind)
+ b.Build(BuildCfg{})
+ ref := "/sect/no-render.md"
+ b.Assert(b.CheckExists("public/sect/no-render/index.html"), qt.Equals, false)
+ p := getPage(b, ref)
+ b.Assert(p, qt.Not(qt.IsNil))
+ b.Assert(p.RelPermalink(), qt.Equals, "")
+ b.Assert(p.OutputFormats(), qt.HasLen, 0)
+ b.Assert(getPageInSitePages(b, ref), qt.Not(qt.IsNil))
+ sect := getPage(b, "/sect")
+ b.Assert(getPageInPagePages(sect, ref), qt.Not(qt.IsNil))
+ })
+
+ c.Run("Build config, no publish resources", func(c *qt.C) {
+ b := newSitesBuilder(c, disableKind)
+ b.Build(BuildCfg{})
+ b.Assert(b.CheckExists("public/sect/no-publishresources/index.html"), qt.Equals, true)
+ b.Assert(b.CheckExists("public/sect/no-publishresources/data.json"), qt.Equals, false)
+ bundle := getPage(b, "/sect/no-publishresources/index.md")
+ b.Assert(bundle, qt.Not(qt.IsNil))
+ b.Assert(bundle.RelPermalink(), qt.Equals, "/blog/sect/no-publishresources/")
+ b.Assert(bundle.Resources(), qt.HasLen, 1)
+ resource := bundle.Resources()[0]
+ b.Assert(resource.RelPermalink(), qt.Equals, "/blog/sect/no-publishresources/data.json")
+ })
}
From: mount.Target,
To: filename,
ToBasedir: base,
+ Module: md.Module.Path(),
Meta: hugofs.FileMeta{
"watch": md.Watch(),
"mountWeight": mountWeight,
module github.com/gohugoio/tests/testHugoModules
+`)
+
+ b.WithSourceFile("go.sum", `
+github.com/gohugoio/hugoTestModule2 v0.0.0-20200131160637-9657d7697877 h1:WLM2bQCKIWo04T6NsIWsX/Vtirhf0TnpY66xyqGlgVY=
+github.com/gohugoio/hugoTestModule2 v0.0.0-20200131160637-9657d7697877/go.mod h1:CBFZS3khIAXKxReMwq0le8sEl/D8hcXmixlOHVv+Gd0=
`)
b.Build(BuildCfg{})
package hugolib
import (
+ "context"
"io"
"path/filepath"
"sort"
"github.com/gohugoio/hugo/output"
"github.com/gohugoio/hugo/parser/metadecoders"
+ "github.com/gohugoio/hugo/common/para"
"github.com/gohugoio/hugo/hugofs"
-
"github.com/pkg/errors"
"github.com/gohugoio/hugo/source"
// As loaded from the /data dirs
data map[string]interface{}
+ content *pageMaps
+
// Keeps track of bundle directories and symlinks to enable partial rebuilding.
ContentChanges *contentChangeMap
init *hugoSitesInit
+ workers *para.Workers
+ numWorkers int
+
*fatalErrorHandler
*testCounters
}
func (h *HugoSites) siteInfos() page.Sites {
infos := make(page.Sites, len(h.Sites))
for i, site := range h.Sites {
- infos[i] = &site.Info
+ infos[i] = site.Info
}
return infos
}
// GetContentPage finds a Page with content given the absolute filename.
// Returns nil if none found.
func (h *HugoSites) GetContentPage(filename string) page.Page {
- for _, s := range h.Sites {
- pos := s.rawAllPages.findPagePosByFilename(filename)
- if pos == -1 {
- continue
- }
- return s.rawAllPages[pos]
- }
+ var p page.Page
- // If not found already, this may be bundled in another content file.
- dir := filepath.Dir(filename)
+ h.content.walkBundles(func(b *contentNode) bool {
+ if b.p == nil || b.fi == nil {
+ return false
+ }
- for _, s := range h.Sites {
- pos := s.rawAllPages.findPagePosByFilnamePrefix(dir)
- if pos == -1 {
- continue
+ if b.fi.Meta().Filename() == filename {
+ p = b.p
+ return true
}
- return s.rawAllPages[pos]
- }
- return nil
+
+ return false
+ })
+
+ return p
}
// NewHugoSites creates a new collection of sites given the input sites, building
var contentChangeTracker *contentChangeMap
+ numWorkers := config.GetNumWorkerMultiplier()
+ if numWorkers > len(sites) {
+ numWorkers = len(sites)
+ }
+ var workers *para.Workers
+ if numWorkers > 1 {
+ workers = para.New(numWorkers)
+ }
+
h := &HugoSites{
running: cfg.Running,
multilingual: langConfig,
multihost: cfg.Cfg.GetBool("multihost"),
Sites: sites,
+ workers: workers,
+ numWorkers: numWorkers,
init: &hugoSitesInit{
data: lazy.New(),
layouts: lazy.New(),
return err
}
- d.Site = &s.Info
+ d.Site = s.Info
siteConfig, err := loadSiteConfig(s.language)
if err != nil {
return errors.Wrap(err, "load site config")
}
s.siteConfigConfig = siteConfig
+
+ pm := &pageMap{
+ contentMap: newContentMap(contentMapConfig{
+ lang: s.Lang(),
+ taxonomyConfig: s.siteCfg.taxonomiesConfig.Values(),
+ taxonomyDisabled: !s.isEnabled(page.KindTaxonomy),
+ taxonomyTermDisabled: !s.isEnabled(page.KindTaxonomyTerm),
+ pageDisabled: !s.isEnabled(page.KindPage),
+ }),
+ s: s,
+ }
+
+ s.PageCollections = newPageCollections(pm)
+
s.siteRefLinker, err = newSiteRefLinker(s.language, s)
return err
}
}
}
+func (h *HugoSites) withSite(fn func(s *Site) error) error {
+ if h.workers == nil {
+ for _, s := range h.Sites {
+ if err := fn(s); err != nil {
+ return err
+ }
+ }
+ return nil
+ }
+
+ g, _ := h.workers.Start(context.Background())
+ for _, s := range h.Sites {
+ s := s
+ g.Run(func() error {
+ return fn(s)
+ })
+ }
+ return g.Wait()
+}
+
func (h *HugoSites) createSitesFromConfig(cfg config.Provider) error {
oldLangs, _ := h.Cfg.Get("languagesSorted").(langs.Languages)
func (h *HugoSites) toSiteInfos() []*SiteInfo {
infos := make([]*SiteInfo, len(h.Sites))
for i, s := range h.Sites {
- infos[i] = &s.Info
+ infos[i] = s.Info
}
return infos
}
// For regular builds, this will allways return true.
// TODO(bep) rename/work this.
func (cfg *BuildCfg) shouldRender(p *pageState) bool {
- if !p.render {
- return false
- }
if p.forceRender {
return true
}
}
func (h *HugoSites) removePageByFilename(filename string) {
- for _, s := range h.Sites {
- s.removePageFilename(filename)
- }
+ h.content.withMaps(func(m *pageMap) error {
+ m.deleteBundleMatching(func(b *contentNode) bool {
+ if b.p == nil {
+ return false
+ }
+
+ if b.fi == nil {
+ return false
+ }
+
+ return b.fi.Meta().Filename() == filename
+ })
+ return nil
+ })
+
}
func (h *HugoSites) createPageCollections() error {
}
func (s *Site) preparePagesForRender(isRenderingSite bool, idx int) error {
-
- for _, p := range s.workAllPages {
- if err := p.initOutputFormat(isRenderingSite, idx); err != nil {
- return err
+ var err error
+ s.pageMap.withEveryBundlePage(func(p *pageState) bool {
+ if err = p.initOutputFormat(isRenderingSite, idx); err != nil {
+ return true
}
- }
-
- for _, p := range s.headlessPages {
- if err := p.initOutputFormat(isRenderingSite, idx); err != nil {
- return err
- }
- }
-
+ return false
+ })
return nil
}
}
func (h *HugoSites) resetPageState() {
- for _, s := range h.Sites {
- for _, p := range s.rawAllPages {
- for _, po := range p.pageOutputs {
- if po.cp == nil {
- continue
- }
- po.cp.Reset()
+ h.content.walkBundles(func(n *contentNode) bool {
+ if n.p == nil {
+ return false
+ }
+ p := n.p
+ for _, po := range p.pageOutputs {
+ if po.cp == nil {
+ continue
}
+ po.cp.Reset()
}
- }
+
+ return false
+ })
}
func (h *HugoSites) resetPageStateFromEvents(idset identity.Identities) {
- for _, s := range h.Sites {
- PAGES:
- for _, p := range s.rawAllPages {
- OUTPUTS:
- for _, po := range p.pageOutputs {
- if po.cp == nil {
- continue
- }
- for id := range idset {
- if po.cp.dependencyTracker.Search(id) != nil {
- po.cp.Reset()
- continue OUTPUTS
- }
+ h.content.walkBundles(func(n *contentNode) bool {
+ if n.p == nil {
+ return false
+ }
+ p := n.p
+ OUTPUTS:
+ for _, po := range p.pageOutputs {
+ if po.cp == nil {
+ continue
+ }
+ for id := range idset {
+ if po.cp.dependencyTracker.Search(id) != nil {
+ po.cp.Reset()
+ continue OUTPUTS
}
}
+ }
+
+ if p.shortcodeState == nil {
+ return false
+ }
- for _, s := range p.shortcodeState.shortcodes {
- for id := range idset {
- if idm, ok := s.info.(identity.Manager); ok && idm.Search(id) != nil {
- for _, po := range p.pageOutputs {
- if po.cp != nil {
- po.cp.Reset()
- }
+ for _, s := range p.shortcodeState.shortcodes {
+ for id := range idset {
+ if idm, ok := s.info.(identity.Manager); ok && idm.Search(id) != nil {
+ for _, po := range p.pageOutputs {
+ if po.cp != nil {
+ po.cp.Reset()
}
- continue PAGES
}
+ return false
}
}
}
- }
+ return false
+ })
+
}
// Used in partial reloading to determine if the change is in a bundle.
"fmt"
"runtime/trace"
- "github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/output"
- "golang.org/x/sync/errgroup"
- "golang.org/x/sync/semaphore"
"github.com/pkg/errors"
return nil
}
- numWorkers := config.GetNumWorkerMultiplier()
- sem := semaphore.NewWeighted(int64(numWorkers))
- g, ctx := errgroup.WithContext(context.Background())
-
- for _, s := range h.Sites {
- s := s
- g.Go(func() error {
- err := sem.Acquire(ctx, 1)
- if err != nil {
- return err
- }
- defer sem.Release(1)
-
- if err := s.assemblePagesMap(s); err != nil {
- return err
- }
-
- if err := s.pagesMap.assemblePageMeta(); err != nil {
- return err
- }
-
- if err := s.pagesMap.assembleTaxonomies(s); err != nil {
- return err
- }
-
- if err := s.createWorkAllPages(); err != nil {
- return err
- }
-
- return nil
-
- })
- }
-
- if err := g.Wait(); err != nil {
+ if err := h.content.AssemblePages(); err != nil {
return err
}
if !config.PartialReRender {
h.renderFormats = output.Formats{}
- for _, s := range h.Sites {
+ h.withSite(func(s *Site) error {
s.initRenderFormats()
+ return nil
+ })
+
+ for _, s := range h.Sites {
h.renderFormats = append(h.renderFormats, s.renderFormats...)
}
}
"time"
qt "github.com/frankban/quicktest"
+ "github.com/gohugoio/hugo/htesting"
"github.com/gohugoio/hugo/resources/page"
"github.com/fortytw2/leaktest"
c.Assert(len(doc4.Translations()), qt.Equals, 0)
// Taxonomies and their URLs
- c.Assert(len(enSite.Taxonomies), qt.Equals, 1)
- tags := enSite.Taxonomies["tags"]
+ c.Assert(len(enSite.Taxonomies()), qt.Equals, 1)
+ tags := enSite.Taxonomies()["tags"]
c.Assert(len(tags), qt.Equals, 2)
c.Assert(doc1en, qt.Equals, tags["tag1"][0].Page)
b.AssertFileContent("public/fr/sitemap.xml", "http://example.com/blog/fr/sect/doc1/")
// Check taxonomies
- enTags := enSite.Taxonomies["tags"]
- frTags := frSite.Taxonomies["plaques"]
+ enTags := enSite.Taxonomies()["tags"]
+ frTags := frSite.Taxonomies()["plaques"]
c.Assert(len(enTags), qt.Equals, 2, qt.Commentf("Tags in en: %v", enTags))
c.Assert(len(frTags), qt.Equals, 2, qt.Commentf("Tags in fr: %v", frTags))
c.Assert(enTags["tag1"], qt.Not(qt.IsNil))
content := readDestination(s.T, s.Fs, filename)
for _, match := range matches {
if !strings.Contains(content, match) {
- s.Fatalf("No match for %q in content for %s\n%q", match, filename, content)
+ s.Fatalf("No match for\n%q\nin content for %s\n%q\nDiff:\n%s", match, filename, content, htesting.DiffStrings(content, match))
}
}
}
"github.com/mitchellh/mapstructure"
- "github.com/gohugoio/hugo/tpl"
-
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/markup/converter"
- "github.com/gohugoio/hugo/common/maps"
+ "github.com/gohugoio/hugo/tpl"
"github.com/gohugoio/hugo/hugofs/files"
return b.getPagesAndSections()
}
-// TODO(bep) cm add a test
func (p *pageState) RegularPages() page.Pages {
p.regularPagesInit.Do(func() {
var pages page.Pages
case page.KindSection, page.KindHome:
pages = p.getPagesAndSections()
case page.KindTaxonomy:
- termInfo := p.bucket
- plural := maps.GetString(termInfo.meta, "plural")
- term := maps.GetString(termInfo.meta, "termKey")
- taxonomy := p.s.Taxonomies[plural].Get(term)
+ b := p.treeRef.n
+ viewInfo := b.viewInfo
+ taxonomy := p.s.Taxonomies()[viewInfo.name.plural].Get(viewInfo.termKey)
pages = taxonomy.Pages()
case page.KindTaxonomyTerm:
- pages = p.getPagesAndSections()
+ pages = p.bucket.getTaxonomies()
default:
pages = p.s.Pages()
}
return string(p.source.parsed.Input()[start:])
}
-func (p *pageState) Resources() resource.Resources {
- p.resourcesInit.Do(func() {
-
- sort := func() {
- sort.SliceStable(p.resources, func(i, j int) bool {
- ri, rj := p.resources[i], p.resources[j]
- if ri.ResourceType() < rj.ResourceType() {
- return true
- }
-
- p1, ok1 := ri.(page.Page)
- p2, ok2 := rj.(page.Page)
+func (p *pageState) sortResources() {
+ sort.SliceStable(p.resources, func(i, j int) bool {
+ ri, rj := p.resources[i], p.resources[j]
+ if ri.ResourceType() < rj.ResourceType() {
+ return true
+ }
- if ok1 != ok2 {
- return ok2
- }
+ p1, ok1 := ri.(page.Page)
+ p2, ok2 := rj.(page.Page)
- if ok1 {
- return page.DefaultPageSort(p1, p2)
- }
+ if ok1 != ok2 {
+ return ok2
+ }
- return ri.RelPermalink() < rj.RelPermalink()
- })
+ if ok1 {
+ return page.DefaultPageSort(p1, p2)
}
- sort()
+ return ri.RelPermalink() < rj.RelPermalink()
+ })
+}
+func (p *pageState) Resources() resource.Resources {
+ p.resourcesInit.Do(func() {
+ p.sortResources()
if len(p.m.resourcesMetadata) > 0 {
resources.AssignMetadata(p.m.resourcesMetadata, p.resources...)
- sort()
+ p.sortResources()
}
-
})
return p.resources
}
}
func (p *pageState) Site() page.Site {
- return &p.s.Info
+ return p.s.Info
}
func (p *pageState) String() string {
ps.OutputFormatsProvider = pp
ps.targetPathDescriptor = pp.targetPathDescriptor
ps.RefProvider = newPageRef(ps)
- ps.SitesProvider = &ps.s.Info
+ ps.SitesProvider = ps.s.Info
return nil
}
section = sections[0]
}
case page.KindTaxonomyTerm, page.KindTaxonomy:
- section = maps.GetString(p.bucket.meta, "singular")
-
+ b := p.getTreeRef().n
+ section = b.viewInfo.name.singular
default:
}
return p.m.contentConverter
}
-func (p *pageState) addResources(r ...resource.Resource) {
- p.resources = append(p.resources, r...)
-}
-
func (p *pageState) mapContent(bucket *pagesMapBucket, meta *pageMeta) error {
s := p.shortcodeState
// … it's safe to keep some "global" state
var currShortcode shortcode
var ordinal int
+ var frontMatterSet bool
Loop:
for {
p.s.BuildFlags.HasLateTemplate.CAS(false, true)
rn.AddBytes(it)
case it.IsFrontMatter():
- f := metadecoders.FormatFromFrontMatterType(it.Type)
+ f := pageparser.FormatFromFrontMatterType(it.Type)
m, err := metadecoders.Default.UnmarshalToMap(it.Val, f)
if err != nil {
if fe, ok := err.(herrors.FileError); ok {
if err := meta.setMetadata(bucket, p, m); err != nil {
return err
}
+ frontMatterSet = true
next := iter.Peek()
if !next.IsDone() {
}
}
+ if !frontMatterSet {
+ // Page content without front matter. Assign default front matter from
+ // cascades etc.
+ if err := meta.setMetadata(bucket, p, nil); err != nil {
+ return err
+ }
+ }
+
p.cmap = rn
return nil
return err
}
- if idx >= len(p.pageOutputs) {
- panic(fmt.Sprintf("invalid page state for %q: got output format index %d, have %d", p.pathOrTitle(), idx, len(p.pageOutputs)))
+ if len(p.pageOutputs) == 1 {
+ idx = 0
}
p.pageOutput = p.pageOutputs[idx]
-
if p.pageOutput == nil {
panic(fmt.Sprintf("pageOutput is nil for output idx %d", idx))
}
p.pageOutput.cp = cp
}
- for _, r := range p.Resources().ByType(pageResourceType) {
- rp := r.(*pageState)
- if err := rp.shiftToOutputFormat(isRenderingSite, idx); err != nil {
- return errors.Wrap(err, "failed to shift outputformat in Page resource")
- }
- }
-
return nil
}
return ""
}
-func (p *pageState) sourceRefs() []string {
- refs := []string{p.sourceRef()}
-
- if !p.File().IsZero() {
- meta := p.File().FileInfo().Meta()
- path := meta.PathFile()
-
- if path != "" {
- ref := "/" + filepath.ToSlash(path)
- if ref != refs[0] {
- refs = append(refs, ref)
- }
-
- }
- }
- return refs
-}
-
-type pageStatePages []*pageState
-
-// Implement sorting.
-func (ps pageStatePages) Len() int { return len(ps) }
-
-func (ps pageStatePages) Less(i, j int) bool { return page.DefaultPageSort(ps[i], ps[j]) }
-
-func (ps pageStatePages) Swap(i, j int) { ps[i], ps[j] = ps[j], ps[i] }
-
-// findPagePos Given a page, it will find the position in Pages
-// will return -1 if not found
-func (ps pageStatePages) findPagePos(page *pageState) int {
- for i, x := range ps {
- if x.File().Filename() == page.File().Filename() {
- return i
- }
- }
- return -1
-}
-
-func (ps pageStatePages) findPagePosByFilename(filename string) int {
- for i, x := range ps {
- if x.File().Filename() == filename {
- return i
- }
- }
- return -1
-}
-
-func (ps pageStatePages) findPagePosByFilnamePrefix(prefix string) int {
- if prefix == "" {
- return -1
- }
-
- lenDiff := -1
- currPos := -1
- prefixLen := len(prefix)
-
- // Find the closest match
- for i, x := range ps {
- if strings.HasPrefix(x.File().Filename(), prefix) {
- diff := len(x.File().Filename()) - prefixLen
- if lenDiff == -1 || diff < lenDiff {
- lenDiff = diff
- currPos = i
- }
- }
- }
- return currPos
-}
-
func (s *Site) sectionsFromFile(fi source.File) []string {
dirname := fi.Dir()
"github.com/gohugoio/hugo/resources/resource"
)
+type treeRefProvider interface {
+ getTreeRef() *contentTreeRef
+}
+
+func (p *pageCommon) getTreeRef() *contentTreeRef {
+ return p.treeRef
+}
+
+type nextPrevProvider interface {
+ getNextPrev() *nextPrev
+}
+
+func (p *pageCommon) getNextPrev() *nextPrev {
+ return p.posNextPrev
+}
+
+type nextPrevInSectionProvider interface {
+ getNextPrevInSection() *nextPrev
+}
+
+func (p *pageCommon) getNextPrevInSection() *nextPrev {
+ return p.posNextPrevSection
+}
+
type pageCommon struct {
s *Site
m *pageMeta
- bucket *pagesMapBucket
+ bucket *pagesMapBucket
+ treeRef *contentTreeRef
// Laziliy initialized dependencies.
init *lazy.Init
- metaInit sync.Once
- metaInitFn func(bucket *pagesMapBucket) error
-
// All of these represents the common parts of a page.Page
maps.Scratcher
navigation.PageMenusProvider
import (
"sync"
- "github.com/gohugoio/hugo/common/maps"
-
"github.com/gohugoio/hugo/resources/page"
)
switch p.Kind() {
case page.KindTaxonomy:
- bucket := p.bucket
- meta := bucket.meta
- plural := maps.GetString(meta, "plural")
- singular := maps.GetString(meta, "singular")
+ b := p.treeRef.n
+ name := b.viewInfo.name
+ termKey := b.viewInfo.termKey
- taxonomy := p.s.Taxonomies[plural].Get(maps.GetString(meta, "termKey"))
+ taxonomy := p.s.Taxonomies()[name.plural].Get(termKey)
- p.data[singular] = taxonomy
- p.data["Singular"] = meta["singular"]
- p.data["Plural"] = plural
- p.data["Term"] = meta["term"]
+ p.data[name.singular] = taxonomy
+ p.data["Singular"] = name.singular
+ p.data["Plural"] = name.plural
+ p.data["Term"] = b.viewInfo.term()
case page.KindTaxonomyTerm:
- bucket := p.bucket
- meta := bucket.meta
- plural := maps.GetString(meta, "plural")
- singular := maps.GetString(meta, "singular")
+ b := p.treeRef.n
+ name := b.viewInfo.name
- p.data["Singular"] = singular
- p.data["Plural"] = plural
- p.data["Terms"] = p.s.Taxonomies[plural]
+ p.data["Singular"] = name.singular
+ p.data["Plural"] = name.plural
+ p.data["Terms"] = p.s.Taxonomies()[name.plural]
// keep the following just for legacy reasons
p.data["OrderedIndex"] = p.data["Terms"]
p.data["Index"] = p.data["Terms"]
// a fixed pageOutput.
standalone bool
- bundleType string
+ draft bool // Only published when running with -D flag
+ buildConfig pagemeta.BuildConfig
+
+ bundleType files.ContentClass
// Params contains configuration defined in the params section of page frontmatter.
params map[string]interface{}
aliases []string
- draft bool
-
description string
keywords []string
resource.Dates
- // This is enabled if it is a leaf bundle (the "index.md" type) and it is marked as headless in front matter.
- // Being headless means that
- // 1. The page itself is not rendered to disk
- // 2. It is not available in .Site.Pages etc.
- // 3. But you can get it via .Site.GetPage
- headless bool
-
// Set if this page is bundled inside another.
bundled bool
return al
}
-func (p *pageMeta) BundleType() string {
+func (p *pageMeta) BundleType() files.ContentClass {
return p.bundleType
}
return p.weight
}
-func (pm *pageMeta) setMetadata(bucket *pagesMapBucket, p *pageState, frontmatter map[string]interface{}) error {
- if frontmatter == nil && bucket.cascade == nil {
- return errors.New("missing frontmatter data")
+func (pm *pageMeta) mergeBucketCascades(b1, b2 *pagesMapBucket) {
+ if b1.cascade == nil {
+ b1.cascade = make(map[string]interface{})
+ }
+ if b2 != nil && b2.cascade != nil {
+ for k, v := range b2.cascade {
+ if _, found := b1.cascade[k]; !found {
+ b1.cascade[k] = v
+ }
+ }
}
+}
+func (pm *pageMeta) setMetadata(parentBucket *pagesMapBucket, p *pageState, frontmatter map[string]interface{}) error {
pm.params = make(maps.Params)
+ if frontmatter == nil && (parentBucket == nil || parentBucket.cascade == nil) {
+ return nil
+ }
+
if frontmatter != nil {
// Needed for case insensitive fetching of params values
maps.ToLower(frontmatter)
- if p.IsNode() {
+ if p.bucket != nil {
// Check for any cascade define on itself.
if cv, found := frontmatter["cascade"]; found {
- cvm := maps.ToStringMap(cv)
- if bucket.cascade == nil {
- bucket.cascade = cvm
- } else {
- for k, v := range cvm {
- bucket.cascade[k] = v
- }
- }
- }
- }
-
- if bucket != nil && bucket.cascade != nil {
- for k, v := range bucket.cascade {
- if _, found := frontmatter[k]; !found {
- frontmatter[k] = v
- }
+ p.bucket.cascade = maps.ToStringMap(cv)
}
}
} else {
frontmatter = make(map[string]interface{})
- for k, v := range bucket.cascade {
+ }
+
+ var cascade map[string]interface{}
+
+ if p.bucket != nil {
+ if parentBucket != nil {
+ // Merge missing keys from parent into this.
+ pm.mergeBucketCascades(p.bucket, parentBucket)
+ }
+ cascade = p.bucket.cascade
+ } else if parentBucket != nil {
+ cascade = parentBucket.cascade
+ }
+
+ for k, v := range cascade {
+ if _, found := frontmatter[k]; !found {
frontmatter[k] = v
}
}
p.s.Log.ERROR.Printf("Failed to handle dates for page %q: %s", p.pathOrTitle(), err)
}
+ pm.buildConfig, err = pagemeta.DecodeBuildConfig(frontmatter["_build"])
+ if err != nil {
+ return err
+ }
+
var sitemapSet bool
var draft, published, isCJKLanguage *bool
pm.keywords = cast.ToStringSlice(v)
pm.params[loki] = pm.keywords
case "headless":
- // For now, only the leaf bundles ("index.md") can be headless (i.e. produce no output).
- // We may expand on this in the future, but that gets more complex pretty fast.
- if p.File().TranslationBaseName() == "index" {
- pm.headless = cast.ToBool(v)
+ // Legacy setting for leaf bundles.
+ // This is since Hugo 0.63 handled in a more general way for all
+ // pages.
+ isHeadless := cast.ToBool(v)
+ pm.params[loki] = isHeadless
+ if p.File().TranslationBaseName() == "index" && isHeadless {
+ pm.buildConfig.List = false
+ pm.buildConfig.Render = false
}
- pm.params[loki] = pm.headless
case "outputs":
o := cast.ToStringSlice(v)
if len(o) > 0 {
return nil
}
-func (p *pageMeta) applyDefaultValues(ps *pageState) error {
+func (p *pageMeta) noList() bool {
+ return !p.buildConfig.List
+}
+
+func (p *pageMeta) noRender() bool {
+ return !p.buildConfig.Render
+}
+
+func (p *pageMeta) applyDefaultValues(n *contentNode) error {
+ if p.buildConfig.IsZero() {
+ p.buildConfig, _ = pagemeta.DecodeBuildConfig(nil)
+ }
+
+ if !p.s.isEnabled(p.Kind()) {
+ (&p.buildConfig).Disable()
+ }
+
if p.markup == "" {
if !p.File().IsZero() {
// Fall back to file extension
case page.KindHome:
p.title = p.s.Info.title
case page.KindSection:
- sectionName := helpers.FirstUpper(p.sections[0])
+ var sectionName string
+ if n != nil {
+ sectionName = n.rootSection()
+ } else {
+ sectionName = p.sections[0]
+ }
+
+ sectionName = helpers.FirstUpper(sectionName)
if p.s.Cfg.GetBool("pluralizeListTitles") {
p.title = inflect.Pluralize(sectionName)
} else {
p.title = sectionName
}
case page.KindTaxonomy:
+ // TODO(bep) improve
key := p.sections[len(p.sections)-1]
p.title = strings.Replace(p.s.titleFunc(key), "-", " ", -1)
case page.KindTaxonomyTerm:
markup = "markdown"
}
- cp, err := p.newContentConverter(ps, markup, renderingConfigOverrides)
+ cp, err := p.newContentConverter(n.p, markup, renderingConfigOverrides)
if err != nil {
return err
}
}
func (p *pageMeta) newContentConverter(ps *pageState, markup string, renderingConfigOverrides map[string]interface{}) (converter.Converter, error) {
+ if ps == nil {
+ panic("no Page provided")
+ }
cp := p.s.ContentSpec.Converters.Get(markup)
if cp == nil {
return nil, errors.Errorf("no content renderer found for markup %q", p.markup)
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/source"
- "github.com/gohugoio/hugo/parser/pageparser"
- "github.com/pkg/errors"
-
"github.com/gohugoio/hugo/output"
"github.com/gohugoio/hugo/lazy"
"github.com/gohugoio/hugo/resources/page"
- "github.com/gohugoio/hugo/resources/resource"
)
func newPageBase(metaProvider *pageMeta) (*pageState, error) {
InternalDependencies: s,
init: lazy.New(),
m: metaProvider,
- s: s},
+ s: s,
+ },
}
siteAdapter := pageSiteAdapter{s: s, p: ps}
}
-func newPageFromMeta(meta map[string]interface{}, metaProvider *pageMeta) (*pageState, error) {
+func newPageBucket(p *pageState) *pagesMapBucket {
+ return &pagesMapBucket{owner: p}
+}
+
+func newPageFromMeta(
+ n *contentNode,
+ parentBucket *pagesMapBucket,
+ meta map[string]interface{},
+ metaProvider *pageMeta) (*pageState, error) {
+
if metaProvider.f == nil {
metaProvider.f = page.NewZeroFile(metaProvider.s.DistinctWarningLog)
}
return nil, err
}
- initMeta := func(bucket *pagesMapBucket) error {
- if meta != nil || bucket != nil {
- if err := metaProvider.setMetadata(bucket, ps, meta); err != nil {
- return ps.wrapError(err)
- }
- }
+ bucket := parentBucket
- if err := metaProvider.applyDefaultValues(ps); err != nil {
- return err
- }
+ if ps.IsNode() {
+ ps.bucket = newPageBucket(ps)
+ }
- return nil
+ if meta != nil || parentBucket != nil {
+ if err := metaProvider.setMetadata(bucket, ps, meta); err != nil {
+ return nil, ps.wrapError(err)
+ }
}
- if metaProvider.standalone {
- initMeta(nil)
- } else {
- // Because of possible cascade keywords, we need to delay this
- // until we have the complete page graph.
- ps.metaInitFn = initMeta
+ if err := metaProvider.applyDefaultValues(n); err != nil {
+ return nil, err
}
ps.init.Add(func() (interface{}, error) {
}
if ps.m.standalone {
- ps.pageOutput = makeOut(ps.m.outputFormats()[0], true)
+ ps.pageOutput = makeOut(ps.m.outputFormats()[0], !ps.m.noRender())
} else {
- ps.pageOutputs = make([]*pageOutput, len(ps.s.h.renderFormats))
- created := make(map[string]*pageOutput)
outputFormatsForPage := ps.m.outputFormats()
- for i, f := range ps.s.h.renderFormats {
- po, found := created[f.Name]
- if !found {
- _, shouldRender := outputFormatsForPage.GetByName(f.Name)
- po = makeOut(f, shouldRender)
- created[f.Name] = po
+
+ if !ps.m.noRender() {
+ ps.pageOutputs = make([]*pageOutput, len(ps.s.h.renderFormats))
+ created := make(map[string]*pageOutput)
+ for i, f := range ps.s.h.renderFormats {
+ po, found := created[f.Name]
+ if !found {
+ _, shouldRender := outputFormatsForPage.GetByName(f.Name)
+ po = makeOut(f, shouldRender)
+ created[f.Name] = po
+ }
+ ps.pageOutputs[i] = po
}
- ps.pageOutputs[i] = po
+ } else {
+ // We need one output format for potential resources to publish.
+ ps.pageOutputs = []*pageOutput{makeOut(outputFormatsForPage[0], false)}
}
}
func newPageStandalone(m *pageMeta, f output.Format) (*pageState, error) {
m.configuredOutputFormats = output.Formats{f}
m.standalone = true
- p, err := newPageFromMeta(nil, m)
+ p, err := newPageFromMeta(nil, nil, nil, m)
if err != nil {
return nil, err
}
-func newPageWithContent(f *fileInfo, s *Site, bundled bool, content resource.OpenReadSeekCloser) (*pageState, error) {
- sections := s.sectionsFromFile(f)
- kind := s.kindFromFileInfoOrSections(f, sections)
- if kind == page.KindTaxonomy {
- s.PathSpec.MakePathsSanitized(sections)
- }
-
- metaProvider := &pageMeta{kind: kind, sections: sections, bundled: bundled, s: s, f: f}
-
- ps, err := newPageBase(metaProvider)
- if err != nil {
- return nil, err
- }
-
- gi, err := s.h.gitInfoForPage(ps)
- if err != nil {
- return nil, errors.Wrap(err, "failed to load Git data")
- }
- ps.gitInfo = gi
-
- r, err := content()
- if err != nil {
- return nil, err
- }
- defer r.Close()
-
- parseResult, err := pageparser.Parse(
- r,
- pageparser.Config{EnableEmoji: s.siteCfg.enableEmoji},
- )
- if err != nil {
- return nil, err
- }
-
- ps.pageContent = pageContent{
- source: rawPageContent{
- parsed: parseResult,
- posMainContent: -1,
- posSummaryEnd: -1,
- posBodyStart: -1,
- },
- }
-
- ps.shortcodeState = newShortcodeHandler(ps, ps.s, nil)
-
- ps.metaInitFn = func(bucket *pagesMapBucket) error {
- if err := ps.mapContent(bucket, metaProvider); err != nil {
- return ps.wrapError(err)
- }
-
- if err := metaProvider.applyDefaultValues(ps); err != nil {
- return err
- }
-
- return nil
- }
-
- ps.init.Add(func() (interface{}, error) {
-
- pp, err := newPagePaths(s, ps, metaProvider)
- if err != nil {
- return nil, err
- }
-
- // Prepare output formats for all sites.
- ps.pageOutputs = make([]*pageOutput, len(ps.s.h.renderFormats))
- created := make(map[string]*pageOutput)
- outputFormatsForPage := ps.m.outputFormats()
-
- for i, f := range ps.s.h.renderFormats {
- if po, found := created[f.Name]; found {
- ps.pageOutputs[i] = po
- continue
- }
-
- _, render := outputFormatsForPage.GetByName(f.Name)
- po := newPageOutput(ps, pp, f, render)
-
- // Create a content provider for the first,
- // we may be able to reuse it.
- if i == 0 {
- contentProvider, err := newPageContentOutput(ps, po)
- if err != nil {
- return nil, err
- }
- po.initContentProvider(contentProvider)
- }
-
- ps.pageOutputs[i] = po
- created[f.Name] = po
- }
-
- if err := ps.initCommonProviders(pp); err != nil {
- return nil, err
- }
-
- return nil, nil
- })
-
- return ps, nil
-}
-
type pageDeprecatedWarning struct {
p *pageState
}
ft, found := pp.targetPaths[f.Name]
if !found {
// Link to the main output format
- ft = pp.targetPaths[pp.OutputFormats()[0].Format.Name]
+ ft = pp.targetPaths[pp.firstOutputFormat.Format.Name]
}
targetPathsProvider = ft
linksProvider = ft
}
outputFormats := pm.outputFormats()
- if len(outputFormats) == 0 {
- outputFormats = pm.s.outputFormats[pm.Kind()]
- }
-
if len(outputFormats) == 0 {
return pagePaths{}, nil
}
- if pm.headless {
+ if pm.noRender() {
outputFormats = outputFormats[:1]
}
var relPermalink, permalink string
- // If a page is headless or bundled in another, it will not get published
- // on its own and it will have no links.
- if !pm.headless && !pm.bundled {
+ // If a page is headless or marked as "no render", or bundled in another,
+ // it will not get published on its own and it will have no links.
+ if !pm.noRender() && !pm.bundled {
relPermalink = paths.RelPermalink(s.PathSpec)
permalink = paths.PermalinkForOutputFormat(s.PathSpec, f)
}
}
+ var out page.OutputFormats
+ if !pm.noRender() {
+ out = pageOutputFormats
+ }
+
return pagePaths{
- outputFormats: pageOutputFormats,
+ outputFormats: out,
+ firstOutputFormat: pageOutputFormats[0],
targetPaths: targets,
targetPathDescriptor: targetPathDescriptor,
}, nil
}
type pagePaths struct {
- outputFormats page.OutputFormats
+ outputFormats page.OutputFormats
+ firstOutputFormat page.OutputFormat
targetPaths map[string]targetPathsHolder
targetPathDescriptor page.TargetPathDescriptor
package hugolib
import (
+ "path"
+ "strings"
+
"github.com/gohugoio/hugo/common/types"
- "github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/resources/page"
)
return false, nil
}
- pp, err := unwrapPage(other)
- if err != nil || pp == nil {
- return false, err
+ tp, ok := other.(treeRefProvider)
+ if !ok {
+ return false, nil
}
- if pt.p.Kind() == page.KindPage && len(pt.p.SectionsEntries()) == len(pp.SectionsEntries()) {
- // A regular page is never its section's ancestor.
+ ref1, ref2 := pt.p.getTreeRef(), tp.getTreeRef()
+
+ if !ref1.isSection() {
return false, nil
}
- return helpers.HasStringsPrefix(pp.SectionsEntries(), pt.p.SectionsEntries()), nil
+ return strings.HasPrefix(ref2.key, ref1.key), nil
}
func (pt pageTree) CurrentSection() page.Page {
if pt.p == nil {
return false, nil
}
- pp, err := unwrapPage(other)
- if err != nil || pp == nil {
- return false, err
+
+ tp, ok := other.(treeRefProvider)
+ if !ok {
+ return false, nil
}
- if pp.Kind() == page.KindPage && len(pt.p.SectionsEntries()) == len(pp.SectionsEntries()) {
- // A regular page is never its section's descendant.
+ ref1, ref2 := pt.p.getTreeRef(), tp.getTreeRef()
+
+ if !ref2.isSection() {
return false, nil
}
- return helpers.HasStringsPrefix(pt.p.SectionsEntries(), pp.SectionsEntries()), nil
-}
-func (pt pageTree) FirstSection() page.Page {
- p := pt.p
+ return strings.HasPrefix(ref1.key, ref2.key), nil
- parent := p.Parent()
+}
- if types.IsNil(parent) || parent.IsHome() {
- return p
+func (pt pageTree) FirstSection() page.Page {
+ ref := pt.p.getTreeRef()
+ key := ref.key
+ if !ref.isSection() {
+ key = path.Dir(key)
}
-
- for {
- current := parent
- parent = parent.Parent()
- if types.IsNil(parent) || parent.IsHome() {
- return current
- }
+ _, b := ref.m.getFirstSection(key)
+ if b == nil {
+ return nil
}
-
+ return b.p
}
func (pt pageTree) InSection(other interface{}) (bool, error) {
return false, nil
}
- pp, err := unwrapPage(other)
- if err != nil {
- return false, err
- }
-
- if pp == nil {
+ tp, ok := other.(treeRefProvider)
+ if !ok {
return false, nil
}
- return pp.CurrentSection().Eq(pt.p.CurrentSection()), nil
+ ref1, ref2 := pt.p.getTreeRef(), tp.getTreeRef()
+
+ s1, _ := ref1.getCurrentSection()
+ s2, _ := ref2.getCurrentSection()
+
+ return s1 == s2, nil
}
}
func (pt pageTree) Parent() page.Page {
- if pt.p.parent != nil {
- return pt.p.parent
+ p := pt.p
+
+ if p.parent != nil {
+ return p.parent
+ }
+
+ if pt.p.IsHome() {
+ return nil
}
- if pt.p.bucket == nil || pt.p.bucket.parent == nil {
+ _, b := p.getTreeRef().getSection()
+ if b == nil {
return nil
}
- return pt.p.bucket.parent.owner
+ return b.p
}
func (pt pageTree) Sections() page.Pages {
// This is all the kinds we can expect to find in .Site.Pages.
allKindsInPages = []string{page.KindPage, page.KindHome, page.KindSection, page.KindTaxonomy, page.KindTaxonomyTerm}
- allKinds = append(allKindsInPages, []string{kindRSS, kindSitemap, kindRobotsTXT, kind404}...)
)
const (
s := b.H.Sites[0]
checkDate := func(t time.Time, msg string) {
- b.Assert(t.Year(), qt.Equals, 2017)
+ b.Assert(t.Year(), qt.Equals, 2017, qt.Commentf(msg))
}
checkDated := func(d resource.Dated, msg string) {
b.Assert(len(b.H.Sites), qt.Equals, 1)
s := b.H.Sites[0]
- b.Assert(s.getPage("/").Date().Year(), qt.Equals, 2017)
+ b.Assert(s.getPage("/").Date().Year(), qt.Equals, 2018)
b.Assert(s.getPage("/no-index").Date().Year(), qt.Equals, 2017)
b.Assert(s.getPage("/with-index-no-date").Date().IsZero(), qt.Equals, true)
b.Assert(s.getPage("/with-index-date").Date().Year(), qt.Equals, 2018)
"strings"
"testing"
+ "github.com/gohugoio/hugo/hugofs/files"
+
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
c.Assert(len(s.RegularPages()), qt.Equals, 8)
singlePage := s.getPage(page.KindPage, "a/1.md")
- c.Assert(singlePage.BundleType(), qt.Equals, "")
+ c.Assert(singlePage.BundleType(), qt.Equals, files.ContentClass(""))
c.Assert(singlePage, qt.Not(qt.IsNil))
c.Assert(s.getPage("page", "a/1"), qt.Equals, singlePage)
leafBundle1 := s.getPage(page.KindPage, "b/my-bundle/index.md")
c.Assert(leafBundle1, qt.Not(qt.IsNil))
- c.Assert(leafBundle1.BundleType(), qt.Equals, "leaf")
+ c.Assert(leafBundle1.BundleType(), qt.Equals, files.ContentClassLeaf)
c.Assert(leafBundle1.Section(), qt.Equals, "b")
sectionB := s.getPage(page.KindSection, "b")
c.Assert(sectionB, qt.Not(qt.IsNil))
home, _ := s.Info.Home()
- c.Assert(home.BundleType(), qt.Equals, "branch")
+ c.Assert(home.BundleType(), qt.Equals, files.ContentClassBranch)
// This is a root bundle and should live in the "home section"
// See https://github.com/gohugoio/hugo/issues/4332
c.Assert(len(s.Pages()), qt.Equals, 16)
// No nn pages
c.Assert(len(s.AllPages()), qt.Equals, 16)
- for _, p := range s.rawAllPages {
- c.Assert(p.Language().Lang != "nn", qt.Equals, true)
- }
- for _, p := range s.AllPages() {
+ s.pageMap.withEveryBundlePage(func(p *pageState) bool {
c.Assert(p.Language().Lang != "nn", qt.Equals, true)
- }
+ return false
+ })
}
s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{})
c.Assert(len(s.RegularPages()), qt.Equals, 1)
- c.Assert(len(s.headlessPages), qt.Equals, 1)
regular := s.getPage(page.KindPage, "a/index")
c.Assert(regular.RelPermalink(), qt.Equals, "/s1/")
defaultContentLanguage = "en"
defaultContentLanguageInSubDir = true
disableKinds = ["taxonomyTerm", "taxonomy"]
-
[languages]
[languages.nn]
languageName = "Nynorsk"
weight = 2
title = "Tittel på Nynorsk"
-
[languages.en]
title = "Title in English"
languageName = "English"
weight = 1
-
`
pageContent := func(id string) string {
"fmt"
"path"
"path/filepath"
- "sort"
"strings"
"sync"
- "time"
- "github.com/gohugoio/hugo/resources/resource"
+ "github.com/gohugoio/hugo/common/herrors"
- "github.com/pkg/errors"
+ "github.com/gohugoio/hugo/helpers"
- "github.com/gohugoio/hugo/cache"
"github.com/gohugoio/hugo/resources/page"
)
-// Used in the page cache to mark more than one hit for a given key.
-var ambiguityFlag = &pageState{}
-
// PageCollections contains the page collections for a site.
type PageCollections struct {
- pagesMap *pagesMap
-
- // Includes absolute all pages (of all types), including drafts etc.
- rawAllPages pageStatePages
-
- // rawAllPages plus additional pages created during the build process.
- workAllPages pageStatePages
-
- // Includes headless bundles, i.e. bundles that produce no output for its content page.
- headlessPages pageStatePages
+ pageMap *pageMap
// Lazy initialized page collections
pages *lazyPagesFactory
regularPages *lazyPagesFactory
allPages *lazyPagesFactory
allRegularPages *lazyPagesFactory
-
- // The index for .Site.GetPage etc.
- pageIndex *cache.Lazy
}
// Pages returns all pages.
return c.allRegularPages.get()
}
-// Get initializes the index if not already done so, then
-// looks up the given page ref, returns nil if no value found.
-func (c *PageCollections) getFromCache(ref string) (page.Page, error) {
- v, found, err := c.pageIndex.Get(ref)
- if err != nil {
- return nil, err
- }
- if !found {
- return nil, nil
- }
-
- p := v.(page.Page)
-
- if p != ambiguityFlag {
- return p, nil
- }
- return nil, fmt.Errorf("page reference %q is ambiguous", ref)
-}
-
type lazyPagesFactory struct {
pages page.Pages
return &lazyPagesFactory{factory: factory}
}
-func newPageCollections() *PageCollections {
- return newPageCollectionsFromPages(nil)
-}
-
-func newPageCollectionsFromPages(pages pageStatePages) *PageCollections {
+func newPageCollections(m *pageMap) *PageCollections {
+ if m == nil {
+ panic("must provide a pageMap")
+ }
- c := &PageCollections{rawAllPages: pages}
+ c := &PageCollections{pageMap: m}
c.pages = newLazyPagesFactory(func() page.Pages {
- pages := make(page.Pages, len(c.workAllPages))
- for i, p := range c.workAllPages {
- pages[i] = p
- }
- return pages
+ return m.createListAllPages()
})
c.regularPages = newLazyPagesFactory(func() page.Pages {
- return c.findPagesByKindInWorkPages(page.KindPage, c.workAllPages)
- })
-
- c.pageIndex = cache.NewLazy(func() (map[string]interface{}, error) {
- index := make(map[string]interface{})
-
- add := func(ref string, p page.Page) {
- ref = strings.ToLower(ref)
- existing := index[ref]
- if existing == nil {
- index[ref] = p
- } else if existing != ambiguityFlag && existing != p {
- index[ref] = ambiguityFlag
- }
- }
-
- for _, pageCollection := range []pageStatePages{c.workAllPages, c.headlessPages} {
- for _, p := range pageCollection {
- if p.IsPage() {
- sourceRefs := p.sourceRefs()
- for _, ref := range sourceRefs {
- add(ref, p)
- }
- sourceRef := sourceRefs[0]
-
- // Ref/Relref supports this potentially ambiguous lookup.
- add(p.File().LogicalName(), p)
-
- translationBaseName := p.File().TranslationBaseName()
-
- dir, _ := path.Split(sourceRef)
- dir = strings.TrimSuffix(dir, "/")
-
- if translationBaseName == "index" {
- add(dir, p)
- add(path.Base(dir), p)
- } else {
- add(translationBaseName, p)
- }
-
- // We need a way to get to the current language version.
- pathWithNoExtensions := path.Join(dir, translationBaseName)
- add(pathWithNoExtensions, p)
- } else {
- sourceRefs := p.sourceRefs()
- for _, ref := range sourceRefs {
- add(ref, p)
- }
-
- ref := p.SectionsPath()
-
- // index the canonical, unambiguous virtual ref
- // e.g. /section
- // (this may already have been indexed above)
- add("/"+ref, p)
- }
- }
- }
-
- return index, nil
+ return c.findPagesByKindIn(page.KindPage, c.pages.get())
})
return c
return p
}
-// Case insensitive page lookup.
-func (c *PageCollections) getPageNew(context page.Page, ref string) (page.Page, error) {
- var anError error
-
- ref = strings.ToLower(ref)
-
- // Absolute (content root relative) reference.
- if strings.HasPrefix(ref, "/") {
- p, err := c.getFromCache(ref)
- if err == nil && p != nil {
- return p, nil
- }
- if err != nil {
- anError = err
- }
-
- } else if context != nil {
- // Try the page-relative path.
- var dir string
- if !context.File().IsZero() {
- dir = filepath.ToSlash(context.File().Dir())
- } else {
- dir = context.SectionsPath()
- }
- ppath := path.Join("/", strings.ToLower(dir), ref)
-
- p, err := c.getFromCache(ppath)
- if err == nil && p != nil {
- return p, nil
- }
- if err != nil {
- anError = err
- }
- }
-
- if !strings.HasPrefix(ref, "/") {
- // Many people will have "post/foo.md" in their content files.
- p, err := c.getFromCache("/" + ref)
- if err == nil && p != nil {
- return p, nil
- }
- if err != nil {
- anError = err
- }
- }
-
- // Last try.
- ref = strings.TrimPrefix(ref, "/")
- p, err := c.getFromCache(ref)
- if err != nil {
- anError = err
- }
-
- if p == nil && anError != nil {
- return nil, wrapErr(errors.Wrap(anError, "failed to resolve ref"), context)
+// getPageRef resolves a Page from ref/relRef, with a slightly more comprehensive
+// search path than getPageNew.
+func (c *PageCollections) getPageRef(context page.Page, ref string) (page.Page, error) {
+ n, err := c.getContentNode(context, true, ref)
+ if err != nil || n == nil || n.p == nil {
+ return nil, err
}
-
- return p, nil
+ return n.p, nil
}
-func (*PageCollections) findPagesByKindIn(kind string, inPages page.Pages) page.Pages {
- var pages page.Pages
- for _, p := range inPages {
- if p.Kind() == kind {
- pages = append(pages, p)
- }
+func (c *PageCollections) getPageNew(context page.Page, ref string) (page.Page, error) {
+ n, err := c.getContentNode(context, false, ref)
+ if err != nil || n == nil || n.p == nil {
+ return nil, err
}
- return pages
+ return n.p, nil
}
-func (c *PageCollections) findPagesByKind(kind string) page.Pages {
- return c.findPagesByKindIn(kind, c.Pages())
-}
+func (c *PageCollections) getSectionOrPage(ref string) (*contentNode, string) {
+ var n *contentNode
-func (c *PageCollections) findWorkPagesByKind(kind string) pageStatePages {
- var pages pageStatePages
- for _, p := range c.workAllPages {
- if p.Kind() == kind {
- pages = append(pages, p)
- }
- }
- return pages
-}
+ s, v, found := c.pageMap.sections.LongestPrefix(ref)
-func (*PageCollections) findPagesByKindInWorkPages(kind string, inPages pageStatePages) page.Pages {
- var pages page.Pages
- for _, p := range inPages {
- if p.Kind() == kind {
- pages = append(pages, p)
- }
+ if found {
+ n = v.(*contentNode)
}
- return pages
-}
-func (c *PageCollections) addPage(page *pageState) {
- c.rawAllPages = append(c.rawAllPages, page)
-}
-
-func (c *PageCollections) removePageFilename(filename string) {
- if i := c.rawAllPages.findPagePosByFilename(filename); i >= 0 {
- c.clearResourceCacheForPage(c.rawAllPages[i])
- c.rawAllPages = append(c.rawAllPages[:i], c.rawAllPages[i+1:]...)
+ if found && s == ref {
+ // A section
+ return n, ""
}
-}
+ m := c.pageMap
+ filename := strings.TrimPrefix(strings.TrimPrefix(ref, s), "/")
+ langSuffix := "." + m.s.Lang()
-func (c *PageCollections) removePage(page *pageState) {
- if i := c.rawAllPages.findPagePos(page); i >= 0 {
- c.clearResourceCacheForPage(c.rawAllPages[i])
- c.rawAllPages = append(c.rawAllPages[:i], c.rawAllPages[i+1:]...)
- }
-}
+ // Trim both extension and any language code.
+ name := helpers.PathNoExt(filename)
+ name = strings.TrimSuffix(name, langSuffix)
-func (c *PageCollections) replacePage(page *pageState) {
- // will find existing page that matches filepath and remove it
- c.removePage(page)
- c.addPage(page)
-}
+ // These are reserved bundle names and will always be stored by their owning
+ // folder name.
+ name = strings.TrimSuffix(name, "/index")
+ name = strings.TrimSuffix(name, "/_index")
-func (c *PageCollections) clearResourceCacheForPage(page *pageState) {
- if len(page.resources) > 0 {
- page.s.ResourceSpec.DeleteCacheByPrefix(page.targetPaths().SubResourceBaseTarget)
+ if !found {
+ return nil, name
}
-}
-func (c *PageCollections) assemblePagesMap(s *Site) error {
+ // Check if it's a section with filename provided.
+ if !n.p.File().IsZero() && n.p.File().LogicalName() == filename {
+ return n, name
+ }
- c.pagesMap = newPagesMap(s)
+ return m.getPage(s, name), name
- rootSections := make(map[string]bool)
+}
- // Add all branch nodes first.
- for _, p := range c.rawAllPages {
- rootSections[p.Section()] = true
- if p.IsPage() {
- continue
- }
- c.pagesMap.addPage(p)
+func (c *PageCollections) getContentNode(context page.Page, isReflink bool, ref string) (*contentNode, error) {
+ defer herrors.Recover()
+ ref = filepath.ToSlash(strings.ToLower(strings.TrimSpace(ref)))
+ if ref == "" {
+ ref = "/"
}
+ inRef := ref
- // Create missing home page and the first level sections if no
- // _index provided.
- s.home = c.pagesMap.getOrCreateHome()
- for k := range rootSections {
- c.pagesMap.createSectionIfNotExists(k)
+ var doSimpleLookup bool
+ if isReflink || context == nil {
+ // For Ref/Reflink and .Site.GetPage do simple name lookups for the potentially ambigous myarticle.md and /myarticle.md,
+ // but not when we get ./myarticle*, section/myarticle.
+ doSimpleLookup = ref[0] != '.' || ref[0] == '/' && strings.Count(ref, "/") == 1
}
- // Attach the regular pages to their section.
- for _, p := range c.rawAllPages {
- if p.IsNode() {
- continue
+ if context != nil && !strings.HasPrefix(ref, "/") {
+ // Try the page-relative path.
+ var base string
+ if context.File().IsZero() {
+ base = context.SectionsPath()
+ } else {
+ base = filepath.ToSlash(filepath.Dir(context.File().FileInfo().Meta().Path()))
}
- c.pagesMap.addPage(p)
+ ref = path.Join("/", strings.ToLower(base), ref)
}
- return nil
-}
-
-func (c *PageCollections) createWorkAllPages() error {
- c.workAllPages = make(pageStatePages, 0, len(c.rawAllPages))
- c.headlessPages = make(pageStatePages, 0)
-
- var (
- homeDates *resource.Dates
- sectionDates *resource.Dates
- siteLastmod time.Time
- siteLastDate time.Time
-
- sectionsParamId = "mainSections"
- sectionsParamIdLower = strings.ToLower(sectionsParamId)
- )
-
- mainSections, mainSectionsFound := c.pagesMap.s.Info.Params()[sectionsParamIdLower]
-
- var (
- bucketsToRemove []string
- rootBuckets []*pagesMapBucket
- walkErr error
- )
-
- c.pagesMap.r.Walk(func(s string, v interface{}) bool {
- bucket := v.(*pagesMapBucket)
- parentBucket := c.pagesMap.parentBucket(s)
-
- if parentBucket != nil {
-
- if !mainSectionsFound && strings.Count(s, "/") == 1 && bucket.owner.IsSection() {
- // Root section
- rootBuckets = append(rootBuckets, bucket)
- }
- }
-
- if bucket.owner.IsHome() {
- if resource.IsZeroDates(bucket.owner) {
- // Calculate dates from the page tree.
- homeDates = &bucket.owner.m.Dates
- }
- }
+ if !strings.HasPrefix(ref, "/") {
+ ref = "/" + ref
+ }
- sectionDates = nil
- if resource.IsZeroDates(bucket.owner) {
- sectionDates = &bucket.owner.m.Dates
- }
+ m := c.pageMap
- if parentBucket != nil {
- bucket.parent = parentBucket
- if bucket.owner.IsSection() {
- parentBucket.bucketSections = append(parentBucket.bucketSections, bucket)
- }
- }
+ // It's either a section, a page in a section or a taxonomy node.
+ // Start with the most likely:
+ n, name := c.getSectionOrPage(ref)
+ if n != nil {
+ return n, nil
+ }
- if bucket.isEmpty() {
- if bucket.owner.IsSection() && bucket.owner.File().IsZero() {
- // Check for any nested section.
- var hasDescendant bool
- c.pagesMap.r.WalkPrefix(s, func(ss string, v interface{}) bool {
- if s != ss {
- hasDescendant = true
- return true
- }
- return false
- })
- if !hasDescendant {
- // This is an auto-created section with, now, nothing in it.
- bucketsToRemove = append(bucketsToRemove, s)
- return false
- }
- }
+ if !strings.HasPrefix(inRef, "/") {
+ // Many people will have "post/foo.md" in their content files.
+ if n, _ := c.getSectionOrPage("/" + inRef); n != nil {
+ return n, nil
}
+ }
- if !bucket.disabled {
- c.workAllPages = append(c.workAllPages, bucket.owner)
+ // Check if it's a taxonomy node
+ s, v, found := m.taxonomies.LongestPrefix(ref)
+ if found {
+ if !m.onSameLevel(ref, s) {
+ return nil, nil
}
+ return v.(*contentNode), nil
+ }
- if !bucket.view {
- for _, p := range bucket.headlessPages {
- ps := p.(*pageState)
- ps.parent = bucket.owner
- c.headlessPages = append(c.headlessPages, ps)
- }
- for _, p := range bucket.pages {
- ps := p.(*pageState)
- ps.parent = bucket.owner
- c.workAllPages = append(c.workAllPages, ps)
-
- if homeDates != nil {
- homeDates.UpdateDateAndLastmodIfAfter(ps)
- }
-
- if sectionDates != nil {
- sectionDates.UpdateDateAndLastmodIfAfter(ps)
- }
-
- if p.Lastmod().After(siteLastmod) {
- siteLastmod = p.Lastmod()
- }
- if p.Date().After(siteLastDate) {
- siteLastDate = p.Date()
- }
+ getByName := func(s string) (*contentNode, error) {
+ n := m.pageReverseIndex.Get(s)
+ if n != nil {
+ if n == ambigousContentNode {
+ return nil, fmt.Errorf("page reference %q is ambiguous", ref)
}
+ return n, nil
}
- return false
- })
-
- if walkErr != nil {
- return walkErr
+ return nil, nil
}
- c.pagesMap.s.lastmod = siteLastmod
-
- if !mainSectionsFound {
+ var module string
+ if context != nil && !context.File().IsZero() {
+ module = context.File().FileInfo().Meta().Module()
+ }
- // Calculare main section
- var (
- maxRootBucketWeight int
- maxRootBucket *pagesMapBucket
- )
+ if module == "" && !c.pageMap.s.home.File().IsZero() {
+ module = c.pageMap.s.home.File().FileInfo().Meta().Module()
+ }
- for _, b := range rootBuckets {
- weight := len(b.pages) + (len(b.bucketSections) * 5)
- if weight >= maxRootBucketWeight {
- maxRootBucket = b
- maxRootBucketWeight = weight
- }
+ if module != "" {
+ n, err := getByName(module + ref)
+ if err != nil {
+ return nil, err
}
-
- if maxRootBucket != nil {
- // Try to make this as backwards compatible as possible.
- mainSections = []string{maxRootBucket.owner.Section()}
+ if n != nil {
+ return n, nil
}
}
- c.pagesMap.s.Info.Params()[sectionsParamId] = mainSections
- c.pagesMap.s.Info.Params()[sectionsParamIdLower] = mainSections
-
- for _, key := range bucketsToRemove {
- c.pagesMap.r.Delete(key)
+ if !doSimpleLookup {
+ return nil, nil
}
- sort.Sort(c.workAllPages)
+ // Ref/relref supports this potentially ambigous lookup.
+ return getByName(name)
+
+}
- return nil
+func (*PageCollections) findPagesByKindIn(kind string, inPages page.Pages) page.Pages {
+ var pages page.Pages
+ for _, p := range inPages {
+ if p.Kind() == kind {
+ pages = append(pages, p)
+ }
+ }
+ return pages
}
}
}
-func BenchmarkGetPageRegular(b *testing.B) {
+func createGetPageRegularBenchmarkSite(t testing.TB) *Site {
+
var (
- c = qt.New(b)
+ c = qt.New(t)
cfg, fs = newTestCfg()
- r = rand.New(rand.NewSource(time.Now().UnixNano()))
)
+ pc := func(title string) string {
+ return fmt.Sprintf(pageCollectionsPageTemplate, title)
+ }
+
for i := 0; i < 10; i++ {
for j := 0; j < 100; j++ {
- content := fmt.Sprintf(pageCollectionsPageTemplate, fmt.Sprintf("Title%d_%d", i, j))
- writeSource(b, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", j)), content)
+ content := pc(fmt.Sprintf("Title%d_%d", i, j))
+ writeSource(c, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", j)), content)
}
}
- s := buildSingleSite(b, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true})
+ return buildSingleSite(c, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true})
- pagePaths := make([]string, b.N)
+}
- for i := 0; i < b.N; i++ {
- pagePaths[i] = path.Join(fmt.Sprintf("sect%d", r.Intn(10)), fmt.Sprintf("page%d.md", r.Intn(100)))
- }
+func TestBenchmarkGetPageRegular(t *testing.T) {
+ c := qt.New(t)
+ s := createGetPageRegularBenchmarkSite(t)
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- page, _ := s.getPageNew(nil, pagePaths[i])
- c.Assert(page, qt.Not(qt.IsNil))
+ for i := 0; i < 10; i++ {
+ pp := path.Join("/", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", i))
+ page, _ := s.getPageNew(nil, pp)
+ c.Assert(page, qt.Not(qt.IsNil), qt.Commentf(pp))
}
}
-type testCase struct {
+func BenchmarkGetPageRegular(b *testing.B) {
+ r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+ b.Run("From root", func(b *testing.B) {
+ s := createGetPageRegularBenchmarkSite(b)
+ c := qt.New(b)
+
+ pagePaths := make([]string, b.N)
+
+ for i := 0; i < b.N; i++ {
+ pagePaths[i] = path.Join(fmt.Sprintf("/sect%d", r.Intn(10)), fmt.Sprintf("page%d.md", r.Intn(100)))
+ }
+
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ page, _ := s.getPageNew(nil, pagePaths[i])
+ c.Assert(page, qt.Not(qt.IsNil))
+ }
+ })
+
+ b.Run("Page relative", func(b *testing.B) {
+ s := createGetPageRegularBenchmarkSite(b)
+ c := qt.New(b)
+ allPages := s.RegularPages()
+
+ pagePaths := make([]string, b.N)
+ pages := make([]page.Page, b.N)
+
+ for i := 0; i < b.N; i++ {
+ pagePaths[i] = fmt.Sprintf("page%d.md", r.Intn(100))
+ pages[i] = allPages[r.Intn(len(allPages)/3)]
+ }
+
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ page, _ := s.getPageNew(pages[i], pagePaths[i])
+ c.Assert(page, qt.Not(qt.IsNil))
+ }
+ })
+
+}
+
+type getPageTest struct {
+ name string
kind string
context page.Page
- path []string
+ pathVariants []string
expectedTitle string
}
-func (t *testCase) check(p page.Page, err error, errorMsg string, c *qt.C) {
+func (t *getPageTest) check(p page.Page, err error, errorMsg string, c *qt.C) {
+ c.Helper()
errorComment := qt.Commentf(errorMsg)
switch t.kind {
case "Ambiguous":
c = qt.New(t)
)
+ pc := func(title string) string {
+ return fmt.Sprintf(pageCollectionsPageTemplate, title)
+ }
+
for i := 0; i < 10; i++ {
for j := 0; j < 10; j++ {
- content := fmt.Sprintf(pageCollectionsPageTemplate, fmt.Sprintf("Title%d_%d", i, j))
+ content := pc(fmt.Sprintf("Title%d_%d", i, j))
writeSource(t, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", j)), content)
}
}
- content := fmt.Sprintf(pageCollectionsPageTemplate, "home page")
+ content := pc("home page")
writeSource(t, fs, filepath.Join("content", "_index.md"), content)
- content = fmt.Sprintf(pageCollectionsPageTemplate, "about page")
+ content = pc("about page")
writeSource(t, fs, filepath.Join("content", "about.md"), content)
- content = fmt.Sprintf(pageCollectionsPageTemplate, "section 3")
+ content = pc("section 3")
writeSource(t, fs, filepath.Join("content", "sect3", "_index.md"), content)
- content = fmt.Sprintf(pageCollectionsPageTemplate, "UniqueBase")
- writeSource(t, fs, filepath.Join("content", "sect3", "unique.md"), content)
+ writeSource(t, fs, filepath.Join("content", "sect3", "unique.md"), pc("UniqueBase"))
+ writeSource(t, fs, filepath.Join("content", "sect3", "Unique2.md"), pc("UniqueBase2"))
- content = fmt.Sprintf(pageCollectionsPageTemplate, "another sect7")
+ content = pc("another sect7")
writeSource(t, fs, filepath.Join("content", "sect3", "sect7", "_index.md"), content)
- content = fmt.Sprintf(pageCollectionsPageTemplate, "deep page")
+ content = pc("deep page")
writeSource(t, fs, filepath.Join("content", "sect3", "subsect", "deep.md"), content)
+ // Bundle variants
+ writeSource(t, fs, filepath.Join("content", "sect3", "b1", "index.md"), pc("b1 bundle"))
+ writeSource(t, fs, filepath.Join("content", "sect3", "index", "index.md"), pc("index bundle"))
+
s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true})
sec3, err := s.getPageNew(nil, "/sect3")
c.Assert(err, qt.IsNil)
c.Assert(sec3, qt.Not(qt.IsNil))
- tests := []testCase{
+ tests := []getPageTest{
// legacy content root relative paths
- {page.KindHome, nil, []string{}, "home page"},
- {page.KindPage, nil, []string{"about.md"}, "about page"},
- {page.KindSection, nil, []string{"sect3"}, "section 3"},
- {page.KindPage, nil, []string{"sect3/page1.md"}, "Title3_1"},
- {page.KindPage, nil, []string{"sect4/page2.md"}, "Title4_2"},
- {page.KindSection, nil, []string{"sect3/sect7"}, "another sect7"},
- {page.KindPage, nil, []string{"sect3/subsect/deep.md"}, "deep page"},
- {page.KindPage, nil, []string{filepath.FromSlash("sect5/page3.md")}, "Title5_3"}, //test OS-specific path
-
- // shorthand refs (potentially ambiguous)
- {page.KindPage, nil, []string{"unique.md"}, "UniqueBase"},
- {"Ambiguous", nil, []string{"page1.md"}, ""},
+ {"Root relative, no slash, home", page.KindHome, nil, []string{""}, "home page"},
+ {"Root relative, no slash, root page", page.KindPage, nil, []string{"about.md", "ABOUT.md"}, "about page"},
+ {"Root relative, no slash, section", page.KindSection, nil, []string{"sect3"}, "section 3"},
+ {"Root relative, no slash, section page", page.KindPage, nil, []string{"sect3/page1.md"}, "Title3_1"},
+ {"Root relative, no slash, sub setion", page.KindSection, nil, []string{"sect3/sect7"}, "another sect7"},
+ {"Root relative, no slash, nested page", page.KindPage, nil, []string{"sect3/subsect/deep.md"}, "deep page"},
+ {"Root relative, no slash, OS slashes", page.KindPage, nil, []string{filepath.FromSlash("sect5/page3.md")}, "Title5_3"},
+
+ {"Short ref, unique", page.KindPage, nil, []string{"unique.md", "unique"}, "UniqueBase"},
+ {"Short ref, unique, upper case", page.KindPage, nil, []string{"Unique2.md", "unique2.md", "unique2"}, "UniqueBase2"},
+ {"Short ref, ambiguous", "Ambiguous", nil, []string{"page1.md"}, ""},
// ISSUE: This is an ambiguous ref, but because we have to support the legacy
// content root relative paths without a leading slash, the lookup
// returns /sect7. This undermines ambiguity detection, but we have no choice.
//{"Ambiguous", nil, []string{"sect7"}, ""},
- {page.KindSection, nil, []string{"sect7"}, "Sect7s"},
-
- // absolute paths
- {page.KindHome, nil, []string{"/"}, "home page"},
- {page.KindPage, nil, []string{"/about.md"}, "about page"},
- {page.KindSection, nil, []string{"/sect3"}, "section 3"},
- {page.KindPage, nil, []string{"/sect3/page1.md"}, "Title3_1"},
- {page.KindPage, nil, []string{"/sect4/page2.md"}, "Title4_2"},
- {page.KindSection, nil, []string{"/sect3/sect7"}, "another sect7"},
- {page.KindPage, nil, []string{"/sect3/subsect/deep.md"}, "deep page"},
- {page.KindPage, nil, []string{filepath.FromSlash("/sect5/page3.md")}, "Title5_3"}, //test OS-specific path
- {page.KindPage, nil, []string{"/sect3/unique.md"}, "UniqueBase"}, //next test depends on this page existing
+ {"Section, ambigous", page.KindSection, nil, []string{"sect7"}, "Sect7s"},
+
+ {"Absolute, home", page.KindHome, nil, []string{"/", ""}, "home page"},
+ {"Absolute, page", page.KindPage, nil, []string{"/about.md", "/about"}, "about page"},
+ {"Absolute, sect", page.KindSection, nil, []string{"/sect3"}, "section 3"},
+ {"Absolute, page in subsection", page.KindPage, nil, []string{"/sect3/page1.md", "/Sect3/Page1.md"}, "Title3_1"},
+ {"Absolute, section, subsection with same name", page.KindSection, nil, []string{"/sect3/sect7"}, "another sect7"},
+ {"Absolute, page, deep", page.KindPage, nil, []string{"/sect3/subsect/deep.md"}, "deep page"},
+ {"Absolute, page, OS slashes", page.KindPage, nil, []string{filepath.FromSlash("/sect5/page3.md")}, "Title5_3"}, //test OS-specific path
+ {"Absolute, unique", page.KindPage, nil, []string{"/sect3/unique.md"}, "UniqueBase"},
+ {"Absolute, unique, case", page.KindPage, nil, []string{"/sect3/Unique2.md", "/sect3/unique2.md", "/sect3/unique2", "/sect3/Unique2"}, "UniqueBase2"},
+ //next test depends on this page existing
// {"NoPage", nil, []string{"/unique.md"}, ""}, // ISSUE #4969: this is resolving to /sect3/unique.md
- {"NoPage", nil, []string{"/missing-page.md"}, ""},
- {"NoPage", nil, []string{"/missing-section"}, ""},
+ {"Absolute, missing page", "NoPage", nil, []string{"/missing-page.md"}, ""},
+ {"Absolute, missing section", "NoPage", nil, []string{"/missing-section"}, ""},
// relative paths
- {page.KindHome, sec3, []string{".."}, "home page"},
- {page.KindHome, sec3, []string{"../"}, "home page"},
- {page.KindPage, sec3, []string{"../about.md"}, "about page"},
- {page.KindSection, sec3, []string{"."}, "section 3"},
- {page.KindSection, sec3, []string{"./"}, "section 3"},
- {page.KindPage, sec3, []string{"page1.md"}, "Title3_1"},
- {page.KindPage, sec3, []string{"./page1.md"}, "Title3_1"},
- {page.KindPage, sec3, []string{"../sect4/page2.md"}, "Title4_2"},
- {page.KindSection, sec3, []string{"sect7"}, "another sect7"},
- {page.KindSection, sec3, []string{"./sect7"}, "another sect7"},
- {page.KindPage, sec3, []string{"./subsect/deep.md"}, "deep page"},
- {page.KindPage, sec3, []string{"./subsect/../../sect7/page9.md"}, "Title7_9"},
- {page.KindPage, sec3, []string{filepath.FromSlash("../sect5/page3.md")}, "Title5_3"}, //test OS-specific path
- {page.KindPage, sec3, []string{"./unique.md"}, "UniqueBase"},
- {"NoPage", sec3, []string{"./sect2"}, ""},
+ {"Dot relative, home", page.KindHome, sec3, []string{".."}, "home page"},
+ {"Dot relative, home, slash", page.KindHome, sec3, []string{"../"}, "home page"},
+ {"Dot relative about", page.KindPage, sec3, []string{"../about.md"}, "about page"},
+ {"Dot", page.KindSection, sec3, []string{"."}, "section 3"},
+ {"Dot slash", page.KindSection, sec3, []string{"./"}, "section 3"},
+ {"Page relative, no dot", page.KindPage, sec3, []string{"page1.md"}, "Title3_1"},
+ {"Page relative, dot", page.KindPage, sec3, []string{"./page1.md"}, "Title3_1"},
+ {"Up and down another section", page.KindPage, sec3, []string{"../sect4/page2.md"}, "Title4_2"},
+ {"Rel sect7", page.KindSection, sec3, []string{"sect7"}, "another sect7"},
+ {"Rel sect7 dot", page.KindSection, sec3, []string{"./sect7"}, "another sect7"},
+ {"Dot deep", page.KindPage, sec3, []string{"./subsect/deep.md"}, "deep page"},
+ {"Dot dot inner", page.KindPage, sec3, []string{"./subsect/../../sect7/page9.md"}, "Title7_9"},
+ {"Dot OS slash", page.KindPage, sec3, []string{filepath.FromSlash("../sect5/page3.md")}, "Title5_3"}, //test OS-specific path
+ {"Dot unique", page.KindPage, sec3, []string{"./unique.md"}, "UniqueBase"},
+ {"Dot sect", "NoPage", sec3, []string{"./sect2"}, ""},
//{"NoPage", sec3, []string{"sect2"}, ""}, // ISSUE: /sect3 page relative query is resolving to /sect2
- // absolute paths ignore context
- {page.KindHome, sec3, []string{"/"}, "home page"},
- {page.KindPage, sec3, []string{"/about.md"}, "about page"},
- {page.KindPage, sec3, []string{"/sect4/page2.md"}, "Title4_2"},
- {page.KindPage, sec3, []string{"/sect3/subsect/deep.md"}, "deep page"}, //next test depends on this page existing
- {"NoPage", sec3, []string{"/subsect/deep.md"}, ""},
+ {"Abs, ignore context, home", page.KindHome, sec3, []string{"/"}, "home page"},
+ {"Abs, ignore context, about", page.KindPage, sec3, []string{"/about.md"}, "about page"},
+ {"Abs, ignore context, page in section", page.KindPage, sec3, []string{"/sect4/page2.md"}, "Title4_2"},
+ {"Abs, ignore context, page subsect deep", page.KindPage, sec3, []string{"/sect3/subsect/deep.md"}, "deep page"}, //next test depends on this page existing
+ {"Abs, ignore context, page deep", "NoPage", sec3, []string{"/subsect/deep.md"}, ""},
+
+ // Taxonomies
+ {"Taxonomy term", page.KindTaxonomyTerm, nil, []string{"categories"}, "Categories"},
+ {"Taxonomy", page.KindTaxonomy, nil, []string{"categories/hugo", "categories/Hugo"}, "Hugo"},
+
+ // Bundle variants
+ {"Bundle regular", page.KindPage, nil, []string{"sect3/b1", "sect3/b1/index.md", "sect3/b1/index.en.md"}, "b1 bundle"},
+ {"Bundle index name", page.KindPage, nil, []string{"sect3/index/index.md", "sect3/index"}, "index bundle"},
}
for _, test := range tests {
- errorMsg := fmt.Sprintf("Test case %s %v -> %s", test.context, test.path, test.expectedTitle)
+ c.Run(test.name, func(c *qt.C) {
+ errorMsg := fmt.Sprintf("Test case %v %v -> %s", test.context, test.pathVariants, test.expectedTitle)
+
+ // test legacy public Site.GetPage (which does not support page context relative queries)
+ if test.context == nil {
+ for _, ref := range test.pathVariants {
+ args := append([]string{test.kind}, ref)
+ page, err := s.Info.GetPage(args...)
+ test.check(page, err, errorMsg, c)
+ }
+ }
+
+ // test new internal Site.getPageNew
+ for _, ref := range test.pathVariants {
+ page2, err := s.getPageNew(test.context, ref)
+ test.check(page2, err, errorMsg, c)
+ }
+
+ })
+ }
- // test legacy public Site.GetPage (which does not support page context relative queries)
- if test.context == nil {
- args := append([]string{test.kind}, test.path...)
- page, err := s.Info.GetPage(args...)
- test.check(page, err, errorMsg, c)
- }
+}
- // test new internal Site.getPageNew
- var ref string
- if len(test.path) == 1 {
- ref = filepath.ToSlash(test.path[0])
- } else {
- ref = path.Join(test.path...)
- }
- page2, err := s.getPageNew(test.context, ref)
- test.check(page2, err, errorMsg, c)
+// https://github.com/gohugoio/hugo/issues/6034
+func TestGetPageRelative(t *testing.T) {
+ b := newTestSitesBuilder(t)
+ for i, section := range []string{"what", "where", "who"} {
+ isDraft := i == 2
+ b.WithContent(
+ section+"/_index.md", fmt.Sprintf("---title: %s\n---", section),
+ section+"/members.md", fmt.Sprintf("---title: members %s\ndraft: %t\n---", section, isDraft),
+ )
}
+ b.WithTemplates("_default/list.html", `
+{{ with .GetPage "members.md" }}
+ Members: {{ .Title }}
+{{ else }}
+NOT FOUND
+{{ end }}
+`)
+
+ b.Build(BuildCfg{})
+
+ b.AssertFileContent("public/what/index.html", `Members: members what`)
+ b.AssertFileContent("public/where/index.html", `Members: members where`)
+ b.AssertFileContent("public/who/index.html", `NOT FOUND`)
+
}
"os"
pth "path"
"path/filepath"
- "strings"
+ "reflect"
- "github.com/gohugoio/hugo/config"
+ "github.com/gohugoio/hugo/common/maps"
- "github.com/gohugoio/hugo/hugofs/files"
-
- "github.com/gohugoio/hugo/resources"
-
- "github.com/pkg/errors"
- "golang.org/x/sync/errgroup"
-
- "github.com/gohugoio/hugo/common/hugio"
+ "github.com/gohugoio/hugo/parser/pageparser"
- "github.com/gohugoio/hugo/resources/resource"
+ "github.com/gohugoio/hugo/hugofs/files"
"github.com/gohugoio/hugo/source"
"github.com/spf13/afero"
)
+const (
+ walkIsRootFileMetaKey = "walkIsRootFileMetaKey"
+)
+
func newPagesCollector(
sp *source.SourceSpec,
+ contentMap *pageMaps,
logger *loggers.Logger,
contentTracker *contentChangeMap,
proc pagesCollectorProcessorProvider, filenames ...string) *pagesCollector {
return &pagesCollector{
- fs: sp.SourceFs,
- proc: proc,
- sp: sp,
- logger: logger,
- filenames: filenames,
- tracker: contentTracker,
+ fs: sp.SourceFs,
+ contentMap: contentMap,
+ proc: proc,
+ sp: sp,
+ logger: logger,
+ filenames: filenames,
+ tracker: contentTracker,
}
}
-func newPagesProcessor(h *HugoSites, sp *source.SourceSpec, partialBuild bool) *pagesProcessor {
-
- return &pagesProcessor{
- h: h,
- sp: sp,
- partialBuild: partialBuild,
- numWorkers: config.GetNumWorkerMultiplier() * 3,
- }
+type contentDirKey struct {
+ dirname string
+ filename string
+ tp bundleDirType
}
type fileinfoBundle struct {
fs afero.Fs
logger *loggers.Logger
+ contentMap *pageMaps
+
// Ordered list (bundle headers first) used in partial builds.
filenames []string
proc pagesCollectorProcessorProvider
}
-type contentDirKey struct {
- dirname string
- filename string
- tp bundleDirType
+// isCascadingEdit returns whether the dir represents a cascading edit.
+// That is, if a front matter cascade section is removed, added or edited.
+// If this is the case we must re-evaluate its descendants.
+func (c *pagesCollector) isCascadingEdit(dir contentDirKey) (bool, string) {
+ // This is eiter a section or a taxonomy node. Find it.
+ prefix := cleanTreeKey(dir.dirname)
+
+ section := "/"
+ var isCascade bool
+
+ c.contentMap.walkBranchesPrefix(prefix, func(s string, n *contentNode) bool {
+ if n.fi == nil || dir.filename != n.fi.Meta().Filename() {
+ return false
+ }
+
+ f, err := n.fi.Meta().Open()
+ if err != nil {
+ // File may have been removed, assume a cascading edit.
+ // Some false positives is not too bad.
+ isCascade = true
+ return true
+ }
+
+ pf, err := pageparser.ParseFrontMatterAndContent(f)
+ f.Close()
+ if err != nil {
+ isCascade = true
+ return true
+ }
+
+ if n.p == nil || n.p.bucket == nil {
+ return true
+ }
+
+ section = s
+
+ maps.ToLower(pf.FrontMatter)
+ cascade1, ok := pf.FrontMatter["cascade"]
+ hasCascade := n.p.bucket.cascade != nil && len(n.p.bucket.cascade) > 0
+ if !ok {
+ isCascade = hasCascade
+ return true
+ }
+
+ if !hasCascade {
+ isCascade = true
+ return true
+ }
+
+ isCascade = !reflect.DeepEqual(cascade1, n.p.bucket.cascade)
+
+ return true
+
+ })
+
+ return isCascade, section
}
// Collect.
-func (c *pagesCollector) Collect() error {
+func (c *pagesCollector) Collect() (collectErr error) {
c.proc.Start(context.Background())
+ defer func() {
+ collectErr = c.proc.Wait()
+ }()
- var collectErr error
if len(c.filenames) == 0 {
// Collect everything.
collectErr = c.collectDir("", false, nil)
} else {
+ for _, pm := range c.contentMap.pmaps {
+ pm.cfg.isRebuild = true
+ }
dirs := make(map[contentDirKey]bool)
for _, filename := range c.filenames {
dir, btype := c.tracker.resolveAndRemove(filename)
}
for dir := range dirs {
+ for _, pm := range c.contentMap.pmaps {
+ pm.s.ResourceSpec.DeleteBySubstring(dir.dirname)
+ }
+
switch dir.tp {
- case bundleLeaf, bundleBranch:
+ case bundleLeaf:
collectErr = c.collectDir(dir.dirname, true, nil)
+ case bundleBranch:
+ isCascading, section := c.isCascadingEdit(dir)
+ if isCascading {
+ c.contentMap.deleteSection(section)
+ }
+ collectErr = c.collectDir(dir.dirname, !isCascading, nil)
default:
// We always start from a directory.
collectErr = c.collectDir(dir.dirname, true, func(fim hugofs.FileMetaInfo) bool {
}
- err := c.proc.Wait()
+ return
- if collectErr != nil {
- return collectErr
+}
+
+func (c *pagesCollector) isBundleHeader(fi hugofs.FileMetaInfo) bool {
+ class := fi.Meta().Classifier()
+ return class == files.ContentClassLeaf || class == files.ContentClassBranch
+}
+
+func (c *pagesCollector) getLang(fi hugofs.FileMetaInfo) string {
+ lang := fi.Meta().Lang()
+ if lang != "" {
+ return lang
}
- return err
+ return c.sp.DefaultContentLanguage
+}
+
+func (c *pagesCollector) addToBundle(info hugofs.FileMetaInfo, btyp bundleDirType, bundles pageBundles) error {
+ getBundle := func(lang string) *fileinfoBundle {
+ return bundles[lang]
+ }
+
+ cloneBundle := func(lang string) *fileinfoBundle {
+ // Every bundled content file needs a content file header.
+ // Use the default content language if found, else just
+ // pick one.
+ var (
+ source *fileinfoBundle
+ found bool
+ )
+
+ source, found = bundles[c.sp.DefaultContentLanguage]
+ if !found {
+ for _, b := range bundles {
+ source = b
+ break
+ }
+ }
+
+ if source == nil {
+ panic(fmt.Sprintf("no source found, %d", len(bundles)))
+ }
+
+ clone := c.cloneFileInfo(source.header)
+ clone.Meta()["lang"] = lang
+
+ return &fileinfoBundle{
+ header: clone,
+ }
+ }
+
+ lang := c.getLang(info)
+ bundle := getBundle(lang)
+ isBundleHeader := c.isBundleHeader(info)
+ if bundle != nil && isBundleHeader {
+ // index.md file inside a bundle, see issue 6208.
+ info.Meta()["classifier"] = files.ContentClassContent
+ isBundleHeader = false
+ }
+ classifier := info.Meta().Classifier()
+ isContent := classifier == files.ContentClassContent
+ if bundle == nil {
+ if isBundleHeader {
+ bundle = &fileinfoBundle{header: info}
+ bundles[lang] = bundle
+ } else {
+ if btyp == bundleBranch {
+ // No special logic for branch bundles.
+ // Every language needs its own _index.md file.
+ // Also, we only clone bundle headers for lonsesome, bundled,
+ // content files.
+ return c.handleFiles(info)
+ }
+
+ if isContent {
+ bundle = cloneBundle(lang)
+ bundles[lang] = bundle
+ }
+ }
+ }
+
+ if !isBundleHeader && bundle != nil {
+ bundle.resources = append(bundle.resources, info)
+ }
+
+ if classifier == files.ContentClassFile {
+ translations := info.Meta().Translations()
+
+ for lang, b := range bundles {
+ if !stringSliceContains(lang, translations...) && !b.containsResource(info.Name()) {
+
+ // Clone and add it to the bundle.
+ clone := c.cloneFileInfo(info)
+ clone.Meta()["lang"] = lang
+ b.resources = append(b.resources, clone)
+ }
+ }
+ }
+
+ return nil
+}
+
+func (c *pagesCollector) cloneFileInfo(fi hugofs.FileMetaInfo) hugofs.FileMetaInfo {
+ cm := hugofs.FileMeta{}
+ meta := fi.Meta()
+ if meta == nil {
+ panic(fmt.Sprintf("not meta: %v", fi.Name()))
+ }
+ for k, v := range meta {
+ cm[k] = v
+ }
+
+ return hugofs.NewFileMetaInfo(fi, cm)
}
func (c *pagesCollector) collectDir(dirname string, partial bool, inFilter func(fim hugofs.FileMetaInfo) bool) error {
}
}
}
+ walkRoot := dir.Meta().GetBool(walkIsRootFileMetaKey)
readdir = filtered
// We merge language directories, so there can be duplicates, but they
}
meta := fi.Meta()
+ if walkRoot {
+ meta[walkIsRootFileMetaKey] = true
+ }
class := meta.Classifier()
translationBase := meta.TranslationBaseNameWithExt()
key := pth.Join(meta.Lang(), translationBase)
return nil
}
+ fim := fi.(hugofs.FileMetaInfo)
+ // Make sure the pages in this directory gets re-rendered,
+ // even in fast render mode.
+ fim.Meta()[walkIsRootFileMetaKey] = true
+
w := hugofs.NewWalkway(hugofs.WalkwayConfig{
Fs: c.fs,
Logger: c.logger,
Root: dirname,
- Info: fi.(hugofs.FileMetaInfo),
+ Info: fim,
HookPre: preHook,
HookPost: postHook,
WalkFn: wfn})
}
-func (c *pagesCollector) isBundleHeader(fi hugofs.FileMetaInfo) bool {
- class := fi.Meta().Classifier()
- return class == files.ContentClassLeaf || class == files.ContentClassBranch
-}
-
-func (c *pagesCollector) getLang(fi hugofs.FileMetaInfo) string {
- lang := fi.Meta().Lang()
- if lang != "" {
- return lang
- }
-
- return c.sp.DefaultContentLanguage
-}
-
-func (c *pagesCollector) addToBundle(info hugofs.FileMetaInfo, btyp bundleDirType, bundles pageBundles) error {
- getBundle := func(lang string) *fileinfoBundle {
- return bundles[lang]
- }
-
- cloneBundle := func(lang string) *fileinfoBundle {
- // Every bundled content file needs a content file header.
- // Use the default content language if found, else just
- // pick one.
- var (
- source *fileinfoBundle
- found bool
- )
-
- source, found = bundles[c.sp.DefaultContentLanguage]
- if !found {
- for _, b := range bundles {
- source = b
- break
- }
- }
-
- if source == nil {
- panic(fmt.Sprintf("no source found, %d", len(bundles)))
- }
-
- clone := c.cloneFileInfo(source.header)
- clone.Meta()["lang"] = lang
-
- return &fileinfoBundle{
- header: clone,
- }
- }
-
- lang := c.getLang(info)
- bundle := getBundle(lang)
- isBundleHeader := c.isBundleHeader(info)
- if bundle != nil && isBundleHeader {
- // index.md file inside a bundle, see issue 6208.
- info.Meta()["classifier"] = files.ContentClassContent
- isBundleHeader = false
- }
- classifier := info.Meta().Classifier()
- isContent := classifier == files.ContentClassContent
- if bundle == nil {
- if isBundleHeader {
- bundle = &fileinfoBundle{header: info}
- bundles[lang] = bundle
- } else {
- if btyp == bundleBranch {
- // No special logic for branch bundles.
- // Every language needs its own _index.md file.
- // Also, we only clone bundle headers for lonsesome, bundled,
- // content files.
- return c.handleFiles(info)
- }
-
- if isContent {
- bundle = cloneBundle(lang)
- bundles[lang] = bundle
- }
- }
- }
-
- if !isBundleHeader && bundle != nil {
- bundle.resources = append(bundle.resources, info)
- }
-
- if classifier == files.ContentClassFile {
- translations := info.Meta().Translations()
-
- for lang, b := range bundles {
- if !stringSliceContains(lang, translations...) && !b.containsResource(info.Name()) {
-
- // Clone and add it to the bundle.
- clone := c.cloneFileInfo(info)
- clone.Meta()["lang"] = lang
- b.resources = append(b.resources, clone)
- }
- }
- }
-
- return nil
-}
-
-func (c *pagesCollector) cloneFileInfo(fi hugofs.FileMetaInfo) hugofs.FileMetaInfo {
- cm := hugofs.FileMeta{}
- meta := fi.Meta()
- if meta == nil {
- panic(fmt.Sprintf("not meta: %v", fi.Name()))
- }
- for k, v := range meta {
- cm[k] = v
- }
-
- return hugofs.NewFileMetaInfo(fi, cm)
-}
-
func (c *pagesCollector) handleBundleBranch(readdir []hugofs.FileMetaInfo) error {
// Maps bundles to its language.
bundles := pageBundles{}
+ var contentFiles []hugofs.FileMetaInfo
+
for _, fim := range readdir {
if fim.IsDir() {
switch meta.Classifier() {
case files.ContentClassContent:
- if err := c.handleFiles(fim); err != nil {
- return err
- }
+ contentFiles = append(contentFiles, fim)
default:
if err := c.addToBundle(fim, bundleBranch, bundles); err != nil {
return err
}
- return c.proc.Process(bundles)
+ // Make sure the section is created before its pages.
+ if err := c.proc.Process(bundles); err != nil {
+ return err
+ }
+
+ return c.handleFiles(contentFiles...)
}
return nil
}
-type pagesCollectorProcessorProvider interface {
- Process(item interface{}) error
- Start(ctx context.Context) context.Context
- Wait() error
-}
-
-type pagesProcessor struct {
- h *HugoSites
- sp *source.SourceSpec
-
- itemChan chan interface{}
- itemGroup *errgroup.Group
-
- // The output Pages
- pagesChan chan *pageState
- pagesGroup *errgroup.Group
-
- numWorkers int
-
- partialBuild bool
-}
-
-func (proc *pagesProcessor) Process(item interface{}) error {
- proc.itemChan <- item
- return nil
-}
-
-func (proc *pagesProcessor) Start(ctx context.Context) context.Context {
- proc.pagesChan = make(chan *pageState, proc.numWorkers)
- proc.pagesGroup, ctx = errgroup.WithContext(ctx)
- proc.itemChan = make(chan interface{}, proc.numWorkers)
- proc.itemGroup, ctx = errgroup.WithContext(ctx)
-
- proc.pagesGroup.Go(func() error {
- for p := range proc.pagesChan {
- s := p.s
- p.forceRender = proc.partialBuild
-
- if p.forceRender {
- s.replacePage(p)
- } else {
- s.addPage(p)
- }
- }
- return nil
- })
-
- for i := 0; i < proc.numWorkers; i++ {
- proc.itemGroup.Go(func() error {
- for item := range proc.itemChan {
- select {
- case <-proc.h.Done():
- return nil
- default:
- if err := proc.process(item); err != nil {
- proc.h.SendError(err)
- }
- }
- }
-
- return nil
- })
- }
-
- return ctx
-}
-
-func (proc *pagesProcessor) Wait() error {
- close(proc.itemChan)
-
- err := proc.itemGroup.Wait()
-
- close(proc.pagesChan)
-
- if err != nil {
- return err
- }
-
- return proc.pagesGroup.Wait()
-}
-
-func (proc *pagesProcessor) newPageFromBundle(b *fileinfoBundle) (*pageState, error) {
- p, err := proc.newPageFromFi(b.header, nil)
- if err != nil {
- return nil, err
- }
-
- if len(b.resources) > 0 {
-
- resources := make(resource.Resources, len(b.resources))
-
- for i, rfi := range b.resources {
- meta := rfi.Meta()
- classifier := meta.Classifier()
- var r resource.Resource
- switch classifier {
- case files.ContentClassContent:
- rp, err := proc.newPageFromFi(rfi, p)
- if err != nil {
- return nil, err
- }
- rp.m.resourcePath = filepath.ToSlash(strings.TrimPrefix(rp.Path(), p.File().Dir()))
-
- r = rp
-
- case files.ContentClassFile:
- r, err = proc.newResource(rfi, p)
- if err != nil {
- return nil, err
- }
- default:
- panic(fmt.Sprintf("invalid classifier: %q", classifier))
- }
-
- resources[i] = r
-
- }
-
- p.addResources(resources...)
- }
-
- return p, nil
-}
-
-func (proc *pagesProcessor) newPageFromFi(fim hugofs.FileMetaInfo, owner *pageState) (*pageState, error) {
- fi, err := newFileInfo(proc.sp, fim)
- if err != nil {
- return nil, err
- }
-
- var s *Site
- meta := fim.Meta()
-
- if owner != nil {
- s = owner.s
- } else {
- lang := meta.Lang()
- s = proc.getSite(lang)
- }
-
- r := func() (hugio.ReadSeekCloser, error) {
- return meta.Open()
- }
-
- p, err := newPageWithContent(fi, s, owner != nil, r)
- if err != nil {
- return nil, err
- }
- p.parent = owner
- return p, nil
-}
-
-func (proc *pagesProcessor) newResource(fim hugofs.FileMetaInfo, owner *pageState) (resource.Resource, error) {
-
- // TODO(bep) consolidate with multihost logic + clean up
- outputFormats := owner.m.outputFormats()
- seen := make(map[string]bool)
- var targetBasePaths []string
- // Make sure bundled resources are published to all of the ouptput formats'
- // sub paths.
- for _, f := range outputFormats {
- p := f.Path
- if seen[p] {
- continue
- }
- seen[p] = true
- targetBasePaths = append(targetBasePaths, p)
-
- }
-
- meta := fim.Meta()
- r := func() (hugio.ReadSeekCloser, error) {
- return meta.Open()
- }
-
- target := strings.TrimPrefix(meta.Path(), owner.File().Dir())
-
- return owner.s.ResourceSpec.New(
- resources.ResourceSourceDescriptor{
- TargetPaths: owner.getTargetPaths,
- OpenReadSeekCloser: r,
- FileInfo: fim,
- RelTargetFilename: target,
- TargetBasePaths: targetBasePaths,
- })
-}
-
-func (proc *pagesProcessor) getSite(lang string) *Site {
- if lang == "" {
- return proc.h.Sites[0]
- }
-
- for _, s := range proc.h.Sites {
- if lang == s.Lang() {
- return s
- }
- }
- return proc.h.Sites[0]
-}
-
-func (proc *pagesProcessor) copyFile(fim hugofs.FileMetaInfo) error {
- meta := fim.Meta()
- s := proc.getSite(meta.Lang())
- f, err := meta.Open()
- if err != nil {
- return errors.Wrap(err, "copyFile: failed to open")
- }
-
- target := filepath.Join(s.PathSpec.GetTargetLanguageBasePath(), meta.Path())
-
- defer f.Close()
-
- return s.publish(&s.PathSpec.ProcessingStats.Files, target, f)
-
-}
-
-func (proc *pagesProcessor) process(item interface{}) error {
- send := func(p *pageState, err error) {
- if err != nil {
- proc.sendError(err)
- } else {
- proc.pagesChan <- p
- }
- }
-
- switch v := item.(type) {
- // Page bundles mapped to their language.
- case pageBundles:
- for _, bundle := range v {
- if proc.shouldSkip(bundle.header) {
- continue
- }
- send(proc.newPageFromBundle(bundle))
- }
- case hugofs.FileMetaInfo:
- if proc.shouldSkip(v) {
- return nil
- }
- meta := v.Meta()
-
- classifier := meta.Classifier()
- switch classifier {
- case files.ContentClassContent:
- send(proc.newPageFromFi(v, nil))
- case files.ContentClassFile:
- proc.sendError(proc.copyFile(v))
- default:
- panic(fmt.Sprintf("invalid classifier: %q", classifier))
- }
- default:
- panic(fmt.Sprintf("unrecognized item type in Process: %T", item))
- }
-
- return nil
-}
-
-func (proc *pagesProcessor) sendError(err error) {
- if err == nil {
- return
- }
- proc.h.SendError(err)
-}
-
-func (proc *pagesProcessor) shouldSkip(fim hugofs.FileMetaInfo) bool {
- return proc.sp.DisabledLanguages[fim.Meta().Lang()]
-}
-
func stringSliceContains(k string, values ...string) bool {
for _, v := range values {
if k == v {
"path/filepath"
"testing"
- "github.com/pkg/errors"
-
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/source"
t.Run("Collect", func(t *testing.T) {
c := qt.New(t)
proc := &testPagesCollectorProcessor{}
- coll := newPagesCollector(sourceSpec, loggers.NewErrorLogger(), nil, proc)
+ coll := newPagesCollector(sourceSpec, nil, loggers.NewErrorLogger(), nil, proc)
c.Assert(coll.Collect(), qt.IsNil)
c.Assert(len(proc.items), qt.Equals, 4)
})
- t.Run("error in Wait", func(t *testing.T) {
- c := qt.New(t)
- coll := newPagesCollector(sourceSpec, loggers.NewErrorLogger(), nil,
- &testPagesCollectorProcessor{waitErr: errors.New("failed")})
- c.Assert(coll.Collect(), qt.Not(qt.IsNil))
- })
}
type testPagesCollectorProcessor struct {
+++ /dev/null
-// Copyright 2019 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hugolib
-
-import (
- "fmt"
- "path"
- "path/filepath"
- "strings"
- "sync"
-
- "github.com/gohugoio/hugo/common/maps"
-
- radix "github.com/armon/go-radix"
- "github.com/spf13/cast"
-
- "github.com/gohugoio/hugo/resources/page"
-)
-
-func newPagesMap(s *Site) *pagesMap {
- return &pagesMap{
- r: radix.New(),
- s: s,
- }
-}
-
-type pagesMap struct {
- r *radix.Tree
- s *Site
-}
-
-func (m *pagesMap) Get(key string) *pagesMapBucket {
- key = m.cleanKey(key)
- v, found := m.r.Get(key)
- if !found {
- return nil
- }
-
- return v.(*pagesMapBucket)
-}
-
-func (m *pagesMap) getKey(p *pageState) string {
- if !p.File().IsZero() {
- return m.cleanKey(p.File().Dir())
- }
- return m.cleanKey(p.SectionsPath())
-}
-
-func (m *pagesMap) getOrCreateHome() *pageState {
- var home *pageState
- b, found := m.r.Get("/")
- if !found {
- home = m.s.newPage(page.KindHome)
- m.addBucketFor("/", home, nil)
- } else {
- home = b.(*pagesMapBucket).owner
- }
-
- return home
-}
-
-func (m *pagesMap) initPageMeta(p *pageState, bucket *pagesMapBucket) error {
- var err error
- p.metaInit.Do(func() {
- if p.metaInitFn != nil {
- err = p.metaInitFn(bucket)
- }
- })
- return err
-}
-
-func (m *pagesMap) initPageMetaFor(prefix string, bucket *pagesMapBucket) error {
- parentBucket := m.parentBucket(prefix)
-
- m.mergeCascades(bucket, parentBucket)
-
- if err := m.initPageMeta(bucket.owner, bucket); err != nil {
- return err
- }
-
- if !bucket.view {
- for _, p := range bucket.pages {
- ps := p.(*pageState)
- if err := m.initPageMeta(ps, bucket); err != nil {
- return err
- }
-
- for _, p := range ps.resources.ByType(pageResourceType) {
- if err := m.initPageMeta(p.(*pageState), bucket); err != nil {
- return err
- }
- }
- }
-
- // Now that the metadata is initialized (with dates, draft set etc.)
- // we can remove the pages that we for some reason should not include
- // in this build.
- tmp := bucket.pages[:0]
- for _, x := range bucket.pages {
- if m.s.shouldBuild(x) {
- if x.(*pageState).m.headless {
- bucket.headlessPages = append(bucket.headlessPages, x)
- } else {
- tmp = append(tmp, x)
- }
-
- }
- }
- bucket.pages = tmp
- }
-
- return nil
-}
-
-func (m *pagesMap) createSectionIfNotExists(section string) {
- key := m.cleanKey(section)
- _, found := m.r.Get(key)
- if !found {
- kind := m.s.kindFromSectionPath(section)
- p := m.s.newPage(kind, section)
- m.addBucketFor(key, p, nil)
- }
-}
-
-func (m *pagesMap) addBucket(p *pageState) {
- key := m.getKey(p)
-
- m.addBucketFor(key, p, nil)
-}
-
-func (m *pagesMap) addBucketFor(key string, p *pageState, meta map[string]interface{}) *pagesMapBucket {
- var isView bool
- switch p.Kind() {
- case page.KindTaxonomy, page.KindTaxonomyTerm:
- isView = true
- }
-
- disabled := !m.s.isEnabled(p.Kind())
-
- var cascade map[string]interface{}
- if p.bucket != nil {
- cascade = p.bucket.cascade
- }
-
- bucket := &pagesMapBucket{
- owner: p,
- view: isView,
- cascade: cascade,
- meta: meta,
- disabled: disabled,
- }
-
- p.bucket = bucket
-
- m.r.Insert(key, bucket)
-
- return bucket
-}
-
-func (m *pagesMap) addPage(p *pageState) {
- if !p.IsPage() {
- m.addBucket(p)
- return
- }
-
- if !m.s.isEnabled(page.KindPage) {
- return
- }
-
- key := m.getKey(p)
-
- var bucket *pagesMapBucket
-
- _, v, found := m.r.LongestPrefix(key)
- if !found {
- panic(fmt.Sprintf("[BUG] bucket with key %q not found", key))
- }
-
- bucket = v.(*pagesMapBucket)
- bucket.pages = append(bucket.pages, p)
-}
-
-func (m *pagesMap) assemblePageMeta() error {
- var walkErr error
- m.r.Walk(func(s string, v interface{}) bool {
- bucket := v.(*pagesMapBucket)
-
- if err := m.initPageMetaFor(s, bucket); err != nil {
- walkErr = err
- return true
- }
- return false
- })
-
- return walkErr
-}
-
-func (m *pagesMap) assembleTaxonomies(s *Site) error {
- s.Taxonomies = make(TaxonomyList)
-
- type bucketKey struct {
- plural string
- termKey string
- }
-
- // Temporary cache.
- taxonomyBuckets := make(map[bucketKey]*pagesMapBucket)
-
- for singular, plural := range s.siteCfg.taxonomiesConfig {
- s.Taxonomies[plural] = make(Taxonomy)
- bkey := bucketKey{
- plural: plural,
- }
-
- bucket := m.Get(plural)
-
- if bucket == nil {
- // Create the page and bucket
- n := s.newPage(page.KindTaxonomyTerm, plural)
-
- key := m.cleanKey(plural)
- bucket = m.addBucketFor(key, n, nil)
- if err := m.initPageMetaFor(key, bucket); err != nil {
- return err
- }
- }
-
- if bucket.meta == nil {
- bucket.meta = map[string]interface{}{
- "singular": singular,
- "plural": plural,
- }
- }
-
- // Add it to the temporary cache.
- taxonomyBuckets[bkey] = bucket
-
- // Taxonomy entries used in page front matter will be picked up later,
- // but there may be some yet to be used.
- pluralPrefix := m.cleanKey(plural) + "/"
- m.r.WalkPrefix(pluralPrefix, func(k string, v interface{}) bool {
- tb := v.(*pagesMapBucket)
- termKey := strings.TrimPrefix(k, pluralPrefix)
- if tb.meta == nil {
- tb.meta = map[string]interface{}{
- "singular": singular,
- "plural": plural,
- "term": tb.owner.Title(),
- "termKey": termKey,
- }
- }
-
- bucket.pages = append(bucket.pages, tb.owner)
- bkey.termKey = termKey
- taxonomyBuckets[bkey] = tb
-
- return false
- })
-
- }
-
- addTaxonomy := func(singular, plural, term string, weight int, p page.Page) error {
- bkey := bucketKey{
- plural: plural,
- }
-
- termKey := s.getTaxonomyKey(term)
-
- b1 := taxonomyBuckets[bkey]
-
- var b2 *pagesMapBucket
- bkey.termKey = termKey
- b, found := taxonomyBuckets[bkey]
- if found {
- b2 = b
- } else {
-
- // Create the page and bucket
- n := s.newTaxonomyPage(term, plural, termKey)
- meta := map[string]interface{}{
- "singular": singular,
- "plural": plural,
- "term": term,
- "termKey": termKey,
- }
-
- key := m.cleanKey(path.Join(plural, termKey))
- b2 = m.addBucketFor(key, n, meta)
- if err := m.initPageMetaFor(key, b2); err != nil {
- return err
- }
- b1.pages = append(b1.pages, b2.owner)
- taxonomyBuckets[bkey] = b2
-
- }
-
- w := page.NewWeightedPage(weight, p, b2.owner)
-
- s.Taxonomies[plural].add(termKey, w)
-
- b1.owner.m.Dates.UpdateDateAndLastmodIfAfter(p)
- b2.owner.m.Dates.UpdateDateAndLastmodIfAfter(p)
-
- return nil
- }
-
- m.r.Walk(func(k string, v interface{}) bool {
- b := v.(*pagesMapBucket)
- if b.view {
- return false
- }
-
- for singular, plural := range s.siteCfg.taxonomiesConfig {
- for _, p := range b.pages {
-
- vals := getParam(p, plural, false)
-
- w := getParamToLower(p, plural+"_weight")
- weight, err := cast.ToIntE(w)
- if err != nil {
- m.s.Log.ERROR.Printf("Unable to convert taxonomy weight %#v to int for %q", w, p.Path())
- // weight will equal zero, so let the flow continue
- }
-
- if vals != nil {
- if v, ok := vals.([]string); ok {
- for _, idx := range v {
- if err := addTaxonomy(singular, plural, idx, weight, p); err != nil {
- m.s.Log.ERROR.Printf("Failed to add taxonomy %q for %q: %s", plural, p.Path(), err)
- }
- }
- } else if v, ok := vals.(string); ok {
- if err := addTaxonomy(singular, plural, v, weight, p); err != nil {
- m.s.Log.ERROR.Printf("Failed to add taxonomy %q for %q: %s", plural, p.Path(), err)
- }
- } else {
- m.s.Log.ERROR.Printf("Invalid %s in %q\n", plural, p.Path())
- }
- }
-
- }
- }
- return false
- })
-
- for _, plural := range s.siteCfg.taxonomiesConfig {
- for k := range s.Taxonomies[plural] {
- s.Taxonomies[plural][k].Sort()
- }
- }
-
- return nil
-}
-
-func (m *pagesMap) cleanKey(key string) string {
- key = filepath.ToSlash(strings.ToLower(key))
- key = strings.Trim(key, "/")
- return "/" + key
-}
-
-func (m *pagesMap) mergeCascades(b1, b2 *pagesMapBucket) {
- if b1.cascade == nil {
- b1.cascade = make(maps.Params)
- }
- if b2 != nil && b2.cascade != nil {
- for k, v := range b2.cascade {
- if _, found := b1.cascade[k]; !found {
- b1.cascade[k] = v
- }
- }
- }
-}
-
-func (m *pagesMap) parentBucket(prefix string) *pagesMapBucket {
- if prefix == "/" {
- return nil
- }
- _, parentv, found := m.r.LongestPrefix(path.Dir(prefix))
- if !found {
- panic(fmt.Sprintf("[BUG] parent bucket not found for %q", prefix))
- }
- return parentv.(*pagesMapBucket)
-
-}
-
-func (m *pagesMap) withEveryPage(f func(p *pageState)) {
- m.r.Walk(func(k string, v interface{}) bool {
- b := v.(*pagesMapBucket)
- f(b.owner)
- if !b.view {
- for _, p := range b.pages {
- f(p.(*pageState))
- }
- }
-
- return false
- })
-}
-
-type pagesMapBucket struct {
- // Set if the pages in this bucket is also present in another bucket.
- view bool
-
- // Some additional metatadata attached to this node.
- meta map[string]interface{}
-
- // Cascading front matter.
- cascade map[string]interface{}
-
- owner *pageState // The branch node
-
- // When disableKinds is enabled for this node.
- disabled bool
-
- // Used to navigate the sections tree
- parent *pagesMapBucket
- bucketSections []*pagesMapBucket
-
- pagesInit sync.Once
- pages page.Pages
- headlessPages page.Pages
-
- pagesAndSectionsInit sync.Once
- pagesAndSections page.Pages
-
- sectionsInit sync.Once
- sections page.Pages
-}
-
-func (b *pagesMapBucket) isEmpty() bool {
- return len(b.pages) == 0 && len(b.headlessPages) == 0 && len(b.bucketSections) == 0
-}
-
-func (b *pagesMapBucket) getPages() page.Pages {
- b.pagesInit.Do(func() {
- page.SortByDefault(b.pages)
- })
- return b.pages
-}
-
-func (b *pagesMapBucket) getPagesAndSections() page.Pages {
- b.pagesAndSectionsInit.Do(func() {
- var pas page.Pages
- pas = append(pas, b.getPages()...)
- for _, p := range b.bucketSections {
- pas = append(pas, p.owner)
- }
- b.pagesAndSections = pas
- page.SortByDefault(b.pagesAndSections)
- })
- return b.pagesAndSections
-}
-
-func (b *pagesMapBucket) getSections() page.Pages {
- b.sectionsInit.Do(func() {
- for _, p := range b.bucketSections {
- b.sections = append(b.sections, p.owner)
- }
- page.SortByDefault(b.sections)
- })
-
- return b.sections
-}
--- /dev/null
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+ "context"
+ "fmt"
+ "path/filepath"
+
+ "github.com/gohugoio/hugo/config"
+ "github.com/gohugoio/hugo/source"
+
+ "github.com/gohugoio/hugo/hugofs/files"
+ "github.com/pkg/errors"
+ "golang.org/x/sync/errgroup"
+
+ "github.com/gohugoio/hugo/common/herrors"
+ "github.com/gohugoio/hugo/hugofs"
+)
+
+func newPagesProcessor(h *HugoSites, sp *source.SourceSpec) *pagesProcessor {
+ procs := make(map[string]pagesCollectorProcessorProvider)
+ for _, s := range h.Sites {
+ procs[s.Lang()] = &sitePagesProcessor{
+ m: s.pageMap,
+ errorSender: s.h,
+ itemChan: make(chan interface{}, config.GetNumWorkerMultiplier()*2),
+ }
+ }
+ return &pagesProcessor{
+ procs: procs,
+ }
+}
+
+type pagesCollectorProcessorProvider interface {
+ Process(item interface{}) error
+ Start(ctx context.Context) context.Context
+ Wait() error
+}
+
+type pagesProcessor struct {
+ // Per language/Site
+ procs map[string]pagesCollectorProcessorProvider
+}
+
+func (proc *pagesProcessor) Process(item interface{}) error {
+ switch v := item.(type) {
+ // Page bundles mapped to their language.
+ case pageBundles:
+ for _, vv := range v {
+ proc.getProcFromFi(vv.header).Process(vv)
+ }
+ case hugofs.FileMetaInfo:
+ proc.getProcFromFi(v).Process(v)
+ default:
+ panic(fmt.Sprintf("unrecognized item type in Process: %T", item))
+
+ }
+
+ return nil
+}
+
+func (proc *pagesProcessor) Start(ctx context.Context) context.Context {
+ for _, p := range proc.procs {
+ ctx = p.Start(ctx)
+ }
+ return ctx
+}
+
+func (proc *pagesProcessor) Wait() error {
+ var err error
+ for _, p := range proc.procs {
+ if e := p.Wait(); e != nil {
+ err = e
+ }
+ }
+ return err
+}
+
+func (proc *pagesProcessor) getProcFromFi(fi hugofs.FileMetaInfo) pagesCollectorProcessorProvider {
+ if p, found := proc.procs[fi.Meta().Lang()]; found {
+ return p
+ }
+ return defaultPageProcessor
+}
+
+type nopPageProcessor int
+
+func (nopPageProcessor) Process(item interface{}) error {
+ return nil
+}
+
+func (nopPageProcessor) Start(ctx context.Context) context.Context {
+ return context.Background()
+}
+
+func (nopPageProcessor) Wait() error {
+ return nil
+}
+
+var defaultPageProcessor = new(nopPageProcessor)
+
+type sitePagesProcessor struct {
+ m *pageMap
+ errorSender herrors.ErrorSender
+
+ itemChan chan interface{}
+ itemGroup *errgroup.Group
+}
+
+func (p *sitePagesProcessor) Process(item interface{}) error {
+ p.itemChan <- item
+ return nil
+}
+
+func (p *sitePagesProcessor) Start(ctx context.Context) context.Context {
+ p.itemGroup, ctx = errgroup.WithContext(ctx)
+ p.itemGroup.Go(func() error {
+ for item := range p.itemChan {
+ if err := p.doProcess(item); err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+ return ctx
+}
+
+func (p *sitePagesProcessor) Wait() error {
+ close(p.itemChan)
+ return p.itemGroup.Wait()
+}
+
+func (p *sitePagesProcessor) copyFile(fim hugofs.FileMetaInfo) error {
+ meta := fim.Meta()
+ f, err := meta.Open()
+ if err != nil {
+ return errors.Wrap(err, "copyFile: failed to open")
+ }
+
+ s := p.m.s
+
+ target := filepath.Join(s.PathSpec.GetTargetLanguageBasePath(), meta.Path())
+
+ defer f.Close()
+
+ return s.publish(&s.PathSpec.ProcessingStats.Files, target, f)
+
+}
+
+func (p *sitePagesProcessor) doProcess(item interface{}) error {
+ m := p.m
+ switch v := item.(type) {
+ case *fileinfoBundle:
+ if err := m.AddFilesBundle(v.header, v.resources...); err != nil {
+ return err
+ }
+ case hugofs.FileMetaInfo:
+ if p.shouldSkip(v) {
+ return nil
+ }
+ meta := v.Meta()
+
+ classifier := meta.Classifier()
+ switch classifier {
+ case files.ContentClassContent:
+ if err := m.AddFilesBundle(v); err != nil {
+ return err
+ }
+ case files.ContentClassFile:
+ if err := p.copyFile(v); err != nil {
+ return err
+ }
+ default:
+ panic(fmt.Sprintf("invalid classifier: %q", classifier))
+ }
+ default:
+ panic(fmt.Sprintf("unrecognized item type in Process: %T", item))
+ }
+ return nil
+
+}
+
+func (p *sitePagesProcessor) shouldSkip(fim hugofs.FileMetaInfo) bool {
+ // TODO(ep) unify
+ return p.m.s.SourceSpec.DisabledLanguages[fim.Meta().Lang()]
+}
*PageCollections
- Taxonomies TaxonomyList
+ taxonomies TaxonomyList
Sections Taxonomy
- Info SiteInfo
+ Info *SiteInfo
language *langs.Language
init *siteInit
}
+func (s *Site) Taxonomies() TaxonomyList {
+ s.init.taxonomies.Do()
+ return s.taxonomies
+}
+
+type taxonomiesConfig map[string]string
+
+func (t taxonomiesConfig) Values() []viewName {
+ var vals []viewName
+ for k, v := range t {
+ vals = append(vals, viewName{singular: k, plural: v})
+ }
+ sort.Slice(vals, func(i, j int) bool {
+ return vals[i].plural < vals[j].plural
+ })
+
+ return vals
+}
+
type siteConfigHolder struct {
sitemap config.Sitemap
- taxonomiesConfig map[string]string
+ taxonomiesConfig taxonomiesConfig
timeout time.Duration
hasCJKLanguage bool
enableEmoji bool
prevNext *lazy.Init
prevNextInSection *lazy.Init
menus *lazy.Init
+ taxonomies *lazy.Init
}
func (init *siteInit) Reset() {
init.prevNext.Reset()
init.prevNextInSection.Reset()
init.menus.Reset()
+ init.taxonomies.Reset()
}
func (s *Site) initInit(init *lazy.Init, pctx pageContext) bool {
var init lazy.Init
s.init.prevNext = init.Branch(func() (interface{}, error) {
- regularPages := s.findWorkPagesByKind(page.KindPage)
+ regularPages := s.RegularPages()
for i, p := range regularPages {
- if p.posNextPrev == nil {
+ np, ok := p.(nextPrevProvider)
+ if !ok {
+ continue
+ }
+
+ pos := np.getNextPrev()
+ if pos == nil {
continue
}
- p.posNextPrev.nextPage = nil
- p.posNextPrev.prevPage = nil
+
+ pos.nextPage = nil
+ pos.prevPage = nil
if i > 0 {
- p.posNextPrev.nextPage = regularPages[i-1]
+ pos.nextPage = regularPages[i-1]
}
if i < len(regularPages)-1 {
- p.posNextPrev.prevPage = regularPages[i+1]
+ pos.prevPage = regularPages[i+1]
}
}
return nil, nil
})
s.init.prevNextInSection = init.Branch(func() (interface{}, error) {
- var rootSection []int
- // TODO(bep) cm attach this to the bucket.
- for i, p1 := range s.workAllPages {
- if p1.IsPage() && p1.Section() == "" {
- rootSection = append(rootSection, i)
- }
- if p1.IsSection() {
- sectionPages := p1.RegularPages()
- for i, p2 := range sectionPages {
- p2s := p2.(*pageState)
- if p2s.posNextPrevSection == nil {
- continue
- }
- p2s.posNextPrevSection.nextPage = nil
- p2s.posNextPrevSection.prevPage = nil
+ var sections page.Pages
+ s.home.treeRef.m.collectSectionsRecursiveIncludingSelf(s.home.treeRef.key, func(n *contentNode) {
+ sections = append(sections, n.p)
+ })
- if i > 0 {
- p2s.posNextPrevSection.nextPage = sectionPages[i-1]
- }
+ setNextPrev := func(pas page.Pages) {
+ for i, p := range pas {
+ np, ok := p.(nextPrevInSectionProvider)
+ if !ok {
+ continue
+ }
- if i < len(sectionPages)-1 {
- p2s.posNextPrevSection.prevPage = sectionPages[i+1]
- }
+ pos := np.getNextPrevInSection()
+ if pos == nil {
+ continue
+ }
+
+ pos.nextPage = nil
+ pos.prevPage = nil
+
+ if i > 0 {
+ pos.nextPage = pas[i-1]
+ }
+
+ if i < len(pas)-1 {
+ pos.prevPage = pas[i+1]
}
}
}
- for i, j := range rootSection {
- p := s.workAllPages[j]
- if i > 0 {
- p.posNextPrevSection.nextPage = s.workAllPages[rootSection[i-1]]
- }
+ for _, sect := range sections {
+ treeRef := sect.(treeRefProvider).getTreeRef()
- if i < len(rootSection)-1 {
- p.posNextPrevSection.prevPage = s.workAllPages[rootSection[i+1]]
- }
+ var pas page.Pages
+ treeRef.m.collectPages(treeRef.key+cmBranchSeparator, func(c *contentNode) {
+ pas = append(pas, c.p)
+ })
+ page.SortByDefault(pas)
+
+ setNextPrev(pas)
}
+ // The root section only goes one level down.
+ treeRef := s.home.getTreeRef()
+
+ var pas page.Pages
+ treeRef.m.collectPages(treeRef.key+cmBranchSeparator, func(c *contentNode) {
+ pas = append(pas, c.p)
+ })
+ page.SortByDefault(pas)
+
+ setNextPrev(pas)
+
return nil, nil
})
return nil, nil
})
+ s.init.taxonomies = init.Branch(func() (interface{}, error) {
+ err := s.pageMap.assembleTaxonomies()
+ return nil, err
+ })
+
}
type siteRenderingContext struct {
func (s *Site) initRenderFormats() {
formatSet := make(map[string]bool)
formats := output.Formats{}
- for _, p := range s.workAllPages {
- for _, f := range p.m.configuredOutputFormats {
+ s.pageMap.pageTrees.WalkRenderable(func(s string, n *contentNode) bool {
+ for _, f := range n.p.m.configuredOutputFormats {
if !formatSet[f.Name] {
formats = append(formats, f)
formatSet[f.Name] = true
}
}
- }
+ return false
+ })
// Add the per kind configured output formats
for _, kind := range allKindsInPages {
// newSite creates a new site with the given configuration.
func newSite(cfg deps.DepsCfg) (*Site, error) {
- c := newPageCollections()
-
if cfg.Language == nil {
cfg.Language = langs.NewDefaultLanguage(cfg.Cfg)
}
return nil, err
}
+ if disabledKinds[kindRSS] {
+ // Legacy
+ tmp := siteOutputFormatsConfig[:0]
+ for _, x := range siteOutputFormatsConfig {
+ if !strings.EqualFold(x.Name, "rss") {
+ tmp = append(tmp, x)
+ }
+ }
+ siteOutputFormatsConfig = tmp
+ }
+
outputFormats, err := createSiteOutputFormats(siteOutputFormatsConfig, cfg.Language)
if err != nil {
return nil, err
}
s := &Site{
- PageCollections: c,
- language: cfg.Language,
- disabledKinds: disabledKinds,
- titleFunc: titleFunc,
- relatedDocsHandler: page.NewRelatedDocsHandler(relatedContentConfig),
- outputFormats: outputFormats,
- rc: &siteRenderingContext{output.HTMLFormat},
- outputFormatsConfig: siteOutputFormatsConfig,
- mediaTypesConfig: siteMediaTypesConfig,
- frontmatterHandler: frontMatterHandler,
+
+ language: cfg.Language,
+ disabledKinds: disabledKinds,
+
+ outputFormats: outputFormats,
+ outputFormatsConfig: siteOutputFormatsConfig,
+ mediaTypesConfig: siteMediaTypesConfig,
+
enableInlineShortcodes: cfg.Language.GetBool("enableInlineShortcodes"),
siteCfg: siteConfig,
+
+ titleFunc: titleFunc,
+
+ rc: &siteRenderingContext{output.HTMLFormat},
+
+ frontmatterHandler: frontMatterHandler,
+ relatedDocsHandler: page.NewRelatedDocsHandler(relatedContentConfig),
}
s.prepareInits()
// TODO(bep) type
func (s *SiteInfo) Taxonomies() interface{} {
- return s.s.Taxonomies
+ return s.s.Taxonomies()
}
func (s *SiteInfo) Params() maps.Params {
if refURL.Path != "" {
var err error
- target, err = s.s.getPageNew(p, refURL.Path)
+ target, err = s.s.getPageRef(p, refURL.Path)
var pos text.Position
if err != nil || target == nil {
if p, ok := source.(text.Positioner); ok {
OutputFormats: site.outputFormatsConfig,
}
site.Deps, err = first.Deps.ForLanguage(depsCfg, func(d *deps.Deps) error {
- d.Site = &site.Info
+ d.Site = site.Info
return nil
})
if err != nil {
}
}
- s.Info = SiteInfo{
+ s.Info = &SiteInfo{
title: lang.GetString("title"),
Author: lang.GetStringMap("author"),
Social: lang.GetStringMapString("social"),
func (s *Site) readAndProcessContent(filenames ...string) error {
sourceSpec := source.NewSourceSpec(s.PathSpec, s.BaseFs.Content.Fs)
- proc := newPagesProcessor(s.h, sourceSpec, len(filenames) > 0)
+ proc := newPagesProcessor(s.h, sourceSpec)
- c := newPagesCollector(sourceSpec, s.Log, s.h.ContentChanges, proc, filenames...)
+ c := newPagesCollector(sourceSpec, s.h.content, s.Log, s.h.ContentChanges, proc, filenames...)
- return c.Collect()
+ if err := c.Collect(); err != nil {
+ return err
+ }
+
+ s.h.content = newPageMaps(s.h)
+
+ return nil
}
func (s *Site) getMenusFromConfig() navigation.Menus {
sectionPagesMenu := s.Info.sectionPagesMenu
if sectionPagesMenu != "" {
- for _, p := range s.workAllPages {
- if p.Kind() == page.KindSection {
- // From Hugo 0.22 we have nested sections, but until we get a
- // feel of how that would work in this setting, let us keep
- // this menu for the top level only.
- id := p.Section()
- if _, ok := flat[twoD{sectionPagesMenu, id}]; ok {
- continue
- }
-
- me := navigation.MenuEntry{Identifier: id,
- Name: p.LinkTitle(),
- Weight: p.Weight(),
- Page: p}
- flat[twoD{sectionPagesMenu, me.KeyName()}] = &me
+ s.pageMap.sections.Walk(func(s string, v interface{}) bool {
+ p := v.(*contentNode).p
+ if p.IsHome() {
+ return false
}
- }
+ // From Hugo 0.22 we have nested sections, but until we get a
+ // feel of how that would work in this setting, let us keep
+ // this menu for the top level only.
+ id := p.Section()
+ if _, ok := flat[twoD{sectionPagesMenu, id}]; ok {
+ return false
+ }
+
+ me := navigation.MenuEntry{Identifier: id,
+ Name: p.LinkTitle(),
+ Weight: p.Weight(),
+ Page: p}
+ flat[twoD{sectionPagesMenu, me.KeyName()}] = &me
+
+ return false
+ })
+
}
// Add menu entries provided by pages
- for _, p := range s.workAllPages {
+ s.pageMap.pageTrees.WalkRenderable(func(ss string, n *contentNode) bool {
+ p := n.p
+
for name, me := range p.pageMenus.menus() {
if _, ok := flat[twoD{name, me.KeyName()}]; ok {
- s.SendError(p.wrapError(errors.Errorf("duplicate menu entry with identifier %q in menu %q", me.KeyName(), name)))
+ err := p.wrapError(errors.Errorf("duplicate menu entry with identifier %q in menu %q", me.KeyName(), name))
+ s.Log.WARN.Println(err)
continue
}
flat[twoD{name, me.KeyName()}] = me
}
- }
+
+ return false
+ })
// Create Children Menus First
for _, e := range flat {
s.init.Reset()
if sourceChanged {
- s.PageCollections = newPageCollectionsFromPages(s.rawAllPages)
- for _, p := range s.rawAllPages {
+ s.PageCollections = newPageCollections(s.pageMap)
+ s.pageMap.withEveryBundlePage(func(p *pageState) bool {
p.pagePages = &pagePages{}
p.parent = nil
p.Scratcher = maps.NewScratcher()
- }
+ return false
+ })
} else {
- s.pagesMap.withEveryPage(func(p *pageState) {
+ s.pageMap.withEveryBundlePage(func(p *pageState) bool {
p.Scratcher = maps.NewScratcher()
+ return false
})
}
}
return s.kindFromSections(sections)
}
+
return page.KindPage
}
return page.KindSection
}
-func (s *Site) newTaxonomyPage(title string, sections ...string) *pageState {
- p, err := newPageFromMeta(
- map[string]interface{}{"title": title},
- &pageMeta{
- s: s,
- kind: page.KindTaxonomy,
- sections: sections,
- })
+func (s *Site) newPage(
+ n *contentNode,
+ parentbBucket *pagesMapBucket,
+ kind, title string,
+ sections ...string) *pageState {
- if err != nil {
- panic(err)
+ m := map[string]interface{}{}
+ if title != "" {
+ m["title"] = title
}
- return p
-
-}
-
-func (s *Site) newPage(kind string, sections ...string) *pageState {
p, err := newPageFromMeta(
- map[string]interface{}{},
+ n,
+ parentbBucket,
+ m,
&pageMeta{
s: s,
kind: kind,
}
}
+func TestBenchmarkSiteDeepContentEdit(t *testing.T) {
+ b := getBenchmarkSiteDeepContent(t).Running()
+ b.Build(BuildCfg{})
+
+ p := b.H.Sites[0].RegularPages()[12]
+
+ b.EditFiles(p.File().Filename(), fmt.Sprintf(`---
+title: %s
+---
+
+Edited!!`, p.Title()))
+
+ counters := &testCounters{}
+
+ b.Build(BuildCfg{testCounters: counters})
+
+ // We currently rebuild all the language versions of the same content file.
+ // We could probably optimize that case, but it's not trivial.
+ b.Assert(int(counters.contentRenderCounter), qt.Equals, 4)
+ b.AssertFileContent("public"+p.RelPermalink()+"index.html", "Edited!!")
+
+}
+
func BenchmarkSiteNew(b *testing.B) {
rnd := rand.New(rand.NewSource(32))
benchmarks := getBenchmarkSiteNewTestCases()
)
func createDefaultOutputFormats(allFormats output.Formats, cfg config.Provider) map[string]output.Formats {
- rssOut, _ := allFormats.GetByName(output.RSSFormat.Name)
+ rssOut, rssFound := allFormats.GetByName(output.RSSFormat.Name)
htmlOut, _ := allFormats.GetByName(output.HTMLFormat.Name)
robotsOut, _ := allFormats.GetByName(output.RobotsTxtFormat.Name)
sitemapOut, _ := allFormats.GetByName(output.SitemapFormat.Name)
- return map[string]output.Formats{
+ defaultListTypes := output.Formats{htmlOut}
+ if rssFound {
+ defaultListTypes = append(defaultListTypes, rssOut)
+ }
+
+ m := map[string]output.Formats{
page.KindPage: {htmlOut},
- page.KindHome: {htmlOut, rssOut},
- page.KindSection: {htmlOut, rssOut},
- page.KindTaxonomy: {htmlOut, rssOut},
- page.KindTaxonomyTerm: {htmlOut, rssOut},
+ page.KindHome: defaultListTypes,
+ page.KindSection: defaultListTypes,
+ page.KindTaxonomy: defaultListTypes,
+ page.KindTaxonomyTerm: defaultListTypes,
// Below are for consistency. They are currently not used during rendering.
- kindRSS: {rssOut},
kindSitemap: {sitemapOut},
kindRobotsTXT: {robotsOut},
kind404: {htmlOut},
}
+ // May be disabled
+ if rssFound {
+ m[kindRSS] = output.Formats{rssOut}
+ }
+
+ return m
+
}
func createSiteOutputFormats(allFormats output.Formats, cfg config.Provider) (map[string]output.Formats, error) {
cfg := ctx.cfg
- if !cfg.PartialReRender && ctx.outIdx == 0 && len(s.headlessPages) > 0 {
- wg.Add(1)
- go headlessPagesPublisher(s, wg)
- }
-
-L:
- for _, page := range s.workAllPages {
- if cfg.shouldRender(page) {
+ s.pageMap.pageTrees.Walk(func(ss string, n *contentNode) bool {
+ if cfg.shouldRender(n.p) {
select {
case <-s.h.Done():
- break L
+ return true
default:
- pages <- page
+ pages <- n.p
}
}
- }
+ return false
+ })
close(pages)
return nil
}
-func headlessPagesPublisher(s *Site, wg *sync.WaitGroup) {
- defer wg.Done()
- for _, p := range s.headlessPages {
- if err := p.renderResources(); err != nil {
- s.SendError(p.errorf(err, "failed to render page resources"))
- }
- }
-}
-
func pageRenderer(
ctx *siteRenderContext,
s *Site,
defer wg.Done()
for p := range pages {
- f := p.outputFormat()
-
- // TODO(bep) get rid of this odd construct. RSS is an output format.
- if f.Name == "RSS" && !s.isEnabled(kindRSS) {
- continue
+ if p.m.buildConfig.PublishResources {
+ if err := p.renderResources(); err != nil {
+ s.SendError(p.errorf(err, "failed to render page resources"))
+ continue
+ }
}
- if err := p.renderResources(); err != nil {
- s.SendError(p.errorf(err, "failed to render page resources"))
+ if !p.render {
+ // Nothing more to do for this page.
continue
}
}
if !found {
- s.logMissingLayout("", p.Kind(), f.Name)
+ s.logMissingLayout("", p.Kind(), p.f.Name)
continue
}
}
func (s *Site) render404() error {
- if !s.isEnabled(kind404) {
- return nil
- }
-
p, err := newPageStandalone(&pageMeta{
s: s,
kind: kind404,
return err
}
+ if !p.render {
+ return nil
+ }
+
var d output.LayoutDescriptor
d.Kind = kind404
}
func (s *Site) renderSitemap() error {
- if !s.isEnabled(kindSitemap) {
- return nil
- }
-
p, err := newPageStandalone(&pageMeta{
s: s,
kind: kindSitemap,
return err
}
+ if !p.render {
+ return nil
+ }
+
targetPath := p.targetPaths().TargetFilename
if targetPath == "" {
}
func (s *Site) renderRobotsTXT() error {
- if !s.isEnabled(kindRobotsTXT) {
- return nil
- }
-
if !s.Cfg.GetBool("enableRobotsTXT") {
return nil
}
return err
}
+ if !p.render {
+ return nil
+ }
+
templ := s.lookupLayouts("robots.txt", "_default/robots.txt", "_internal/_default/robots.txt")
return s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, "Robots Txt", p.targetPaths().TargetFilename, p, templ)
// renderAliases renders shell pages that simply have a redirect in the header.
func (s *Site) renderAliases() error {
- for _, p := range s.workAllPages {
-
+ var err error
+ s.pageMap.pageTrees.WalkRenderable(func(ss string, n *contentNode) bool {
+ p := n.p
if len(p.Aliases()) == 0 {
- continue
+ return false
}
for _, of := range p.OutputFormats() {
if !of.Format.IsHTML {
- continue
+ return false
}
plink := of.Permalink()
a = path.Join(lang, a)
}
- if err := s.writeDestAlias(a, plink, f, p); err != nil {
- return err
+ err = s.writeDestAlias(a, plink, f, p)
+ if err != nil {
+ return true
}
}
}
- }
+ return false
+ })
- return nil
+ return err
}
// renderMainLanguageRedirect creates a redirect to the main language home,
c := qt.New(t)
sections := strings.Split(test.sections, ",")
p := s.getPage(page.KindSection, sections...)
- c.Assert(p, qt.Not(qt.IsNil))
+ c.Assert(p, qt.Not(qt.IsNil), qt.Commentf(fmt.Sprint(sections)))
if p.Pages() != nil {
c.Assert(p.Data().(page.Data).Pages(), deepEqualsPages, p.Pages())
writeSourcesToSource(t, "content", fs, sources...)
s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{})
- if s.Taxonomies["tags"]["a"][0].Page.Title() != "foo" {
- t.Errorf("Pages in unexpected order, 'foo' expected first, got '%v'", s.Taxonomies["tags"]["a"][0].Page.Title())
+ if s.Taxonomies()["tags"]["a"][0].Page.Title() != "foo" {
+ t.Errorf("Pages in unexpected order, 'foo' expected first, got '%v'", s.Taxonomies()["tags"]["a"][0].Page.Title())
}
- if s.Taxonomies["categories"]["d"][0].Page.Title() != "bar" {
- t.Errorf("Pages in unexpected order, 'bar' expected first, got '%v'", s.Taxonomies["categories"]["d"][0].Page.Title())
+ if s.Taxonomies()["categories"]["d"][0].Page.Title() != "bar" {
+ t.Errorf("Pages in unexpected order, 'bar' expected first, got '%v'", s.Taxonomies()["categories"]["d"][0].Page.Title())
}
- if s.Taxonomies["categories"]["e"][0].Page.Title() != "bza" {
- t.Errorf("Pages in unexpected order, 'bza' expected first, got '%v'", s.Taxonomies["categories"]["e"][0].Page.Title())
+ if s.Taxonomies()["categories"]["e"][0].Page.Title() != "bza" {
+ t.Errorf("Pages in unexpected order, 'bza' expected first, got '%v'", s.Taxonomies()["categories"]["e"][0].Page.Title())
}
}
//test empty link, as well as fragment only link
{"", "", true, ""},
} {
- checkLinkCase(site, test.link, currentPage, test.relative, test.outputFormat, test.expected, t, i)
- //make sure fragment links are also handled
- checkLinkCase(site, test.link+"#intro", currentPage, test.relative, test.outputFormat, test.expected+"#intro", t, i)
+ t.Run(fmt.Sprint(i), func(t *testing.T) {
+ checkLinkCase(site, test.link, currentPage, test.relative, test.outputFormat, test.expected, t, i)
+
+ //make sure fragment links are also handled
+ checkLinkCase(site, test.link+"#intro", currentPage, test.relative, test.outputFormat, test.expected+"#intro", t, i)
+ })
}
// TODO: and then the failure cases.
s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{})
st := make([]string, 0)
- for _, t := range s.Taxonomies["tags"].ByCount() {
+ for _, t := range s.Taxonomies()["tags"].ByCount() {
st = append(st, t.Page().Title()+":"+t.Name)
}
}
for taxonomy, count := range taxonomyTermPageCounts {
+ msg := qt.Commentf(taxonomy)
term := s.getPage(page.KindTaxonomyTerm, taxonomy)
- b.Assert(term, qt.Not(qt.IsNil))
- b.Assert(len(term.Pages()), qt.Equals, count, qt.Commentf(taxonomy))
+ b.Assert(term, qt.Not(qt.IsNil), msg)
+ b.Assert(len(term.Pages()), qt.Equals, count, msg)
for _, p := range term.Pages() {
b.Assert(p.Kind(), qt.Equals, page.KindTaxonomy)
s := b.H.Sites[0]
- ta := s.findPagesByKind(page.KindTaxonomy)
- te := s.findPagesByKind(page.KindTaxonomyTerm)
+ filterbyKind := func(kind string) page.Pages {
+ var pages page.Pages
+ for _, p := range s.Pages() {
+ if p.Kind() == kind {
+ pages = append(pages, p)
+ }
+ }
+ return pages
+ }
+
+ ta := filterbyKind(page.KindTaxonomy)
+ te := filterbyKind(page.KindTaxonomyTerm)
b.Assert(len(te), qt.Equals, 4)
b.Assert(len(ta), qt.Equals, 7)
}
-// See https://github.com/gohugoio/hugo/issues/6222
-// We need to revisit this once we figure out what to do with the
-// draft etc _index pages, but for now we need to avoid the crash.
func TestTaxonomiesIndexDraft(t *testing.T) {
t.Parallel()
draft: true
---
-This is the invisible content.
+Content.
-`)
+`,
+ "page.md", `---
+title: "The Page"
+categories: ["cool"]
+---
+
+Content.
+
+`,
+ )
b.WithTemplates("index.html", `
{{ range .Site.Pages }}
b.Build(BuildCfg{})
- // We publish the index page, but the content will be empty.
- b.AssertFileContent("public/index.html", " /categories/|The Categories|0||")
+ b.AssertFileContentFn("public/index.html", func(s string) bool {
+ return !strings.Contains(s, "categories")
+ })
+
+}
+
+// https://github.com/gohugoio/hugo/issues/6173
+func TestTaxonomiesWithBundledResources(t *testing.T) {
+ b := newTestSitesBuilder(t)
+ b.WithTemplates("_default/list.html", `
+List {{ .Title }}:
+{{ range .Resources }}
+Resource: {{ .RelPermalink }}|{{ .MediaType }}
+{{ end }}
+ `)
+
+ b.WithContent("p1.md", `---
+title: Page
+categories: ["funny"]
+---
+ `,
+ "categories/_index.md", "---\ntitle: Categories Page\n---",
+ "categories/data.json", "Category data",
+ "categories/funny/_index.md", "---\ntitle: Funnny Category\n---",
+ "categories/funny/funnydata.json", "Category funny data",
+ )
+
+ b.Build(BuildCfg{})
+
+ b.AssertFileContent("public/categories/index.html", `Resource: /categories/data.json|application/json`)
+ b.AssertFileContent("public/categories/funny/index.html", `Resource: /categories/funny/funnydata.json|application/json`)
+
+}
+
+func TestTaxonomiesRemoveOne(t *testing.T) {
+ b := newTestSitesBuilder(t).Running()
+ b.WithTemplates("index.html", `
+ {{ $cats := .Site.Taxonomies.categories.cats }}
+ {{ if $cats }}
+ Len cats: {{ len $cats }}
+ {{ range $cats }}
+ Cats:|{{ .Page.RelPermalink }}|
+ {{ end }}
+ {{ end }}
+ {{ $funny := .Site.Taxonomies.categories.funny }}
+ {{ if $funny }}
+ Len funny: {{ len $funny }}
+ {{ range $funny }}
+ Funny:|{{ .Page.RelPermalink }}|
+ {{ end }}
+ {{ end }}
+ `)
+
+ b.WithContent("p1.md", `---
+title: Page
+categories: ["funny", "cats"]
+---
+ `, "p2.md", `---
+title: Page2
+categories: ["funny", "cats"]
+---
+ `,
+ )
+
+ b.Build(BuildCfg{})
+
+ b.AssertFileContent("public/index.html", `
+Len cats: 2
+Len funny: 2
+Cats:|/p1/|
+Cats:|/p2/|
+Funny:|/p1/|
+Funny:|/p2/|`)
+
+ // Remove one category from one of the pages.
+ b.EditFiles("content/p1.md", `---
+title: Page
+categories: ["funny"]
+---
+ `)
+
+ b.Build(BuildCfg{})
+
+ b.AssertFileContent("public/index.html", `
+Len cats: 1
+Len funny: 2
+Cats:|/p2/|
+Funny:|/p1/|
+Funny:|/p2/|`)
+
+}
+
+//https://github.com/gohugoio/hugo/issues/6590
+func TestTaxonomiesListPages(t *testing.T) {
+ b := newTestSitesBuilder(t)
+ b.WithTemplates("_default/list.html", `
+
+{{ template "print-taxo" "categories.cats" }}
+{{ template "print-taxo" "categories.funny" }}
+
+{{ define "print-taxo" }}
+{{ $node := index site.Taxonomies (split $ ".") }}
+{{ if $node }}
+Len {{ $ }}: {{ len $node }}
+{{ range $node }}
+ {{ $ }}:|{{ .Page.RelPermalink }}|
+{{ end }}
+{{ else }}
+{{ $ }} not found.
+{{ end }}
+{{ end }}
+ `)
+
+ b.WithContent("_index.md", `---
+title: Home
+categories: ["funny", "cats"]
+---
+ `, "blog/p1.md", `---
+title: Page1
+categories: ["funny"]
+---
+ `, "blog/_index.md", `---
+title: Blog Section
+categories: ["cats"]
+---
+ `,
+ )
+
+ b.Build(BuildCfg{})
+
+ b.AssertFileContent("public/index.html", `
+
+Len categories.cats: 2
+categories.cats:|/blog/|
+categories.cats:|/|
+
+Len categories.funny: 2
+categories.funny:|/|
+categories.funny:|/blog/p1/|
+`)
}
import (
"fmt"
"path/filepath"
+ "strings"
"testing"
"github.com/gohugoio/hugo/identity"
}
}
-func printRecursiveIdentities(level int, id identity.Provider) {
- if level == 0 {
- fmt.Println(id.GetIdentity(), "===>")
- }
- if ids, ok := id.(identity.IdentitiesProvider); ok {
- level++
- for _, id := range ids.GetIdentities() {
- printRecursiveIdentities(level, id)
- }
- } else {
- ident(level)
- fmt.Println("ID", id)
- }
-}
-
-func ident(n int) {
- for i := 0; i < n; i++ {
- fmt.Print(" ")
- }
+func ident(level int) string {
+ return strings.Repeat(" ", level)
}
"time"
"unicode/utf8"
+ "github.com/gohugoio/hugo/htesting"
+
"github.com/gohugoio/hugo/output"
"github.com/gohugoio/hugo/parser/metadecoders"
if expected != got {
fmt.Println(got)
- diff := helpers.DiffStrings(expected, got)
+ diff := htesting.DiffStrings(expected, got)
s.Fatalf("diff:\n%s\nexpected\n%s\ngot\n%s", diff, expected, got)
}
}
return p
}
+func (s *sitesBuilder) GetPageRel(p page.Page, ref string) page.Page {
+ p, err := s.H.Sites[0].getPageNew(p, ref)
+ s.Assert(err, qt.IsNil)
+ return p
+}
+
func newTestHelper(cfg config.Provider, fs *hugofs.Fs, t testing.TB) testHelper {
return testHelper{
Cfg: cfg,
out := make(map[string]page.Pages)
for _, s := range sites {
- for _, p := range s.workAllPages {
+ s.pageMap.pageTrees.Walk(func(ss string, n *contentNode) bool {
+ p := n.p
// TranslationKey is implemented for all page types.
base := p.TranslationKey()
pageTranslations = append(pageTranslations, p)
out[base] = pageTranslations
- }
+
+ return false
+ })
}
return out
func assignTranslationsToPages(allTranslations map[string]page.Pages, sites []*Site) {
for _, s := range sites {
- for _, p := range s.workAllPages {
+ s.pageMap.pageTrees.Walk(func(ss string, n *contentNode) bool {
+ p := n.p
base := p.TranslationKey()
translations, found := allTranslations[base]
if !found {
- continue
+ return false
}
-
p.setTranslations(translations)
- }
+ return false
+ })
}
}
"strings"
"github.com/gohugoio/hugo/media"
-
- "github.com/gohugoio/hugo/parser/pageparser"
)
type Format string
return ""
}
-// FormatFromFrontMatterType will return empty if not supported.
-func FormatFromFrontMatterType(typ pageparser.ItemType) Format {
- switch typ {
- case pageparser.TypeFrontMatterJSON:
- return JSON
- case pageparser.TypeFrontMatterORG:
- return ORG
- case pageparser.TypeFrontMatterTOML:
- return TOML
- case pageparser.TypeFrontMatterYAML:
- return YAML
- default:
- return ""
- }
-}
-
// FormatFromContentString tries to detect the format (JSON, YAML or TOML)
// in the given string.
// It return an empty string if no format could be detected.
"github.com/gohugoio/hugo/media"
- "github.com/gohugoio/hugo/parser/pageparser"
-
qt "github.com/frankban/quicktest"
)
}
}
-func TestFormatFromFrontMatterType(t *testing.T) {
- c := qt.New(t)
- for _, test := range []struct {
- typ pageparser.ItemType
- expect Format
- }{
- {pageparser.TypeFrontMatterJSON, JSON},
- {pageparser.TypeFrontMatterTOML, TOML},
- {pageparser.TypeFrontMatterYAML, YAML},
- {pageparser.TypeFrontMatterORG, ORG},
- {pageparser.TypeIgnore, ""},
- } {
- c.Assert(FormatFromFrontMatterType(test.typ), qt.Equals, test.expect)
- }
-}
-
func TestFormatFromContentString(t *testing.T) {
t.Parallel()
c := qt.New(t)
"io"
"io/ioutil"
+ "github.com/gohugoio/hugo/parser/metadecoders"
"github.com/pkg/errors"
)
return parseSection(r, cfg, lexIntroSection)
}
+type ContentFrontMatter struct {
+ Content []byte
+ FrontMatter map[string]interface{}
+ FrontMatterFormat metadecoders.Format
+}
+
+// ParseFrontMatterAndContent is a convenience method to extract front matter
+// and content from a content page.
+func ParseFrontMatterAndContent(r io.Reader) (ContentFrontMatter, error) {
+ var cf ContentFrontMatter
+
+ psr, err := Parse(r, Config{})
+ if err != nil {
+ return cf, err
+ }
+
+ var frontMatterSource []byte
+
+ iter := psr.Iterator()
+
+ walkFn := func(item Item) bool {
+ if frontMatterSource != nil {
+ // The rest is content.
+ cf.Content = psr.Input()[item.Pos:]
+ // Done
+ return false
+ } else if item.IsFrontMatter() {
+ cf.FrontMatterFormat = FormatFromFrontMatterType(item.Type)
+ frontMatterSource = item.Val
+ }
+ return true
+
+ }
+
+ iter.PeekWalk(walkFn)
+
+ cf.FrontMatter, err = metadecoders.Default.UnmarshalToMap(frontMatterSource, cf.FrontMatterFormat)
+ return cf, err
+}
+
+func FormatFromFrontMatterType(typ ItemType) metadecoders.Format {
+ switch typ {
+ case TypeFrontMatterJSON:
+ return metadecoders.JSON
+ case TypeFrontMatterORG:
+ return metadecoders.ORG
+ case TypeFrontMatterTOML:
+ return metadecoders.TOML
+ case TypeFrontMatterYAML:
+ return metadecoders.YAML
+ default:
+ return ""
+ }
+}
+
// ParseMain parses starting with the main section. Used in tests.
func ParseMain(r io.Reader, cfg Config) (Result, error) {
return parseSection(r, cfg, lexMainSection)
import (
"strings"
"testing"
+
+ qt "github.com/frankban/quicktest"
+ "github.com/gohugoio/hugo/parser/metadecoders"
)
func BenchmarkParse(b *testing.B) {
}
}
}
+
+func TestFormatFromFrontMatterType(t *testing.T) {
+ c := qt.New(t)
+ for _, test := range []struct {
+ typ ItemType
+ expect metadecoders.Format
+ }{
+ {TypeFrontMatterJSON, metadecoders.JSON},
+ {TypeFrontMatterTOML, metadecoders.TOML},
+ {TypeFrontMatterYAML, metadecoders.YAML},
+ {TypeFrontMatterORG, metadecoders.ORG},
+ {TypeIgnore, ""},
+ } {
+ c.Assert(FormatFromFrontMatterType(test.typ), qt.Equals, test.expect)
+ }
+}
store map[string]*resourceAdapter
}
-func (c *imageCache) isInCache(key string) bool {
- c.mu.RLock()
- _, found := c.store[c.normalizeKey(key)]
- c.mu.RUnlock()
- return found
-}
-
-func (c *imageCache) deleteByPrefix(prefix string) {
+func (c *imageCache) deleteIfContains(s string) {
c.mu.Lock()
defer c.mu.Unlock()
- prefix = c.normalizeKey(prefix)
+ s = c.normalizeKeyBase(s)
for k := range c.store {
- if strings.HasPrefix(k, prefix) {
+ if strings.Contains(k, s) {
delete(c.store, k)
}
}
}
+// The cache key is a lowecase path with Unix style slashes and it always starts with
+// a leading slash.
func (c *imageCache) normalizeKey(key string) string {
- // It is a path with Unix style slashes and it always starts with a leading slash.
- key = filepath.ToSlash(key)
- if !strings.HasPrefix(key, "/") {
- key = "/" + key
- }
+ return "/" + c.normalizeKeyBase(key)
+}
- return key
+func (c *imageCache) normalizeKeyBase(key string) string {
+ return strings.Trim(strings.ToLower(filepath.ToSlash(key)), "/")
}
func (c *imageCache) clear() {
createImage func() (*imageResource, image.Image, error)) (*resourceAdapter, error) {
relTarget := parent.relTargetPathFromConfig(conf)
memKey := parent.relTargetPathForRel(relTarget.path(), false, false, false)
+ memKey = c.normalizeKey(memKey)
// For the file cache we want to generate and store it once if possible.
fileKeyPath := relTarget
}
resized, err := orig.Fill("400x200 center")
+ c.Assert(err, qt.IsNil)
for _, filter := range filters {
resized, err := resized.Filter(filter)
"github.com/gohugoio/hugo/common/hugo"
"github.com/gohugoio/hugo/common/maps"
-
"github.com/gohugoio/hugo/compare"
+ "github.com/gohugoio/hugo/hugofs/files"
"github.com/gohugoio/hugo/navigation"
"github.com/gohugoio/hugo/related"
// BundleType returns the bundle type: "leaf", "branch" or an empty string if it is none.
// See https://gohugo.io/content-management/page-bundles/
- BundleType() string
+ BundleType() files.ContentClass
// A configured description.
Description() string
"github.com/bep/gitmap"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/config"
+ "github.com/gohugoio/hugo/hugofs/files"
"github.com/gohugoio/hugo/langs"
"github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/navigation"
PublishDate time.Time
ExpiryDate time.Time
Aliases []string
- BundleType string
+ BundleType files.ContentClass
Description string
Draft bool
IsHome bool
"html/template"
"time"
+ "github.com/gohugoio/hugo/hugofs/files"
+
"github.com/gohugoio/hugo/hugofs"
"github.com/bep/gitmap"
return ""
}
-func (p *nopPage) BundleType() string {
+func (p *nopPage) BundleType() files.ContentClass {
return ""
}
package pagemeta
+import (
+ "github.com/mitchellh/mapstructure"
+)
+
type URLPath struct {
URL string
Permalink string
Slug string
Section string
}
+
+var defaultBuildConfig = BuildConfig{
+ List: true,
+ Render: true,
+ PublishResources: true,
+ set: true,
+}
+
+// BuildConfig holds configuration options about how to handle a Page in Hugo's
+// build process.
+type BuildConfig struct {
+ // Whether to add it to any of the page collections.
+ // Note that the page can still be found with .Site.GetPage.
+ List bool
+
+ // Whether to render it.
+ Render bool
+
+ // Whether to publish its resources. These will still be published on demand,
+ // but enabling this can be useful if the originals (e.g. images) are
+ // never used.
+ PublishResources bool
+
+ set bool // BuildCfg is non-zero if this is set to true.
+}
+
+// Disable sets all options to their off value.
+func (b *BuildConfig) Disable() {
+ b.List = false
+ b.Render = false
+ b.PublishResources = false
+ b.set = true
+}
+
+func (b BuildConfig) IsZero() bool {
+ return !b.set
+}
+
+func DecodeBuildConfig(m interface{}) (BuildConfig, error) {
+ b := defaultBuildConfig
+ if m == nil {
+ return b, nil
+ }
+ err := mapstructure.WeakDecode(m, &b)
+ return b, err
+}
"path/filepath"
"time"
+ "github.com/gohugoio/hugo/hugofs/files"
+
"github.com/gohugoio/hugo/modules"
"github.com/bep/gitmap"
panic("not implemented")
}
-func (p *testPage) BundleType() string {
+func (p *testPage) BundleType() files.ContentClass {
panic("not implemented")
}
r.ResourceCache.clear()
}
-func (r *Spec) DeleteCacheByPrefix(prefix string) {
- r.imageCache.deleteByPrefix(prefix)
-}
-
-// TODO(bep) unify
-func (r *Spec) IsInImageCache(key string) bool {
- // This is used for cache pruning. We currently only have images, but we could
- // imagine expanding on this.
- return r.imageCache.isInCache(key)
+func (r *Spec) DeleteBySubstring(s string) {
+ r.imageCache.deleteIfContains(s)
}
func (s *Spec) String() string {
return vv.Float()
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
return vv.Uint()
+ case reflect.String:
+ return vv.String()
default:
return v
}
return string(t)
}
+type stringType string
+
type tstCompareType int
const (
}
func TestCase(t *testing.T) {
+ c := qt.New(t)
+ n := New(false)
+
+ c.Assert(n.Eq("az", "az"), qt.Equals, true)
+ c.Assert(n.Eq("az", stringType("az")), qt.Equals, true)
+
+}
+
+func TestStringType(t *testing.T) {
c := qt.New(t)
n := New(true)
import (
"testing"
+ "github.com/gohugoio/hugo/htesting"
+
qt "github.com/frankban/quicktest"
- "github.com/gohugoio/hugo/helpers"
"github.com/spf13/viper"
)
converted, err := ns.Remarshal(v1.format, v2.data)
c.Assert(err, qt.IsNil, fromTo)
- diff := helpers.DiffStrings(v1.data, converted)
+ diff := htesting.DiffStrings(v1.data, converted)
if len(diff) > 0 {
t.Errorf("[%s] Expected \n%v\ngot\n%v\ndiff:\n%v", fromTo, v1.data, converted, diff)
}
c.Assert(err, qt.IsNil, fromTo)
}
- diff := helpers.DiffStrings(expected, converted)
+ diff := htesting.DiffStrings(expected, converted)
if len(diff) > 0 {
t.Fatalf("[%s] Expected \n%v\ngot\n%v\ndiff:\n%v\n", fromTo, expected, converted, diff)
}
projects / brevno-suite / hugo / commitdiff