h.BaseFs.SourceFilesystems,
dynamicEvents)
- onePageName := pickOneWriteOrCreatePath(partitionedEvents.ContentEvents)
+ onePageName := pickOneWriteOrCreatePath(h.Conf.ContentTypes(), partitionedEvents.ContentEvents)
c.printChangeDetected("")
c.changeDetector.PrepareNew()
"github.com/fsnotify/fsnotify"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/hugo"
+
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/common/urls"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
- "github.com/gohugoio/hugo/hugofs/files"
"github.com/gohugoio/hugo/hugolib"
"github.com/gohugoio/hugo/hugolib/filesystems"
"github.com/gohugoio/hugo/livereload"
return
}
-func pickOneWriteOrCreatePath(events []fsnotify.Event) string {
+func pickOneWriteOrCreatePath(contentTypes config.ContentTypesProvider, events []fsnotify.Event) string {
name := ""
for _, ev := range events {
if ev.Op&fsnotify.Write == fsnotify.Write || ev.Op&fsnotify.Create == fsnotify.Create {
- if files.IsIndexContentFile(ev.Name) {
+ if contentTypes.IsIndexContentFile(ev.Name) {
return ev.Name
}
- if files.IsContentFile(ev.Name) {
+ if contentTypes.IsContentFile(ev.Name) {
name = ev.Name
}
}
// Delete deletes the given key from the cache.
+// If c is nil, this method is a no-op.
func (c *Cache[K, T]) Get(key K) (T, bool) {
+ if c == nil {
+ var zero T
+ return zero, false
+ }
c.RLock()
v, found := c.m[key]
c.RUnlock()
c.Unlock()
}
+// ForEeach calls the given function for each key/value pair in the cache.
+func (c *Cache[K, T]) ForEeach(f func(K, T)) {
+ c.RLock()
+ defer c.RUnlock()
+ for k, v := range c.m {
+ f(k, v)
+ }
+}
+
// SliceCache is a simple thread safe cache backed by a map.
type SliceCache[T any] struct {
m map[string][]T
"github.com/gohugoio/hugo/identity"
)
-var defaultPathParser PathParser
-
// PathParser parses a path into a Path.
type PathParser struct {
// Maps the language code to its index in the languages/sites slice.
// Reports whether the given language is disabled.
IsLangDisabled func(string) bool
-}
-// Parse parses component c with path s into Path using the default path parser.
-func Parse(c, s string) *Path {
- return defaultPathParser.Parse(c, s)
+ // Reports whether the given ext is a content file.
+ IsContentExt func(string) bool
}
// NormalizePathString returns a normalized path string using the very basic Hugo rules.
var err error
// Preserve the original case for titles etc.
p.unnormalized, err = pp.doParse(component, s, pp.newPath(component))
-
if err != nil {
return nil, err
}
}
}
- isContentComponent := p.component == files.ComponentFolderContent || p.component == files.ComponentFolderArchetypes
- isContent := isContentComponent && files.IsContentExt(p.Ext())
-
- if isContent {
+ if len(p.identifiers) > 0 {
+ isContentComponent := p.component == files.ComponentFolderContent || p.component == files.ComponentFolderArchetypes
+ isContent := isContentComponent && pp.IsContentExt(p.Ext())
id := p.identifiers[len(p.identifiers)-1]
b := p.s[p.posContainerHigh : id.Low-1]
- switch b {
- case "index":
- p.bundleType = PathTypeLeaf
- case "_index":
- p.bundleType = PathTypeBranch
- default:
- p.bundleType = PathTypeContentSingle
- }
+ if isContent {
+ switch b {
+ case "index":
+ p.bundleType = PathTypeLeaf
+ case "_index":
+ p.bundleType = PathTypeBranch
+ default:
+ p.bundleType = PathTypeContentSingle
+ }
- if slashCount == 2 && p.IsLeafBundle() {
- p.posSectionHigh = 0
+ if slashCount == 2 && p.IsLeafBundle() {
+ p.posSectionHigh = 0
+ }
+ } else if b == files.NameContentData && files.IsContentDataExt(p.Ext()) {
+ p.bundleType = PathTypeContentData
}
}
// Branch bundles, e.g. /blog/_index.md
PathTypeBranch
+
+ // Content data file, _content.gotmpl.
+ PathTypeContentData
)
type Path struct {
return ids
}
-func (p *Path) IsHTML() bool {
- return files.IsHTML(p.Ext())
-}
-
func (p *Path) BundleType() PathType {
return p.bundleType
}
return p.bundleType == PathTypeLeaf
}
+func (p *Path) IsContentData() bool {
+ return p.bundleType == PathTypeContentData
+}
+
func (p Path) ForBundleType(t PathType) *Path {
p.bundleType = t
return &p
"no": 0,
"en": 1,
},
+ IsContentExt: func(ext string) bool {
+ return ext == "md"
+ },
}
func TestParse(t *testing.T) {
c.Assert(p.Path(), qt.Equals, "/a/b/c.txt")
},
},
+ {
+ "Content data file gotmpl",
+ "/a/b/_content.gotmpl",
+ func(c *qt.C, p *Path) {
+ c.Assert(p.Path(), qt.Equals, "/a/b/_content.gotmpl")
+ c.Assert(p.Ext(), qt.Equals, "gotmpl")
+ c.Assert(p.IsContentData(), qt.IsTrue)
+ },
+ },
+ {
+ "Content data file yaml",
+ "/a/b/_content.yaml",
+ func(c *qt.C, p *Path) {
+ c.Assert(p.IsContentData(), qt.IsFalse)
+ },
+ },
}
for _, test := range tests {
c.Run(test.name, func(c *qt.C) {
DisabledLanguages: disabledLangs,
IgnoredLogs: ignoredLogIDs,
KindOutputFormats: kindOutputFormats,
+ ContentTypes: media.DefaultContentTypes.FromTypes(c.MediaTypes.Config),
CreateTitle: helpers.GetTitleFunc(c.TitleCaseStyle),
IsUglyURLSection: isUglyURL,
IgnoreFile: ignoreFile,
BaseURLLiveReload urls.BaseURL
ServerInterface string
KindOutputFormats map[string]output.Formats
+ ContentTypes media.ContentTypes
DisabledKinds map[string]bool
DisabledLanguages map[string]bool
IgnoredLogs map[string]bool
c.Languages = languages
c.LanguagesDefaultFirst = languagesDefaultFirst
- c.ContentPathParser = &paths.PathParser{LanguageIndex: languagesDefaultFirst.AsIndexSet(), IsLangDisabled: c.Base.IsLangDisabled}
+ c.ContentPathParser = &paths.PathParser{LanguageIndex: languagesDefaultFirst.AsIndexSet(), IsLangDisabled: c.Base.IsLangDisabled, IsContentExt: c.Base.C.ContentTypes.IsContentSuffix}
c.configLangs = make([]config.AllProvider, len(c.Languages))
for i, l := range c.LanguagesDefaultFirst {
b.Assert(conf.PrintI18nWarnings, qt.Equals, true)
b.Assert(conf.PrintPathWarnings, qt.Equals, true)
}
+
+func TestRedefineContentTypes(t *testing.T) {
+ files := `
+-- hugo.toml --
+baseURL = "https://example.com"
+[mediaTypes]
+[mediaTypes."text/html"]
+suffixes = ["html", "xhtml"]
+`
+
+ b := hugolib.Test(t, files)
+
+ conf := b.H.Configs.Base
+ contentTypes := conf.C.ContentTypes
+
+ b.Assert(contentTypes.HTML.Suffixes(), qt.DeepEquals, []string{"html", "xhtml"})
+ b.Assert(contentTypes.Markdown.Suffixes(), qt.DeepEquals, []string{"md", "mdown", "markdown"})
+}
return identity.NewManager(name)
}
+func (c ConfigLanguage) ContentTypes() config.ContentTypesProvider {
+ return c.config.C.ContentTypes
+}
+
// GetConfigSection is mostly used in tests. The switch statement isn't complete, but what's in use.
func (c ConfigLanguage) GetConfigSection(s string) any {
switch s {
Dirs() CommonDirs
Quiet() bool
DirsBase() CommonDirs
+ ContentTypes() ContentTypesProvider
GetConfigSection(string) any
GetConfig() any
CanonifyURLs() bool
EnableEmoji() bool
}
+// We cannot import the media package as that would create a circular dependency.
+// This interface defineds a sub set of what media.ContentTypes provides.
+type ContentTypesProvider interface {
+ IsContentSuffix(suffix string) bool
+ IsContentFile(filename string) bool
+ IsIndexContentFile(filename string) bool
+ IsHTMLSuffix(suffix string) bool
+}
+
// Provider provides the configuration settings for Hugo.
type Provider interface {
GetString(key string) string
"github.com/gohugoio/hugo/common/hstrings"
"github.com/gohugoio/hugo/common/paths"
- "github.com/gohugoio/hugo/hugofs/files"
-
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/helpers"
return "", fmt.Errorf("failed to resolve %q to an archetype template", targetPath)
}
- if !files.IsContentFile(b.targetPath) {
+ if !h.Conf.ContentTypes().IsContentFile(b.targetPath) {
return "", fmt.Errorf("target path %q is not a known content format", b.targetPath)
}
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/loggers"
+ "github.com/gohugoio/hugo/media"
"github.com/spf13/afero"
}
func (c *ContentSpec) ResolveMarkup(in string) string {
- if c == nil {
- panic("nil ContentSpec")
- }
in = strings.ToLower(in)
- switch in {
- case "md", "markdown", "mdown":
- return "markdown"
- case "html", "htm":
- return "html"
- default:
- if conv := c.Converters.Get(in); conv != nil {
- return conv.Name()
- }
+
+ if mediaType, found := c.Cfg.ContentTypes().(media.ContentTypes).Types().GetBestMatch(markup.ResolveMarkup(in)); found {
+ return mediaType.SubType
}
+
+ if conv := c.Converters.Get(in); conv != nil {
+ return markup.ResolveMarkup(conv.Name())
+ }
+
return ""
}
openingTag := []byte("<p>")
closingTag := []byte("</p>")
- if markup == "asciidocext" {
+ if markup == media.DefaultContentTypes.AsciiDoc.SubType {
openingTag = []byte("<div class=\"paragraph\">\n<p>")
closingTag = []byte("</p>\n</div>")
}
{"markdown", []byte("<h2 id=`a`>b</h2>\n\n<p>c</p>"), []byte("<h2 id=`a`>b</h2>\n\n<p>c</p>")},
// Issue 12369
{"markdown", []byte("<div class=\"paragraph\">\n<p>foo</p>\n</div>"), []byte("<div class=\"paragraph\">\n<p>foo</p>\n</div>")},
- {"asciidoc
ext", []byte("<div class=\"paragraph\">\n<p>foo</p>\n</div>"), []byte("foo")},
+ {"asciidoc", []byte("<div class=\"paragraph\">\n<p>foo</p>\n</div>"), []byte("foo")},
}
c := newTestContentSpec(nil)
{"md", "markdown"},
{"markdown", "markdown"},
{"mdown", "markdown"},
- {"asciidocext", "asciidocext"},
- {"adoc", "asciidocext"},
- {"ad", "asciidocext"},
+ {"asciidocext", "asciidoc"},
+ {"adoc", "asciidoc"},
+ {"ad", "asciidoc"},
{"rst", "rst"},
{"pandoc", "pandoc"},
{"pdc", "pandoc"},
FilenameHugoStatsJSON = "hugo_stats.json"
)
-var (
- // This should be the only list of valid extensions for content files.
- contentFileExtensions = []string{
- "html", "htm",
- "mdown", "markdown", "md",
- "asciidoc", "adoc", "ad",
- "rest", "rst",
- "org",
- "pandoc", "pdc",
- }
-
- contentFileExtensionsSet map[string]bool
-
- htmlFileExtensions = []string{
- "html", "htm",
- }
-
- htmlFileExtensionsSet map[string]bool
-)
-
-func init() {
- contentFileExtensionsSet = make(map[string]bool)
- for _, ext := range contentFileExtensions {
- contentFileExtensionsSet[ext] = true
- }
- htmlFileExtensionsSet = make(map[string]bool)
- for _, ext := range htmlFileExtensions {
- htmlFileExtensionsSet[ext] = true
- }
+func IsGoTmplExt(ext string) bool {
+ return ext == "gotmpl"
}
-func IsContentFile(filename string) bool {
- return contentFileExtensionsSet[strings.TrimPrefix(filepath.Ext(filename), ".")]
-}
-
-func IsIndexContentFile(filename string) bool {
- if !IsContentFile(filename) {
- return false
- }
-
- base := filepath.Base(filename)
-
- return strings.HasPrefix(base, "index.") || strings.HasPrefix(base, "_index.")
-}
-
-func IsHTML(ext string) bool {
- return htmlFileExtensionsSet[ext]
-}
-
-func IsContentExt(ext string) bool {
- return contentFileExtensionsSet[ext]
+// Supported data file extensions for _content.* files.
+func IsContentDataExt(ext string) bool {
+ return IsGoTmplExt(ext)
}
const (
FolderResources = "resources"
FolderJSConfig = "_jsconfig" // Mounted below /assets with postcss.config.js etc.
+
+ NameContentData = "_content"
)
var (
package files
import (
- "path/filepath"
"testing"
qt "github.com/frankban/quicktest"
)
-func TestIsContentFile(t *testing.T) {
- c := qt.New(t)
-
- c.Assert(IsContentFile(filepath.FromSlash("my/file.md")), qt.Equals, true)
- c.Assert(IsContentFile(filepath.FromSlash("my/file.ad")), qt.Equals, true)
- c.Assert(IsContentFile(filepath.FromSlash("textfile.txt")), qt.Equals, false)
- c.Assert(IsContentExt("md"), qt.Equals, true)
- c.Assert(IsContentExt("json"), qt.Equals, false)
-}
-
func TestComponentFolders(t *testing.T) {
c := qt.New(t)
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/paths"
+ "github.com/gohugoio/hugo/media"
"github.com/spf13/afero"
)
Root string
// The logger to use.
- Logger loggers.Logger
+ Logger loggers.Logger
+ PathParser *paths.PathParser
// One or both of these may be pre-set.
Info FileMetaInfo // The start info.
panic("fs must be set")
}
+ if cfg.PathParser == nil {
+ cfg.PathParser = media.DefaultPathParser
+ }
+
logger := cfg.Logger
if logger == nil {
logger = loggers.NewDefault()
dirEntries = DirEntriesToFileMetaInfos(fis)
for _, fi := range dirEntries {
if fi.Meta().PathInfo == nil {
- fi.Meta().PathInfo = paths.Parse("", filepath.Join(pathRel, fi.Name()))
+ fi.Meta().PathInfo = w.cfg.PathParser.Parse("", filepath.Join(pathRel, fi.Name()))
}
}
enConfig := b.H.Sites[0].conf
m, _ := enConfig.MediaTypes.Config.GetByType("text/html")
- b.Assert(m.Suffixes(), qt.DeepEquals, []string{"html"})
+ b.Assert(m.Suffixes(), qt.DeepEquals, []string{"html", "htm"})
svConfig := b.H.Sites[1].conf
f, _ := svConfig.OutputFormats.Config.GetByName("html")
package hugolib
import (
+ "context"
"fmt"
"path"
"path/filepath"
"github.com/bep/logg"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/common/paths"
+ "github.com/gohugoio/hugo/hugofs/files"
+ "github.com/gohugoio/hugo/hugolib/pagesfromdata"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/source"
"github.com/gohugoio/hugo/resources/page"
+ "github.com/gohugoio/hugo/resources/page/pagemeta"
"github.com/gohugoio/hugo/resources/resource"
"github.com/gohugoio/hugo/hugofs"
var _ contentNodeI = (*resourceSource)(nil)
type resourceSource struct {
- path *paths.Path
- opener hugio.OpenReadSeekCloser
- fi hugofs.FileMetaInfo
+ langIndex int
+ path *paths.Path
+ opener hugio.OpenReadSeekCloser
+ fi hugofs.FileMetaInfo
+ rc *pagemeta.ResourceConfig
r resource.Resource
}
}
func (r *resourceSource) LangIndex() int {
- if r.r != nil && r.isPage() {
- return r.r.(*pageState).s.languagei
- }
-
- return r.fi.Meta().LangIndex
+ return r.langIndex
}
func (r *resourceSource) MarkStale() {
return
}
-func (m *pageMap) AddFi(fi hugofs.FileMetaInfo) error {
+func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, buildConfig *BuildCfg) (pageCount uint64, resourceCount uint64, addErr error) {
if fi.IsDir() {
- return nil
+ return
}
insertResource := func(fim hugofs.FileMetaInfo) error {
+ resourceCount++
pi := fi.Meta().PathInfo
key := pi.Base()
tree := m.treeResources
}
key = pi.Base()
- rs = &resourceSource{r: pageResource}
+ rs = &resourceSource{r: pageResource, langIndex: pageResource.s.languagei}
} else {
- rs = &resourceSource{path: pi, opener: r, fi: fim}
+ rs = &resourceSource{path: pi, opener: r, fi: fim, langIndex: fim.Meta().LangIndex}
}
tree.InsertIntoValuesDimension(key, rs)
},
))
if err := insertResource(fi); err != nil {
- return err
+ addErr = err
+ return
}
+ case paths.PathTypeContentData:
+ pc, rc, err := m.addPagesFromGoTmplFi(fi, buildConfig)
+ pageCount += pc
+ resourceCount += rc
+ if err != nil {
+ addErr = err
+ return
+ }
+
default:
m.s.Log.Trace(logg.StringFunc(
func() string {
return fmt.Sprintf("insert bundle: %q", fi.Meta().Filename)
},
))
+
+ pageCount++
+
// A content file.
p, pi, err := m.s.h.newPage(
&pageMeta{
},
)
if err != nil {
- return err
+ addErr = err
+ return
}
if p == nil {
// Disabled page.
- return nil
+ return
}
- m.treePages.InsertWithLock(pi.Base(), p)
+ m.treePages.InsertIntoValuesDimensionWithLock(pi.Base(), p)
}
- return nil
+ return
+}
+
+func (m *pageMap) addPagesFromGoTmplFi(fi hugofs.FileMetaInfo, buildConfig *BuildCfg) (pageCount uint64, resourceCount uint64, addErr error) {
+ meta := fi.Meta()
+ pi := meta.PathInfo
+
+ m.s.Log.Trace(logg.StringFunc(
+ func() string {
+ return fmt.Sprintf("insert pages from data file: %q", fi.Meta().Filename)
+ },
+ ))
+
+ if !files.IsGoTmplExt(pi.Ext()) {
+ addErr = fmt.Errorf("unsupported data file extension %q", pi.Ext())
+ return
+ }
+
+ s := m.s.h.resolveSite(fi.Meta().Lang)
+ f := source.NewFileInfo(fi)
+ h := s.h
+
+ // Make sure the layouts are initialized.
+ if _, err := h.init.layouts.Do(context.Background()); err != nil {
+ addErr = err
+ return
+ }
+
+ contentAdapter := s.pageMap.treePagesFromTemplateAdapters.Get(pi.Base())
+ var rebuild bool
+ if contentAdapter != nil {
+ // Rebuild
+ contentAdapter = contentAdapter.CloneForGoTmpl(fi)
+ rebuild = true
+ } else {
+ contentAdapter = pagesfromdata.NewPagesFromTemplate(
+ pagesfromdata.PagesFromTemplateOptions{
+ GoTmplFi: fi,
+ Site: s,
+ DepsFromSite: func(s page.Site) pagesfromdata.PagesFromTemplateDeps {
+ ss := s.(*Site)
+ return pagesfromdata.PagesFromTemplateDeps{
+ TmplFinder: ss.TextTmpl(),
+ TmplExec: ss.Tmpl(),
+ }
+ },
+ DependencyManager: s.Conf.NewIdentityManager("pagesfromdata"),
+ Watching: s.Conf.Watching(),
+ HandlePage: func(pt *pagesfromdata.PagesFromTemplate, pc *pagemeta.PageConfig) error {
+ s := pt.Site.(*Site)
+ if err := pc.Compile(pt.GoTmplFi.Meta().PathInfo.Base(), true, "", s.Log, s.conf.MediaTypes.Config); err != nil {
+ return err
+ }
+
+ ps, pi, err := h.newPage(
+ &pageMeta{
+ f: f,
+ s: s,
+ pageMetaParams: &pageMetaParams{
+ pageConfig: pc,
+ },
+ },
+ )
+ if err != nil {
+ return err
+ }
+
+ if ps == nil {
+ // Disabled page.
+ return nil
+ }
+
+ u, n, replaced := s.pageMap.treePages.InsertIntoValuesDimensionWithLock(pi.Base(), ps)
+
+ if h.isRebuild() {
+ if replaced {
+ pt.AddChange(n.GetIdentity())
+ } else {
+ pt.AddChange(u.GetIdentity())
+ }
+ }
+
+ return nil
+ },
+ HandleResource: func(pt *pagesfromdata.PagesFromTemplate, rc *pagemeta.ResourceConfig) error {
+ s := pt.Site.(*Site)
+ if err := rc.Compile(
+ pt.GoTmplFi.Meta().PathInfo.Base(),
+ s.Conf.PathParser(),
+ s.conf.MediaTypes.Config,
+ ); err != nil {
+ return err
+ }
+
+ rs := &resourceSource{path: rc.PathInfo, rc: rc, opener: nil, fi: nil, langIndex: s.languagei}
+
+ _, n, replaced := s.pageMap.treeResources.InsertIntoValuesDimensionWithLock(rc.PathInfo.Base(), rs)
+
+ if h.isRebuild() && replaced {
+ pt.AddChange(n.GetIdentity())
+ }
+ return nil
+ },
+ },
+ )
+
+ s.pageMap.treePagesFromTemplateAdapters.Insert(pi.Base(), contentAdapter)
+
+ }
+
+ handleBuildInfo := func(s *Site, bi pagesfromdata.BuildInfo) {
+ resourceCount += bi.NumResourcesAdded
+ pageCount += bi.NumPagesAdded
+ s.handleContentAdapterChanges(bi, buildConfig)
+ }
+
+ bi, err := contentAdapter.Execute(context.Background())
+ if err != nil {
+ addErr = err
+ return
+ }
+ handleBuildInfo(s, bi)
+
+ if !rebuild && bi.EnableAllLanguages {
+ // Clone and insert the adapter for the other sites.
+ for _, ss := range s.h.Sites {
+ if s == ss {
+ continue
+ }
+
+ clone := contentAdapter.CloneForSite(ss)
+
+ // Make sure it gets executed for the first time.
+ bi, err := clone.Execute(context.Background())
+ if err != nil {
+ addErr = err
+ return
+ }
+ handleBuildInfo(ss, bi)
+
+ // Insert into the correct language tree so it get rebuilt on changes.
+ ss.pageMap.treePagesFromTemplateAdapters.Insert(pi.Base(), clone)
+
+ }
+ }
+
+ return
}
// The home page is represented with the zero string.
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/hugofs/files"
"github.com/gohugoio/hugo/hugolib/doctree"
+ "github.com/gohugoio/hugo/hugolib/pagesfromdata"
"github.com/gohugoio/hugo/identity"
+ "github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/output"
"github.com/gohugoio/hugo/resources"
"github.com/spf13/cast"
cacheContentPlain *dynacache.Partition[string, *resources.StaleValue[contentPlainPlainWords]]
contentTableOfContents *dynacache.Partition[string, *resources.StaleValue[contentTableOfContents]]
+ contentDataFileSeenItems *maps.Cache[string, map[uint64]bool]
+
cfg contentMapConfig
}
// This tree contains all taxonomy entries, e.g "/tags/blue/page1"
treeTaxonomyEntries *doctree.TreeShiftTree[*weightedContentNode]
+ // Stores the state for _content.gotmpl files.
+ // Mostly releveant for rebuilds.
+ treePagesFromTemplateAdapters *doctree.TreeShiftTree[*pagesfromdata.PagesFromTemplate]
+
// A slice of the resource trees.
resourceTrees doctree.MutableTrees
}
t.treePages = t.treePages.Shape(d, v)
t.treeResources = t.treeResources.Shape(d, v)
t.treeTaxonomyEntries = t.treeTaxonomyEntries.Shape(d, v)
+ t.treePagesFromTemplateAdapters = t.treePagesFromTemplateAdapters.Shape(d, v)
t.createMutableTrees()
return &t
sort.SliceStable(res, lessFunc)
- if len(ps.m.pageConfig.Resources) > 0 {
+ if len(ps.m.pageConfig.ResourcesMeta) > 0 {
for i, r := range res {
- res[i] = resources.CloneWithMetadataIfNeeded(ps.m.pageConfig.Resources, r)
+ res[i] = resources.CloneWithMetadataFromMapIfNeeded(ps.m.pageConfig.ResourcesMeta, r)
}
sort.SliceStable(res, lessFunc)
}
numLanguages int
}
-func (s *contentNodeShifter) Delete(n contentNodeI, dimension doctree.Dimension) (bool, bool) {
+func (s *contentNodeShifter) Delete(n contentNodeI, dimension doctree.Dimension) (contentNodeI, bool, bool) {
lidx := dimension[0]
switch v := n.(type) {
case contentNodeIs:
- resource.MarkStale(v[lidx])
- wasDeleted := v[lidx] != nil
+ deleted := v[lidx]
+ resource.MarkStale(deleted)
+ wasDeleted := deleted != nil
v[lidx] = nil
isEmpty := true
for _, vv := range v {
break
}
}
- return wasDeleted, isEmpty
+ return deleted, wasDeleted, isEmpty
case resourceSources:
- resource.MarkStale(v[lidx])
- wasDeleted := v[lidx] != nil
+ deleted := v[lidx]
+ resource.MarkStale(deleted)
+ wasDeleted := deleted != nil
v[lidx] = nil
isEmpty := true
for _, vv := range v {
break
}
}
- return wasDeleted, isEmpty
+ return deleted, wasDeleted, isEmpty
case *resourceSource:
if lidx != v.LangIndex() {
- return false, false
+ return nil, false, false
}
resource.MarkStale(v)
- return true, true
+ return v, true, true
case *pageState:
if lidx != v.s.languagei {
- return false, false
+ return nil, false, false
}
resource.MarkStale(v)
- return true, true
+ return v, true, true
default:
panic(fmt.Sprintf("unknown type %T", n))
}
}
}
-func (s *contentNodeShifter) InsertInto(old, new contentNodeI, dimension doctree.Dimension) contentNodeI {
+func (s *contentNodeShifter) InsertInto(old, new contentNodeI, dimension doctree.Dimension) (contentNodeI, contentNodeI, bool) {
langi := dimension[doctree.DimensionLanguage.Index()]
switch vv := old.(type) {
case *pageState:
panic(fmt.Sprintf("unknown type %T", new))
}
if vv.s.languagei == newp.s.languagei && newp.s.languagei == langi {
- return new
+ return new, vv, true
}
is := make(contentNodeIs, s.numLanguages)
is[vv.s.languagei] = old
is[langi] = new
- return is
+ return is, old, false
case contentNodeIs:
+ oldv := vv[langi]
vv[langi] = new
- return vv
+ return vv, oldv, oldv != nil
case resourceSources:
+ oldv := vv[langi]
vv[langi] = new.(*resourceSource)
- return vv
+ return vv, oldv, oldv != nil
case *resourceSource:
newp, ok := new.(*resourceSource)
if !ok {
panic(fmt.Sprintf("unknown type %T", new))
}
if vv.LangIndex() == newp.LangIndex() && newp.LangIndex() == langi {
- return new
+ return new, vv, true
}
rs := make(resourceSources, s.numLanguages)
rs[vv.LangIndex()] = vv
rs[langi] = newp
- return rs
+ return rs, vv, false
default:
panic(fmt.Sprintf("unknown type %T", old))
}
}
-func (s *contentNodeShifter) Insert(old, new contentNodeI) contentNodeI {
+func (s *contentNodeShifter) Insert(old, new contentNodeI) (contentNodeI, contentNodeI, bool) {
switch vv := old.(type) {
case *pageState:
newp, ok := new.(*pageState)
if newp != old {
resource.MarkStale(old)
}
- return new
+ return new, vv, true
}
is := make(contentNodeIs, s.numLanguages)
is[newp.s.languagei] = new
is[vv.s.languagei] = old
- return is
+ return is, old, false
case contentNodeIs:
newp, ok := new.(*pageState)
if !ok {
resource.MarkStale(oldp)
}
vv[newp.s.languagei] = new
- return vv
+ return vv, oldp, oldp != nil
case *resourceSource:
newp, ok := new.(*resourceSource)
if !ok {
if vv != newp {
resource.MarkStale(vv)
}
- return new
+ return new, vv, true
}
rs := make(resourceSources, s.numLanguages)
rs[newp.LangIndex()] = newp
rs[vv.LangIndex()] = vv
- return rs
+ return rs, vv, false
case resourceSources:
newp, ok := new.(*resourceSource)
if !ok {
resource.MarkStale(oldp)
}
vv[newp.LangIndex()] = newp
- return vv
+ return vv, oldp, oldp != nil
default:
panic(fmt.Sprintf("unknown type %T", old))
}
cacheContentPlain: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentPlainPlainWords]](mcache, fmt.Sprintf("/cont/pla/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}),
contentTableOfContents: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentTableOfContents]](mcache, fmt.Sprintf("/cont/toc/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}),
+ contentDataFileSeenItems: maps.NewCache[string, map[uint64]bool](),
+
cfg: contentMapConfig{
lang: s.Lang(),
taxonomyConfig: taxonomiesConfig.Values(),
type sitePagesAssembler struct {
*Site
- watching bool
- incomingChanges *whatChanged
assembleChanges *whatChanged
ctx context.Context
}
// 1. Handle the cache busters first, as those may produce identities for the page reset step.
// 2. Then reset the page outputs, which may mark some resources as stale.
// 3. Then GC the cache.
+ // TOOD1
if cachebuster != nil {
if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) {
ll := l.WithField("substep", "gc dynacache cachebuster")
}
changes = changes[:n]
+ if h.pageTrees.treePagesFromTemplateAdapters.LenRaw() > 0 {
+ if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) {
+ ll := l.WithField("substep", "resolve content adapter change set").WithField("changes", len(changes))
+ checkedCount := 0
+ matchCount := 0
+ depsFinder := identity.NewFinder(identity.FinderConfig{})
+
+ h.pageTrees.treePagesFromTemplateAdapters.WalkPrefixRaw(doctree.LockTypeRead, "",
+ func(s string, n *pagesfromdata.PagesFromTemplate) (bool, error) {
+ for _, id := range changes {
+ checkedCount++
+ if r := depsFinder.Contains(id, n.DependencyManager, 2); r > identity.FinderNotFound {
+ n.MarkStale()
+ matchCount++
+ break
+ }
+ }
+ return false, nil
+ })
+
+ ll = ll.WithField("checked", checkedCount).WithField("matches", matchCount)
+ return ll, nil
+ }); err != nil {
+ return err
+ }
+ }
+
if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) {
// changesLeft: The IDs that the pages is dependent on.
// changesRight: The IDs that the pages depend on.
rw := pw.Extend()
rw.Tree = sa.pageMap.treeResources
sa.lastmod = time.Time{}
+ rebuild := sa.s.h.isRebuild()
pw.Handle = func(keyPage string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
pageBundle := n.(*pageState)
// Home page gets it's cascade from the site config.
cascade = sa.conf.Cascade.Config
- if pageBundle.m.pageConfig.Cascade == nil {
+ if pageBundle.m.pageConfig.CascadeCompiled == nil {
// Pass the site cascade downwards.
pw.WalkContext.Data().Insert(keyPage, cascade)
}
}
}
- if (pageBundle.IsHome() || pageBundle.IsSection()) && pageBundle.m.setMetaPostCount > 0 {
- oldDates := pageBundle.m.pageConfig.Dates
+ if rebuild {
+ if (pageBundle.IsHome() || pageBundle.IsSection()) && pageBundle.m.setMetaPostCount > 0 {
+ oldDates := pageBundle.m.pageConfig.Dates
- // We need to wait until after the walk to determine if any of the dates have changed.
- pw.WalkContext.AddPostHook(
- func() error {
- if oldDates != pageBundle.m.pageConfig.Dates {
- sa.assembleChanges.Add(pageBundle)
- }
- return nil
- },
- )
+ // We need to wait until after the walk to determine if any of the dates have changed.
+ pw.WalkContext.AddPostHook(
+ func() error {
+ if oldDates != pageBundle.m.pageConfig.Dates {
+ sa.assembleChanges.Add(pageBundle)
+ }
+ return nil
+ },
+ )
+ }
}
// Combine the cascade map with front matter.
// We receive cascade values from above. If this leads to a change compared
// to the previous value, we need to mark the page and its dependencies as changed.
- if pageBundle.m.setMetaPostCascadeChanged {
+ if rebuild && pageBundle.m.setMetaPostCascadeChanged {
sa.assembleChanges.Add(pageBundle)
}
const eventName = "dates"
if n.isContentNodeBranch() {
- if pageBundle.m.pageConfig.Cascade != nil {
+ if pageBundle.m.pageConfig.CascadeCompiled != nil {
// Pass it down.
- pw.WalkContext.Data().Insert(keyPage, pageBundle.m.pageConfig.Cascade)
+ pw.WalkContext.Data().Insert(keyPage, pageBundle.m.pageConfig.CascadeCompiled)
}
wasZeroDates := pageBundle.m.pageConfig.Dates.IsAllDatesZero()
p := n.(*pageState)
if p.Kind() != kinds.KindTerm {
// The other kinds were handled in applyAggregates.
- if p.m.pageConfig.Cascade != nil {
+ if p.m.pageConfig.CascadeCompiled != nil {
// Pass it down.
- pw.WalkContext.Data().Insert(s, p.m.pageConfig.Cascade)
+ pw.WalkContext.Data().Insert(s, p.m.pageConfig.CascadeCompiled)
}
}
singular: viewName.singular,
s: sa.Site,
pathInfo: pi,
- pageMetaParams: pageMetaParams{
+ pageMetaParams: &pageMetaParams{
pageConfig: &pagemeta.PageConfig{
Kind: kinds.KindTerm,
},
targetPaths := ps.targetPaths()
baseTarget := targetPaths.SubResourceBaseTarget
duplicateResourceFiles := true
- if ps.m.pageConfig.IsGoldmark {
+ if ps.m.pageConfig.ContentMediaType.IsMarkdown() {
duplicateResourceFiles = ps.s.ContentSpec.Converters.GetMarkupConfig().Goldmark.DuplicateResourceFiles
}
duplicateResourceFiles = duplicateResourceFiles || ps.s.Conf.IsMultihost()
- sa.pageMap.forEachResourceInPage(
+ err := sa.pageMap.forEachResourceInPage(
ps, lockType,
!duplicateResourceFiles,
func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
}
+ if rs.rc != nil && rs.rc.Content.IsResourceValue() {
+ if rs.rc.Name == "" {
+ rs.rc.Name = relPathOriginal
+ }
+ r, err := ps.m.s.ResourceSpec.NewResourceWrapperFromResourceConfig(rs.rc)
+ if err != nil {
+ return false, err
+ }
+ rs.r = r
+ return false, nil
+ }
+
+ var mt media.Type
+ if rs.rc != nil {
+ mt = rs.rc.ContentMediaType
+ }
+
rd := resources.ResourceSourceDescriptor{
OpenReadSeekCloser: rs.opener,
Path: rs.path,
BasePathTargetPath: baseTarget,
NameNormalized: relPath,
NameOriginal: relPathOriginal,
+ MediaType: mt,
LazyPublish: !ps.m.pageConfig.Build.PublishResources,
}
+
+ if rs.rc != nil {
+ rc := rs.rc
+ rd.OpenReadSeekCloser = rc.Content.ValueAsOpenReadSeekCloser()
+ if rc.Name != "" {
+ rd.NameNormalized = rc.Name
+ rd.NameOriginal = rc.Name
+ }
+ if rc.Title != "" {
+ rd.Title = rc.Title
+ }
+ rd.Params = rc.Params
+ }
+
r, err := ps.m.s.ResourceSpec.NewResource(rd)
if err != nil {
return false, err
},
)
- return false, nil
+ return false, err
},
}
m := &pageMeta{
s: s,
pathInfo: s.Conf.PathParser().Parse(files.ComponentFolderContent, key+f.MediaType.FirstSuffix.FullSuffix),
- pageMetaParams: pageMetaParams{
+ pageMetaParams: &pageMetaParams{
pageConfig: &pagemeta.PageConfig{
Kind: kind,
},
m := &pageMeta{
s: sa.Site,
pathInfo: p,
- pageMetaParams: pageMetaParams{
+ pageMetaParams: &pageMetaParams{
pageConfig: &pagemeta.PageConfig{
Kind: kinds.KindHome,
},
if err != nil {
return err
}
- w.Tree.InsertWithLock(p.Base(), n)
+ w.Tree.InsertIntoValuesDimensionWithLock(p.Base(), n)
sa.home = n
}
m := &pageMeta{
s: sa.Site,
pathInfo: sa.Conf.PathParser().Parse(files.ComponentFolderContent, key+"/_index.md"),
- pageMetaParams: pageMetaParams{
+ pageMetaParams: &pageMetaParams{
pageConfig: &pagemeta.PageConfig{
Kind: kinds.KindTaxonomy,
},
c.Assert(tree.Get("/notfound"), qt.IsNil)
ab2 := &testValue{ID: "/a/b", Lang: 0}
- v, ok := tree.InsertIntoValuesDimension("/a/b", ab2)
+ v, _, ok := tree.InsertIntoValuesDimension("/a/b", ab2)
c.Assert(ok, qt.IsTrue)
c.Assert(v, qt.DeepEquals, ab2)
f(n)
}
-func (s *testShifter) Insert(old, new *testValue) *testValue {
- return new
+func (s *testShifter) Insert(old, new *testValue) (*testValue, *testValue, bool) {
+ return new, old, true
}
-func (s *testShifter) InsertInto(old, new *testValue, dimension doctree.Dimension) *testValue {
- return new
+func (s *testShifter) InsertInto(old, new *testValue, dimension doctree.Dimension) (*testValue, *testValue, bool) {
+ return new, old, true
}
-func (s *testShifter) Delete(n *testValue, dimension doctree.Dimension) (bool, bool) {
- return true, true
+func (s *testShifter) Delete(n *testValue, dimension doctree.Dimension) (*testValue, bool, bool) {
+ return nil, true, true
}
func (s *testShifter) Shift(n *testValue, dimension doctree.Dimension, exact bool) (*testValue, bool, doctree.DimensionFlag) {
// Insert inserts new into the tree into the dimension it provides.
// It may replace old.
- // It returns a T (can be the same as old).
- Insert(old, new T) T
+ // It returns the updated and existing T
+ // and a bool indicating if an existing record is updated.
+ Insert(old, new T) (T, T, bool)
// Insert inserts new into the given dimension.
// It may replace old.
- // It returns a T (can be the same as old).
- InsertInto(old, new T, dimension Dimension) T
+ // It returns the updated and existing T
+ // and a bool indicating if an existing record is updated.
+ InsertInto(old, new T, dimension Dimension) (T, T, bool)
- // Delete deletes T from the given dimension and returns whether the dimension was deleted and if it's empty after the delete.
- Delete(v T, dimension Dimension) (bool, bool)
+ // Delete deletes T from the given dimension and returns the deleted T and whether the dimension was deleted and if it's empty after the delete.
+ Delete(v T, dimension Dimension) (T, bool, bool)
// Shift shifts T into the given dimension
// and returns the shifted T and a bool indicating if the shift was successful and
}
}
-func (r *NodeShiftTree[T]) Delete(key string) {
+func (r *NodeShiftTree[T]) Delete(key string) (T, bool) {
+ return r.delete(key)
+}
+
+func (r *NodeShiftTree[T]) DeleteRaw(key string) {
r.delete(key)
}
return false
})
for _, key := range keys {
- if ok := r.delete(key); ok {
+ if _, ok := r.delete(key); ok {
count++
}
}
return count
}
-func (r *NodeShiftTree[T]) delete(key string) bool {
+func (r *NodeShiftTree[T]) delete(key string) (T, bool) {
var wasDeleted bool
+ var deleted T
if v, ok := r.tree.Get(key); ok {
var isEmpty bool
- wasDeleted, isEmpty = r.shifter.Delete(v.(T), r.dims)
+ deleted, wasDeleted, isEmpty = r.shifter.Delete(v.(T), r.dims)
if isEmpty {
r.tree.Delete(key)
}
}
- return wasDeleted
+ return deleted, wasDeleted
}
func (t *NodeShiftTree[T]) DeletePrefixAll(prefix string) int {
return t.Shape(d, t.dims[d]+1)
}
-func (r *NodeShiftTree[T]) InsertIntoCurrentDimension(s string, v T) (T, bool) {
+func (r *NodeShiftTree[T]) InsertIntoCurrentDimension(s string, v T) (T, T, bool) {
s = mustValidateKey(cleanKey(s))
+ var (
+ updated bool
+ existing T
+ )
if vv, ok := r.tree.Get(s); ok {
- v = r.shifter.InsertInto(vv.(T), v, r.dims)
+ v, existing, updated = r.shifter.InsertInto(vv.(T), v, r.dims)
}
r.tree.Insert(s, v)
- return v, true
+ return v, existing, updated
}
-func (r *NodeShiftTree[T]) InsertIntoValuesDimension(s string, v T) (T, bool) {
+// InsertIntoValuesDimension inserts v into the tree at the given key and the
+// dimension defined by the value.
+// It returns the updated and existing T and a bool indicating if an existing record is updated.
+func (r *NodeShiftTree[T]) InsertIntoValuesDimension(s string, v T) (T, T, bool) {
s = mustValidateKey(cleanKey(s))
+ var (
+ updated bool
+ existing T
+ )
if vv, ok := r.tree.Get(s); ok {
- v = r.shifter.Insert(vv.(T), v)
+ v, existing, updated = r.shifter.Insert(vv.(T), v)
}
r.tree.Insert(s, v)
- return v, true
+ return v, existing, updated
}
func (r *NodeShiftTree[T]) InsertRawWithLock(s string, v any) (any, bool) {
return r.tree.Insert(s, v)
}
-func (r *NodeShiftTree[T]) InsertWithLock(s string, v T) (T, bool) {
+// It returns the updated and existing T and a bool indicating if an existing record is updated.
+func (r *NodeShiftTree[T]) InsertIntoValuesDimensionWithLock(s string, v T) (T, T, bool) {
r.mu.Lock()
defer r.mu.Unlock()
return r.InsertIntoValuesDimension(s, v)
}
// NewSimpleTree creates a new SimpleTree.
-func NewSimpleTree[T any]() *SimpleTree[T] {
+func NewSimpleTree[T comparable]() *SimpleTree[T] {
return &SimpleTree[T]{tree: radix.New()}
}
// SimpleTree is a thread safe radix tree that holds T.
-type SimpleTree[T any] struct {
+type SimpleTree[T comparable] struct {
mu sync.RWMutex
tree *radix.Tree
zero T
return v
}
-func (tree *SimpleTree[T]) WalkPrefix(lockType LockType, s string, f func(s string, v T) (bool, error)) error {
+func (tree *SimpleTree[T]) Lock(lockType LockType) func() {
switch lockType {
case LockTypeNone:
+ return func() {}
case LockTypeRead:
tree.mu.RLock()
- defer tree.mu.RUnlock()
+ return tree.mu.RUnlock
case LockTypeWrite:
tree.mu.Lock()
- defer tree.mu.Unlock()
+ return tree.mu.Unlock
}
+ return func() {}
+}
+
+func (tree *SimpleTree[T]) WalkPrefix(lockType LockType, s string, f func(s string, v T) (bool, error)) error {
+ commit := tree.Lock(lockType)
+ defer commit()
var err error
tree.tree.WalkPrefix(s, func(s string, v any) bool {
var b bool
// MutableTree is a tree that can be modified.
type MutableTree interface {
- Delete(key string)
+ DeleteRaw(key string)
DeleteAll(key string)
DeletePrefix(prefix string) int
DeletePrefixAll(prefix string) int
type MutableTrees []MutableTree
-func (t MutableTrees) Delete(key string) {
+func (t MutableTrees) DeleteRaw(key string) {
for _, tree := range t {
- tree.Delete(key)
+ tree.DeleteRaw(key)
}
}
var _ Tree[string] = (*TreeShiftTree[string])(nil)
-type TreeShiftTree[T any] struct {
+type TreeShiftTree[T comparable] struct {
// This tree is shiftable in one dimension.
d int
// The value of the current dimension.
v int
+ // The zero value of T.
+ zero T
+
// Will be of length equal to the length of the dimension.
trees []*SimpleTree[T]
}
-func NewTreeShiftTree[T any](d, length int) *TreeShiftTree[T] {
+func NewTreeShiftTree[T comparable](d, length int) *TreeShiftTree[T] {
if length <= 0 {
panic("length must be > 0")
}
return t.trees[t.v].Get(s)
}
+func (t *TreeShiftTree[T]) DeleteAllFunc(s string, f func(s string, v T) bool) {
+ for _, tt := range t.trees {
+ if v := tt.Get(s); v != t.zero {
+ if f(s, v) {
+ // Delete.
+ tt.tree.Delete(s)
+ }
+ }
+ }
+}
+
func (t *TreeShiftTree[T]) LongestPrefix(s string) (string, T) {
return t.trees[t.v].LongestPrefix(s)
}
return t.trees[t.v].Insert(s, v)
}
+func (t *TreeShiftTree[T]) Lock(lockType LockType) func() {
+ return t.trees[t.v].Lock(lockType)
+}
+
func (t *TreeShiftTree[T]) WalkPrefix(lockType LockType, s string, f func(s string, v T) (bool, error)) error {
return t.trees[t.v].WalkPrefix(lockType, s, f)
}
-func (t *TreeShiftTree[T]) Delete(key string) {
+func (t *TreeShiftTree[T]) WalkPrefixRaw(lockType LockType, s string, f func(s string, v T) (bool, error)) error {
for _, tt := range t.trees {
- tt.tree.Delete(key)
+ if err := tt.WalkPrefix(lockType, s, f); err != nil {
+ return err
+ }
}
+ return nil
}
-func (t *TreeShiftTree[T]) DeletePrefix(prefix string) int {
+func (t *TreeShiftTree[T]) LenRaw() int {
var count int
for _, tt := range t.trees {
- count += tt.tree.DeletePrefix(prefix)
+ count += tt.tree.Len()
}
return count
}
-func (t *TreeShiftTree[T]) Lock(writable bool) (commit func()) {
- if writable {
- for _, tt := range t.trees {
- tt.mu.Lock()
- }
- return func() {
- for _, tt := range t.trees {
- tt.mu.Unlock()
- }
- }
+func (t *TreeShiftTree[T]) Delete(key string) {
+ for _, tt := range t.trees {
+ tt.tree.Delete(key)
}
+}
+func (t *TreeShiftTree[T]) DeletePrefix(prefix string) int {
+ var count int
for _, tt := range t.trees {
- tt.mu.RLock()
- }
- return func() {
- for _, tt := range t.trees {
- tt.mu.RUnlock()
- }
+ count += tt.tree.DeletePrefix(prefix)
}
+ return count
}
return h.skipRebuildForFilenames[ev.Name]
}
+func (h *HugoSites) isRebuild() bool {
+ return h.buildCounter.Load() > 0
+}
+
+func (h *HugoSites) resolveSite(lang string) *Site {
+ if lang == "" {
+ lang = h.Conf.DefaultContentLanguage()
+ }
+
+ for _, s := range h.Sites {
+ if s.Lang() == lang {
+ return s
+ }
+ }
+
+ return nil
+}
+
// Only used in tests.
type buildCounters struct {
contentRenderCounter atomic.Uint64
hugofs.WalkwayConfig{
Fs: h.PathSpec.BaseFs.Data.Fs,
IgnoreFile: h.SourceSpec.IgnoreFile,
+ PathParser: h.Conf.PathParser(),
WalkFn: func(path string, fi hugofs.FileMetaInfo) error {
if fi.IsDir() {
return nil
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/hugofs/files"
"github.com/gohugoio/hugo/hugofs/glob"
+ "github.com/gohugoio/hugo/hugolib/doctree"
+ "github.com/gohugoio/hugo/hugolib/pagesfromdata"
"github.com/gohugoio/hugo/hugolib/segments"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/output"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/para"
"github.com/gohugoio/hugo/common/paths"
+ "github.com/gohugoio/hugo/common/rungroup"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/resources/page/siteidentities"
close(to)
}(errCollector, errs)
+ for _, s := range h.Sites {
+ s.state = siteStateInit
+ }
+
if h.Metrics != nil {
h.Metrics.Reset()
}
conf := &config
if conf.whatChanged == nil {
// Assume everything has changed
- conf.whatChanged = &whatChanged{contentChanged: true}
+ conf.whatChanged = &whatChanged{needsPagesAssembly: true}
}
var prepareErr error
}
}
+ for _, s := range h.Sites {
+ s.state = siteStateReady
+ }
+
if prepareErr == nil {
if err := h.render(infol, conf); err != nil {
h.SendError(fmt.Errorf("render: %w", err))
})
for _, s := range h.Sites {
- s.resetBuildState(config.whatChanged.contentChanged)
+ s.resetBuildState(config.whatChanged.needsPagesAssembly)
}
h.reset(config)
// This is a rebuild
return h.processPartial(ctx, l, config, init, events)
}
- return h.processFull(ctx, l, *config)
+ return h.processFull(ctx, l, config)
}
// assemble creates missing sections, applies aggregate values (e.g. dates, cascading params),
l = l.WithField("step", "assemble")
defer loggers.TimeTrackf(l, time.Now(), nil, "")
- if !bcfg.whatChanged.contentChanged {
+ if !bcfg.whatChanged.needsPagesAssembly {
+ changes := bcfg.whatChanged.Drain()
+ if len(changes) > 0 {
+ if err := h.resolveAndClearStateForIdentities(ctx, l, nil, changes); err != nil {
+ return err
+ }
+ }
return nil
}
h.translationKeyPages.Reset()
assemblers := make([]*sitePagesAssembler, len(h.Sites))
// Changes detected during assembly (e.g. aggregate date changes)
- assembleChanges := &whatChanged{
- identitySet: make(map[identity.Identity]bool),
- }
+
for i, s := range h.Sites {
assemblers[i] = &sitePagesAssembler{
Site: s,
- watching: s.watching(),
- incomingChanges: bcfg.whatChanged,
- assembleChanges: assembleChanges,
+ assembleChanges: bcfg.whatChanged,
ctx: ctx,
}
}
return err
}
- changes := assembleChanges.Changes()
+ changes := bcfg.whatChanged.Drain()
// Changes from the assemble step (e.g. lastMod, cascade) needs a re-calculation
// of what needs to be re-built.
logger := h.Log
var (
- tmplAdded bool
- tmplChanged bool
- i18nChanged bool
- contentChanged bool
+ tmplAdded bool
+ tmplChanged bool
+ i18nChanged bool
+ needsPagesAssemble bool
)
changedPaths := struct {
switch pathInfo.Component() {
case files.ComponentFolderContent:
logger.Println("Source changed", pathInfo.Path())
- if ids := h.pageTrees.collectAndMarkStaleIdentities(pathInfo); len(ids) > 0 {
- changes = append(changes, ids...)
+ isContentDataFile := pathInfo.IsContentData()
+ if !isContentDataFile {
+ if ids := h.pageTrees.collectAndMarkStaleIdentities(pathInfo); len(ids) > 0 {
+ changes = append(changes, ids...)
+ }
+ } else {
+ h.pageTrees.treePagesFromTemplateAdapters.DeleteAllFunc(pathInfo.Base(),
+ func(s string, n *pagesfromdata.PagesFromTemplate) bool {
+ changes = append(changes, n.DependencyManager)
+
+ // Try to open the file to see if has been deleted.
+ f, err := n.GoTmplFi.Meta().Open()
+ if err == nil {
+ f.Close()
+ }
+ if err != nil {
+ // Remove all pages and resources below.
+ prefix := pathInfo.Base() + "/"
+ h.pageTrees.treePages.DeletePrefixAll(prefix)
+ h.pageTrees.resourceTrees.DeletePrefixAll(prefix)
+ changes = append(changes, identity.NewGlobIdentity(prefix+"*"))
+ }
+ return err != nil
+ })
}
- contentChanged = true
+ needsPagesAssemble = true
if config.RecentlyVisited != nil {
// Fast render mode. Adding them to the visited queue
h.pageTrees.treeTaxonomyEntries.DeletePrefix("")
- if delete {
+ if delete && !isContentDataFile {
_, ok := h.pageTrees.treePages.LongestPrefixAll(pathInfo.Base())
if ok {
h.pageTrees.treePages.DeleteAll(pathInfo.Base())
resourceFiles := h.fileEventsContentPaths(addedOrChangedContent)
changed := &whatChanged{
- contentChanged: contentChanged,
- identitySet: make(identity.Identities),
+ needsPagesAssembly: needsPagesAssemble,
+ identitySet: make(identity.Identities),
}
changed.Add(changes...)
}
}
- // Removes duplicates.
- changes = changed.identitySet.AsSlice()
-
- if err := h.resolveAndClearStateForIdentities(ctx, l, cacheBusterOr, changes); err != nil {
+ if err := h.resolveAndClearStateForIdentities(ctx, l, cacheBusterOr, changed.Drain()); err != nil {
return err
}
}
if resourceFiles != nil {
- if err := h.processFiles(ctx, l, *config, resourceFiles...); err != nil {
+ if err := h.processFiles(ctx, l, config, resourceFiles...); err != nil {
+ return err
+ }
+ }
+
+ if h.isRebuild() {
+ if err := h.processContentAdaptersOnRebuild(ctx, config); err != nil {
return err
}
}
}
}
-func (h *HugoSites) processFull(ctx context.Context, l logg.LevelLogger, config BuildCfg) (err error) {
+func (h *HugoSites) processFull(ctx context.Context, l logg.LevelLogger, config *BuildCfg) (err error) {
if err = h.processFiles(ctx, l, config); err != nil {
err = fmt.Errorf("readAndProcessContent: %w", err)
return
return err
}
-func (s *HugoSites) processFiles(ctx context.Context, l logg.LevelLogger, buildConfig BuildCfg, filenames ...pathChange) error {
+func (s *Site) handleContentAdapterChanges(bi pagesfromdata.BuildInfo, buildConfig *BuildCfg) {
+ if !s.h.isRebuild() {
+ return
+ }
+
+ if len(bi.ChangedIdentities) > 0 {
+ buildConfig.whatChanged.Add(bi.ChangedIdentities...)
+ buildConfig.whatChanged.needsPagesAssembly = true
+ }
+
+ for _, p := range bi.DeletedPaths {
+ pp := path.Join(bi.Path.Base(), p)
+ if v, ok := s.pageMap.treePages.Delete(pp); ok {
+ buildConfig.whatChanged.Add(v.GetIdentity())
+ }
+ }
+}
+
+func (h *HugoSites) processContentAdaptersOnRebuild(ctx context.Context, buildConfig *BuildCfg) error {
+ g := rungroup.Run[*pagesfromdata.PagesFromTemplate](ctx, rungroup.Config[*pagesfromdata.PagesFromTemplate]{
+ NumWorkers: h.numWorkers,
+ Handle: func(ctx context.Context, p *pagesfromdata.PagesFromTemplate) error {
+ bi, err := p.Execute(ctx)
+ if err != nil {
+ return err
+ }
+ s := p.Site.(*Site)
+ s.handleContentAdapterChanges(bi, buildConfig)
+ return nil
+ },
+ })
+
+ h.pageTrees.treePagesFromTemplateAdapters.WalkPrefixRaw(doctree.LockTypeRead, "", func(key string, p *pagesfromdata.PagesFromTemplate) (bool, error) {
+ if p.StaleVersion() > 0 {
+ g.Enqueue(p)
+ }
+ return false, nil
+ })
+
+ return g.Wait()
+}
+
+func (s *HugoSites) processFiles(ctx context.Context, l logg.LevelLogger, buildConfig *BuildCfg, filenames ...pathChange) error {
if s.Deps == nil {
panic("nil deps on site")
}
// For inserts, we can pick an arbitrary pageMap.
pageMap := s.Sites[0].pageMap
- c := newPagesCollector(ctx, s.h, sourceSpec, s.Log, l, pageMap, filenames)
+ c := newPagesCollector(ctx, s.h, sourceSpec, s.Log, l, pageMap, buildConfig, filenames)
if err := c.Collect(); err != nil {
return err
IntegrationTestConfig{
T: t,
TxtarString: files,
- LogLevel: logg.LevelTrace,
+ // LogLevel: logg.LevelTrace,
},
).Build()
continue
}
+ if _, isWrapper := r.(resource.ResourceWrapper); isWrapper {
+ // Skip resources that are wrapped.
+ // These gets published on its own.
+ continue
+ }
+
src, ok := r.(resource.Source)
if !ok {
- initErr = fmt.Errorf("resource %T does not support resource.Source", src)
+ initErr = fmt.Errorf("resource %T does not support resource.Source", r)
return
}
func (p *pageState) getContentConverter() converter.Converter {
var err error
p.contentConverterInit.Do(func() {
- markup := p.m.pageConfig.Markup
+ if p.m.pageConfig.ContentMediaType.IsZero() {
+ panic("ContentMediaType not set")
+ }
+ markup := p.m.pageConfig.ContentMediaType.SubType
+
if markup == "html" {
// Only used for shortcode inner content.
markup = "markdown"
"fmt"
"html/template"
"io"
+ "path/filepath"
"strconv"
"strings"
"unicode/utf8"
source pageparser.Item
}
-func (m *pageMeta) parseFrontMatter(h *HugoSites, pid uint64, sourceKey string) (*contentParseInfo, error) {
- var openSource hugio.OpenReadSeekCloser
- if m.f != nil {
- meta := m.f.FileInfo().Meta()
- openSource = func() (hugio.ReadSeekCloser, error) {
- r, err := meta.Open()
- if err != nil {
- return nil, fmt.Errorf("failed to open file %q: %w", meta.Filename, err)
+func (m *pageMeta) parseFrontMatter(h *HugoSites, pid uint64) (*contentParseInfo, error) {
+ var (
+ sourceKey string
+ openSource hugio.OpenReadSeekCloser
+ hasContent = m.pageConfig.IsFromContentAdapter
+ )
+
+ if m.f != nil && !hasContent {
+ sourceKey = filepath.ToSlash(m.f.Filename())
+ if !hasContent {
+ meta := m.f.FileInfo().Meta()
+ openSource = func() (hugio.ReadSeekCloser, error) {
+ r, err := meta.Open()
+ if err != nil {
+ return nil, fmt.Errorf("failed to open file %q: %w", meta.Filename, err)
+ }
+ return r, nil
}
- return r, nil
}
+ } else if hasContent {
+ openSource = m.pageConfig.Content.ValueAsOpenReadSeekCloser()
}
if sourceKey == "" {
- sourceKey = strconv.Itoa(int(pid))
+ sourceKey = strconv.FormatUint(pid, 10)
}
pi := &contentParseInfo{
pi.itemsStep1 = items
+ if hasContent {
+ // No front matter.
+ return pi, nil
+ }
+
if err := pi.mapFrontMatter(source); err != nil {
return nil, err
}
var result contentSummary // hasVariants bool
if c.pi.hasSummaryDivider {
- isHTML := cp.po.p.m.pageConfig.Markup == "html"
- if isHTML {
+ if cp.po.p.m.pageConfig.ContentMediaType.IsHTML() {
// Use the summary sections as provided by the user.
i := bytes.Index(b, internalSummaryDividerPre)
result.summary = helpers.BytesToHTML(b[:i])
b = b[i+len(internalSummaryDividerPre):]
} else {
- summary, content, err := splitUserDefinedSummaryAndContent(cp.po.p.m.pageConfig.Markup, b)
+ summary, content, err := splitUserDefinedSummaryAndContent(cp.po.p.m.pageConfig.Content.Markup, b)
if err != nil {
cp.po.p.s.Log.Errorf("Failed to set user defined summary for page %q: %s", cp.po.p.pathOrTitle(), err)
} else {
p.pageOutputTemplateVariationsState.Add(1)
}
- isHTML := cp.po.p.m.pageConfig.Markup == "html"
+ isHTML := cp.po.p.m.pageConfig.ContentMediaType.IsHTML()
if !isHTML {
createAndSetToC := func(tocProvider converter.TableOfContentsProvider) {
if err != nil {
return nil, err
}
- html := cp.po.p.s.ContentSpec.TrimShortHTML(b.Bytes(), cp.po.p.m.pageConfig.Markup)
+ html := cp.po.p.s.ContentSpec.TrimShortHTML(b.Bytes(), cp.po.p.m.pageConfig.Content.Markup)
result.summary = helpers.BytesToHTML(html)
} else {
var summary string
singular string // Set for kind == KindTerm and kind == KindTaxonomy.
resource.Staler
- pageMetaParams
+ *pageMetaParams
pageMetaFrontMatter
// Set for standalone pages, e.g. robotsTXT.
func (m *pageMetaParams) init(preserveOringal bool) {
if preserveOringal {
m.paramsOriginal = xmaps.Clone[maps.Params](m.pageConfig.Params)
- m.cascadeOriginal = xmaps.Clone[map[page.PageMatcher]maps.Params](m.pageConfig.Cascade)
+ m.cascadeOriginal = xmaps.Clone[map[page.PageMatcher]maps.Params](m.pageConfig.CascadeCompiled)
}
}
}
func (p *pageMeta) Date() time.Time {
- return p.pageConfig.Date
+ return p.pageConfig.Dates.Date
}
func (p *pageMeta) PublishDate() time.Time {
- return p.pageConfig.PublishDate
+ return p.pageConfig.Dates.PublishDate
}
func (p *pageMeta) Lastmod() time.Time {
- return p.pageConfig.Lastmod
+ return p.pageConfig.Dates.Lastmod
}
func (p *pageMeta) ExpiryDate() time.Time {
- return p.pageConfig.ExpiryDate
+ return p.pageConfig.Dates.ExpiryDate
}
func (p *pageMeta) Description() string {
if frontmatter != nil {
pcfg := p.pageConfig
- if pcfg == nil {
- panic("pageConfig not set")
- }
// Needed for case insensitive fetching of params values
maps.PrepareParams(frontmatter)
pcfg.Params = frontmatter
if err != nil {
return err
}
- pcfg.Cascade = cascade
+ pcfg.CascadeCompiled = cascade
}
// Look for path, lang and kind, all of which values we need early on.
ps.m.setMetaPostCount++
var cascadeHashPre uint64
if ps.m.setMetaPostCount > 1 {
- cascadeHashPre = identity.HashUint64(ps.m.pageConfig.Cascade)
- ps.m.pageConfig.Cascade = xmaps.Clone[map[page.PageMatcher]maps.Params](ps.m.cascadeOriginal)
+ cascadeHashPre = identity.HashUint64(ps.m.pageConfig.CascadeCompiled)
+ ps.m.pageConfig.CascadeCompiled = xmaps.Clone[map[page.PageMatcher]maps.Params](ps.m.cascadeOriginal)
}
// Apply cascades first so they can be overridden later.
if cascade != nil {
- if ps.m.pageConfig.Cascade != nil {
+ if ps.m.pageConfig.CascadeCompiled != nil {
for k, v := range cascade {
- vv, found := ps.m.pageConfig.Cascade[k]
+ vv, found := ps.m.pageConfig.CascadeCompiled[k]
if !found {
- ps.m.pageConfig.Cascade[k] = v
+ ps.m.pageConfig.CascadeCompiled[k] = v
} else {
// Merge
for ck, cv := range v {
}
}
}
- cascade = ps.m.pageConfig.Cascade
+ cascade = ps.m.pageConfig.CascadeCompiled
} else {
- ps.m.pageConfig.Cascade = cascade
+ ps.m.pageConfig.CascadeCompiled = cascade
}
}
if cascade == nil {
- cascade = ps.m.pageConfig.Cascade
+ cascade = ps.m.pageConfig.CascadeCompiled
}
if ps.m.setMetaPostCount > 1 {
- ps.m.setMetaPostCascadeChanged = cascadeHashPre != identity.HashUint64(ps.m.pageConfig.Cascade)
+ ps.m.setMetaPostCascadeChanged = cascadeHashPre != identity.HashUint64(ps.m.pageConfig.CascadeCompiled)
if !ps.m.setMetaPostCascadeChanged {
// No changes, restore any value that may be changed by aggregation.
pm := p.m
var mtime time.Time
var contentBaseName string
+ var ext string
+ var isContentAdapter bool
if p.File() != nil {
+ isContentAdapter = p.File().IsContentAdapter()
contentBaseName = p.File().ContentBaseName()
if p.File().FileInfo() != nil {
mtime = p.File().FileInfo().ModTime()
}
+ if !isContentAdapter {
+ ext = p.File().Ext()
+ }
}
var gitAuthorDate time.Time
p.s.Log.Errorf("Failed to handle dates for page %q: %s", p.pathOrTitle(), err)
}
+ if isContentAdapter {
+ // Done.
+ return nil
+ }
+
var buildConfig any
var isNewBuildKeyword bool
if v, ok := pm.pageConfig.Params["_build"]; ok {
var sitemapSet bool
pcfg := pm.pageConfig
-
params := pcfg.Params
+ if params == nil {
+ panic("params not set for " + p.Title())
+ }
var draft, published, isCJKLanguage *bool
var userParams map[string]any
pcfg.Layout = cast.ToString(v)
params[loki] = pcfg.Layout
case "markup":
- pcfg.Markup = cast.ToString(v)
- params[loki] = pcfg.Markup
+ pcfg.Content.Markup = cast.ToString(v)
+ params[loki] = pcfg.Content.Markup
case "weight":
pcfg.Weight = cast.ToInt(v)
params[loki] = pcfg.Weight
}
if handled {
- pcfg.Resources = resources
+ pcfg.ResourcesMeta = resources
break
}
fallthrough
pcfg.Sitemap = p.s.conf.Sitemap
}
- pcfg.Markup = p.s.ContentSpec.ResolveMarkup(pcfg.Markup)
-
if draft != nil && published != nil {
pcfg.Draft = *draft
p.m.s.Log.Warnf("page %q has both draft and published settings in its frontmatter. Using draft.", p.File().Filename())
params["iscjklanguage"] = pcfg.IsCJKLanguage
+ if err := pcfg.Validate(false); err != nil {
+ return err
+ }
+
+ if err := pcfg.Compile("", false, ext, p.s.Log, p.s.conf.MediaTypes.Config); err != nil {
+ return err
+ }
+
return nil
}
(&p.pageConfig.Build).Disable()
}
- if p.pageConfig.Markup == "" {
+ if p.pageConfig.Content.Markup == "" {
if p.File() != nil {
// Fall back to file extension
- p.pageConfig.Markup = p.s.ContentSpec.ResolveMarkup(p.File().Ext())
+ p.pageConfig.Content.Markup = p.s.ContentSpec.ResolveMarkup(p.File().Ext())
}
- if p.pageConfig.Markup == "" {
- p.pageConfig.Markup = "markdown"
+ if p.pageConfig.Content.Markup == "" {
+ p.pageConfig.Content.Markup = "markdown"
}
}
- p.pageConfig.IsGoldmark = p.s.ContentSpec.Converters.IsGoldmark(p.pageConfig.Markup)
-
if p.pageConfig.Title == "" && p.f == nil {
switch p.Kind() {
case kinds.KindHome:
import (
"fmt"
- "path/filepath"
"sync"
"sync/atomic"
func (h *HugoSites) newPage(m *pageMeta) (*pageState, *paths.Path, error) {
m.Staler = &resources.AtomicStaler{}
- if m.pageConfig == nil {
- m.pageMetaParams = pageMetaParams{
- pageConfig: &pagemeta.PageConfig{
- Params: maps.Params{},
- },
+ if m.pageMetaParams == nil {
+ m.pageMetaParams = &pageMetaParams{
+ pageConfig: &pagemeta.PageConfig{},
}
}
-
- var sourceKey string
- if m.f != nil {
- sourceKey = filepath.ToSlash(m.f.Filename())
+ if m.pageConfig.Params == nil {
+ m.pageConfig.Params = maps.Params{}
}
pid := pageIDCounter.Add(1)
- pi, err := m.parseFrontMatter(h, pid, sourceKey)
+ pi, err := m.parseFrontMatter(h, pid)
if err != nil {
return nil, nil, err
}
s := m.pageConfig.Path
if !paths.HasExt(s) {
var (
- isBranch bool
- ext string = "md"
+ isBranch bool
+ isBranchSet bool
+ ext string = m.pageConfig.ContentMediaType.FirstSuffix.Suffix
)
if pcfg.Kind != "" {
isBranch = kinds.IsBranch(pcfg.Kind)
- } else if m.pathInfo != nil {
- isBranch = m.pathInfo.IsBranchBundle()
- if m.pathInfo.Ext() != "" {
- ext = m.pathInfo.Ext()
- }
- } else if m.f != nil {
- pi := m.f.FileInfo().Meta().PathInfo
- isBranch = pi.IsBranchBundle()
- if pi.Ext() != "" {
- ext = pi.Ext()
+ isBranchSet = true
+ }
+
+ if !pcfg.IsFromContentAdapter {
+ if m.pathInfo != nil {
+ if !isBranchSet {
+ isBranch = m.pathInfo.IsBranchBundle()
+ }
+ if m.pathInfo.Ext() != "" {
+ ext = m.pathInfo.Ext()
+ }
+ } else if m.f != nil {
+ pi := m.f.FileInfo().Meta().PathInfo
+ if !isBranchSet {
+ isBranch = pi.IsBranchBundle()
+ }
+ if pi.Ext() != "" {
+ ext = pi.Ext()
+ }
}
}
+
if isBranch {
s += "/_index." + ext
} else {
s += "/index." + ext
}
+
}
m.pathInfo = h.Conf.PathParser().Parse(files.ComponentFolderContent, s)
} else if m.pathInfo == nil {
} else if m.f != nil {
meta := m.f.FileInfo().Meta()
lang = meta.Lang
- m.s = h.Sites[meta.LangIndex]
} else {
lang = m.pathInfo.Lang()
}
- if lang == "" {
- lang = h.Conf.DefaultContentLanguage()
- }
- var found bool
- for _, ss := range h.Sites {
- if ss.Lang() == lang {
- m.s = ss
- found = true
- break
- }
- }
- if !found {
+ m.s = h.resolveSite(lang)
+
+ if m.s == nil {
return nil, fmt.Errorf("no site found for language %q", lang)
}
}
"github.com/gohugoio/hugo/common/text"
"github.com/gohugoio/hugo/common/types/hstring"
"github.com/gohugoio/hugo/identity"
+ "github.com/gohugoio/hugo/markup"
+ "github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/parser/pageparser"
"github.com/mitchellh/mapstructure"
"github.com/spf13/cast"
if err := mapstructure.WeakDecode(m, &opts); err != nil {
return "", fmt.Errorf("failed to decode options: %w", err)
}
+ if opts.Markup != "" {
+ opts.Markup = markup.ResolveMarkup(opts.Markup)
+ }
}
contentToRenderv := args[sidx]
}
conv := pco.po.p.getContentConverter()
- if opts.Markup != "" && opts.Markup != pco.po.p.m.pageConfig.Markup {
+
+ if opts.Markup != "" && opts.Markup != pco.po.p.m.pageConfig.ContentMediaType.SubType {
var err error
conv, err = pco.po.p.m.newContentConverter(pco.po.p, opts.Markup)
if err != nil {
}
if opts.Display == "inline" {
- markup := pco.po.p.m.pageConfig.Markup
+ markup := pco.po.p.m.pageConfig.Content.Markup
if opts.Markup != "" {
markup = pco.po.p.s.ContentSpec.ResolveMarkup(opts.Markup)
}
startTag := "p"
switch markup {
- case "asciidocext":
+ case media.DefaultContentTypes.AsciiDoc.SubType:
startTag = "div"
}
logger loggers.Logger,
infoLogger logg.LevelLogger,
m *pageMap,
+ buildConfig *BuildCfg,
ids []pathChange,
) *pagesCollector {
return &pagesCollector{
- ctx: ctx,
- h: h,
- fs: sp.BaseFs.Content.Fs,
- m: m,
- sp: sp,
- logger: logger,
- infoLogger: infoLogger,
- ids: ids,
- seenDirs: make(map[string]bool),
+ ctx: ctx,
+ h: h,
+ fs: sp.BaseFs.Content.Fs,
+ m: m,
+ sp: sp,
+ logger: logger,
+ infoLogger: infoLogger,
+ buildConfig: buildConfig,
+ ids: ids,
+ seenDirs: make(map[string]bool),
}
}
fs afero.Fs
+ buildConfig *BuildCfg
+
// List of paths that have changed. Used in partial builds.
ids []pathChange
seenDirs map[string]bool
var (
numWorkers = c.h.numWorkers
numFilesProcessedTotal atomic.Uint64
+ numPagesProcessedTotal atomic.Uint64
+ numResourcesProcessed atomic.Uint64
numFilesProcessedLast uint64
fileBatchTimer = time.Now()
fileBatchTimerMu sync.Mutex
logg.Fields{
logg.Field{Name: "files", Value: numFilesProcessedBatch},
logg.Field{Name: "files_total", Value: numFilesProcessedTotal.Load()},
+ logg.Field{Name: "pages_total", Value: numPagesProcessedTotal.Load()},
+ logg.Field{Name: "resources_total", Value: numResourcesProcessed.Load()},
},
"",
)
c.g = rungroup.Run[hugofs.FileMetaInfo](c.ctx, rungroup.Config[hugofs.FileMetaInfo]{
NumWorkers: numWorkers,
Handle: func(ctx context.Context, fi hugofs.FileMetaInfo) error {
- if err := c.m.AddFi(fi); err != nil {
+ numPages, numResources, err := c.m.AddFi(fi, c.buildConfig)
+ if err != nil {
return hugofs.AddFileInfoToError(err, fi, c.fs)
}
numFilesProcessedTotal.Add(1)
+ numPagesProcessedTotal.Add(numPages)
+ numResourcesProcessed.Add(numResources)
if numFilesProcessedTotal.Load()%1000 == 0 {
logFilesProcessed(false)
}
return nil, nil
}
+ n := 0
+ for _, fi := range readdir {
+ if fi.Meta().PathInfo.IsContentData() {
+ // _content.json
+ // These are not part of any bundle, so just add them directly and remove them from the readdir slice.
+ if err := c.g.Enqueue(fi); err != nil {
+ return nil, err
+ }
+ } else {
+ readdir[n] = fi
+ n++
+ }
+ }
+ readdir = readdir[:n]
+
// Pick the first regular file.
var first hugofs.FileMetaInfo
for _, fi := range readdir {
// Any bundle file will always be first.
firstPi := first.Meta().PathInfo
+
if firstPi == nil {
panic(fmt.Sprintf("collectDirDir: no path info for %q", first.Meta().Filename))
}
Info: root,
Fs: c.fs,
IgnoreFile: c.h.SourceSpec.IgnoreFile,
+ PathParser: c.h.Conf.PathParser(),
HookPre: preHook,
HookPost: postHook,
WalkFn: wfn,
Info: dir,
DirEntries: readdir,
IgnoreFile: c.h.SourceSpec.IgnoreFile,
+ PathParser: c.h.Conf.PathParser(),
WalkFn: walk,
})
--- /dev/null
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pagesfromdata
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "path/filepath"
+
+ "github.com/gohugoio/hugo/common/maps"
+ "github.com/gohugoio/hugo/common/paths"
+ "github.com/gohugoio/hugo/helpers"
+ "github.com/gohugoio/hugo/hugofs"
+ "github.com/gohugoio/hugo/identity"
+ "github.com/gohugoio/hugo/resources/page"
+ "github.com/gohugoio/hugo/resources/page/pagemeta"
+ "github.com/gohugoio/hugo/resources/resource"
+ "github.com/gohugoio/hugo/tpl"
+ "github.com/mitchellh/mapstructure"
+ "github.com/spf13/cast"
+)
+
+type PagesFromDataTemplateContext interface {
+ // AddPage adds a new page to the site.
+ // The first return value will always be an empty string.
+ AddPage(any) (string, error)
+
+ // AddResource adds a new resource to the site.
+ // The first return value will always be an empty string.
+ AddResource(any) (string, error)
+
+ // The site to which the pages will be added.
+ Site() page.Site
+
+ // The same template may be executed multiple times for multiple languages.
+ // The Store can be used to store state between these invocations.
+ Store() *maps.Scratch
+
+ // By default, the template will be executed for the language
+ // defined by the _content.gotmpl file (e.g. its mount definition).
+ // This method can be used to activate the template for all languages.
+ // The return value will always be an empty string.
+ EnableAllLanguages() string
+}
+
+var _ PagesFromDataTemplateContext = (*pagesFromDataTemplateContext)(nil)
+
+type pagesFromDataTemplateContext struct {
+ p *PagesFromTemplate
+}
+
+func (p *pagesFromDataTemplateContext) toPathMap(v any) (string, map[string]any, error) {
+ m, err := maps.ToStringMapE(v)
+ if err != nil {
+ return "", nil, err
+ }
+ pathv, ok := m["path"]
+ if !ok {
+ return "", nil, fmt.Errorf("path not set")
+ }
+ path, err := cast.ToStringE(pathv)
+ if err != nil || path == "" {
+ return "", nil, fmt.Errorf("invalid path %q", path)
+ }
+ return path, m, nil
+}
+
+func (p *pagesFromDataTemplateContext) AddPage(v any) (string, error) {
+ path, m, err := p.toPathMap(v)
+ if err != nil {
+ return "", err
+ }
+
+ if !p.p.buildState.checkHasChangedAndSetSourceInfo(path, m) {
+ return "", nil
+ }
+
+ pd := pagemeta.DefaultPageConfig
+ pd.IsFromContentAdapter = true
+
+ if err := mapstructure.WeakDecode(m, &pd); err != nil {
+ return "", fmt.Errorf("failed to decode page map: %w", err)
+ }
+
+ p.p.buildState.NumPagesAdded++
+
+ if err := pd.Validate(true); err != nil {
+ return "", err
+ }
+
+ return "", p.p.HandlePage(p.p, &pd)
+}
+
+func (p *pagesFromDataTemplateContext) AddResource(v any) (string, error) {
+ path, m, err := p.toPathMap(v)
+ if err != nil {
+ return "", err
+ }
+
+ if !p.p.buildState.checkHasChangedAndSetSourceInfo(path, m) {
+ return "", nil
+ }
+
+ var rd pagemeta.ResourceConfig
+ if err := mapstructure.WeakDecode(m, &rd); err != nil {
+ return "", err
+ }
+
+ p.p.buildState.NumResourcesAdded++
+
+ if err := rd.Validate(); err != nil {
+ return "", err
+ }
+
+ return "", p.p.HandleResource(p.p, &rd)
+}
+
+func (p *pagesFromDataTemplateContext) Site() page.Site {
+ return p.p.Site
+}
+
+func (p *pagesFromDataTemplateContext) Store() *maps.Scratch {
+ return p.p.store
+}
+
+func (p *pagesFromDataTemplateContext) EnableAllLanguages() string {
+ p.p.buildState.EnableAllLanguages = true
+ return ""
+}
+
+func NewPagesFromTemplate(opts PagesFromTemplateOptions) *PagesFromTemplate {
+ return &PagesFromTemplate{
+ PagesFromTemplateOptions: opts,
+ PagesFromTemplateDeps: opts.DepsFromSite(opts.Site),
+ buildState: &BuildState{
+ sourceInfosCurrent: maps.NewCache[string, *sourceInfo](),
+ },
+ store: maps.NewScratch(),
+ }
+}
+
+type PagesFromTemplateOptions struct {
+ Site page.Site
+ DepsFromSite func(page.Site) PagesFromTemplateDeps
+
+ DependencyManager identity.Manager
+
+ Watching bool
+
+ HandlePage func(pt *PagesFromTemplate, p *pagemeta.PageConfig) error
+ HandleResource func(pt *PagesFromTemplate, p *pagemeta.ResourceConfig) error
+
+ GoTmplFi hugofs.FileMetaInfo
+}
+
+type PagesFromTemplateDeps struct {
+ TmplFinder tpl.TemplateParseFinder
+ TmplExec tpl.TemplateExecutor
+}
+
+var _ resource.Staler = (*PagesFromTemplate)(nil)
+
+type PagesFromTemplate struct {
+ PagesFromTemplateOptions
+ PagesFromTemplateDeps
+ buildState *BuildState
+ store *maps.Scratch
+}
+
+func (b *PagesFromTemplate) AddChange(id identity.Identity) {
+ b.buildState.ChangedIdentities = append(b.buildState.ChangedIdentities, id)
+}
+
+func (b *PagesFromTemplate) MarkStale() {
+ b.buildState.StaleVersion++
+}
+
+func (b *PagesFromTemplate) StaleVersion() uint32 {
+ return b.buildState.StaleVersion
+}
+
+type BuildInfo struct {
+ NumPagesAdded uint64
+ NumResourcesAdded uint64
+ EnableAllLanguages bool
+ ChangedIdentities []identity.Identity
+ DeletedPaths []string
+ Path *paths.Path
+}
+
+type BuildState struct {
+ StaleVersion uint32
+
+ EnableAllLanguages bool
+
+ // Paths deleted in the current build.
+ DeletedPaths []string
+
+ // Changed identities in the current build.
+ ChangedIdentities []identity.Identity
+
+ NumPagesAdded uint64
+ NumResourcesAdded uint64
+
+ sourceInfosCurrent *maps.Cache[string, *sourceInfo]
+ sourceInfosPrevious *maps.Cache[string, *sourceInfo]
+}
+
+func (b *BuildState) hash(v any) uint64 {
+ return identity.HashUint64(v)
+}
+
+func (b *BuildState) checkHasChangedAndSetSourceInfo(changedPath string, v any) bool {
+ h := b.hash(v)
+ si, found := b.sourceInfosPrevious.Get(changedPath)
+ if found {
+ b.sourceInfosCurrent.Set(changedPath, si)
+ if si.hash == h {
+ return false
+ }
+ } else {
+ si = &sourceInfo{}
+ b.sourceInfosCurrent.Set(changedPath, si)
+ }
+ si.hash = h
+ return true
+}
+
+func (b *BuildState) resolveDeletedPaths() {
+ if b.sourceInfosPrevious == nil {
+ b.DeletedPaths = nil
+ return
+ }
+ var paths []string
+ b.sourceInfosPrevious.ForEeach(func(k string, _ *sourceInfo) {
+ if _, found := b.sourceInfosCurrent.Get(k); !found {
+ paths = append(paths, k)
+ }
+ })
+
+ b.DeletedPaths = paths
+}
+
+func (b *BuildState) PrepareNextBuild() {
+ b.sourceInfosPrevious = b.sourceInfosCurrent
+ b.sourceInfosCurrent = maps.NewCache[string, *sourceInfo]()
+ b.StaleVersion = 0
+ b.DeletedPaths = nil
+ b.ChangedIdentities = nil
+ b.NumPagesAdded = 0
+ b.NumResourcesAdded = 0
+}
+
+type sourceInfo struct {
+ hash uint64
+}
+
+func (p PagesFromTemplate) CloneForSite(s page.Site) *PagesFromTemplate {
+ // We deliberately make them share the same DepenencyManager and Store.
+ p.PagesFromTemplateOptions.Site = s
+ p.PagesFromTemplateDeps = p.PagesFromTemplateOptions.DepsFromSite(s)
+ p.buildState = &BuildState{
+ sourceInfosCurrent: maps.NewCache[string, *sourceInfo](),
+ }
+ return &p
+}
+
+func (p PagesFromTemplate) CloneForGoTmpl(fi hugofs.FileMetaInfo) *PagesFromTemplate {
+ p.PagesFromTemplateOptions.GoTmplFi = fi
+ return &p
+}
+
+func (p *PagesFromTemplate) GetDependencyManagerForScope(scope int) identity.Manager {
+ return p.DependencyManager
+}
+
+func (p *PagesFromTemplate) Execute(ctx context.Context) (BuildInfo, error) {
+ defer func() {
+ p.buildState.PrepareNextBuild()
+ }()
+
+ f, err := p.GoTmplFi.Meta().Open()
+ if err != nil {
+ return BuildInfo{}, err
+ }
+ defer f.Close()
+
+ tmpl, err := p.TmplFinder.Parse(filepath.ToSlash(p.GoTmplFi.Meta().Filename), helpers.ReaderToString(f))
+ if err != nil {
+ return BuildInfo{}, err
+ }
+
+ data := &pagesFromDataTemplateContext{
+ p: p,
+ }
+
+ ctx = tpl.Context.DependencyManagerScopedProvider.Set(ctx, p)
+
+ if err := p.TmplExec.ExecuteWithContext(ctx, tmpl, io.Discard, data); err != nil {
+ return BuildInfo{}, err
+ }
+
+ if p.Watching {
+ p.buildState.resolveDeletedPaths()
+ }
+
+ bi := BuildInfo{
+ NumPagesAdded: p.buildState.NumPagesAdded,
+ NumResourcesAdded: p.buildState.NumResourcesAdded,
+ EnableAllLanguages: p.buildState.EnableAllLanguages,
+ ChangedIdentities: p.buildState.ChangedIdentities,
+ DeletedPaths: p.buildState.DeletedPaths,
+ Path: p.GoTmplFi.Meta().PathInfo,
+ }
+
+ return bi, nil
+}
+
+//////////////
--- /dev/null
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pagesfromdata_test
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+
+ qt "github.com/frankban/quicktest"
+ "github.com/gohugoio/hugo/hugolib"
+ "github.com/gohugoio/hugo/markup/asciidocext"
+ "github.com/gohugoio/hugo/markup/pandoc"
+ "github.com/gohugoio/hugo/markup/rst"
+)
+
+const filesPagesFromDataTempleBasic = `
+-- hugo.toml --
+disableKinds = ["taxonomy", "term", "rss", "sitemap"]
+baseURL = "https://example.com"
+disableLiveReload = true
+-- assets/a/pixel.png --
+iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==
+-- assets/mydata.yaml --
+p1: "p1"
+draft: false
+-- layouts/partials/get-value.html --
+{{ $val := "p1" }}
+{{ return $val }}
+-- layouts/_default/single.html --
+Single: {{ .Title }}|{{ .Content }}|Params: {{ .Params.param1 }}|Path: {{ .Path }}|
+Dates: Date: {{ .Date.Format "2006-01-02" }}|Lastmod: {{ .Lastmod.Format "2006-01-02" }}|PublishDate: {{ .PublishDate.Format "2006-01-02" }}|ExpiryDate: {{ .ExpiryDate.Format "2006-01-02" }}|
+Len Resources: {{ .Resources | len }}
+Resources: {{ range .Resources }}RelPermalink: {{ .RelPermalink }}|Name: {{ .Name }}|Title: {{ .Title }}|Params: {{ .Params }}|{{ end }}$
+{{ with .Resources.Get "featured.png" }}
+Featured Image: {{ .RelPermalink }}|{{ .Name }}|
+{{ with .Resize "10x10" }}
+Resized Featured Image: {{ .RelPermalink }}|{{ .Width }}|
+{{ end}}
+{{ end }}
+-- layouts/_default/list.html --
+List: {{ .Title }}|{{ .Content }}|
+RegularPagesRecursive: {{ range .RegularPagesRecursive }}{{ .Title }}:{{ .Path }}|{{ end }}$
+Sections: {{ range .Sections }}{{ .Title }}:{{ .Path }}|{{ end }}$
+-- content/docs/pfile.md --
+---
+title: "pfile"
+date: 2023-03-01
+---
+Pfile Content
+-- content/docs/_content.gotmpl --
+{{ $pixel := resources.Get "a/pixel.png" }}
+{{ $dataResource := resources.Get "mydata.yaml" }}
+{{ $data := $dataResource | transform.Unmarshal }}
+{{ $pd := $data.p1 }}
+{{ $pp := partial "get-value.html" }}
+{{ $title := printf "%s:%s" $pd $pp }}
+{{ $date := "2023-03-01" | time.AsTime }}
+{{ $dates := dict "date" $date }}
+{{ $contentMarkdown := dict "value" "**Hello World**" "mediaType" "text/markdown" }}
+{{ $contentMarkdownDefault := dict "value" "**Hello World Default**" }}
+{{ $contentHTML := dict "value" "<b>Hello World!</b> No **markdown** here." "mediaType" "text/html" }}
+{{ $.AddPage (dict "kind" "page" "path" "P1" "title" $title "dates" $dates "content" $contentMarkdown "params" (dict "param1" "param1v" ) ) }}
+{{ $.AddPage (dict "kind" "page" "path" "p2" "title" "p2title" "dates" $dates "content" $contentHTML ) }}
+{{ $.AddPage (dict "kind" "page" "path" "p3" "title" "p3title" "dates" $dates "content" $contentMarkdownDefault "draft" false ) }}
+{{ $.AddPage (dict "kind" "page" "path" "p4" "title" "p4title" "dates" $dates "content" $contentMarkdownDefault "draft" $data.draft ) }}
+
+
+{{ $resourceContent := dict "value" $dataResource }}
+{{ $.AddResource (dict "path" "p1/data1.yaml" "content" $resourceContent) }}
+{{ $.AddResource (dict "path" "p1/mytext.txt" "content" (dict "value" "some text") "name" "textresource" "title" "My Text Resource" "params" (dict "param1" "param1v") )}}
+{{ $.AddResource (dict "path" "p1/sub/mytex2.txt" "content" (dict "value" "some text") "title" "My Text Sub Resource" ) }}
+{{ $.AddResource (dict "path" "P1/Sub/MyMixCaseText2.txt" "content" (dict "value" "some text") "title" "My Text Sub Mixed Case Path Resource" ) }}
+{{ $.AddResource (dict "path" "p1/sub/data1.yaml" "content" $resourceContent "title" "Sub data") }}
+{{ $resourceParams := dict "data2ParaM1" "data2Param1v" }}
+{{ $.AddResource (dict "path" "p1/data2.yaml" "name" "data2.yaml" "title" "My data 2" "params" $resourceParams "content" $resourceContent) }}
+{{ $.AddResource (dict "path" "p1/featuredimage.png" "name" "featured.png" "title" "My Featured Image" "params" $resourceParams "content" (dict "value" $pixel ))}}
+`
+
+func TestPagesFromGoTmplMisc(t *testing.T) {
+ t.Parallel()
+ b := hugolib.Test(t, filesPagesFromDataTempleBasic)
+ b.AssertPublishDir(`
+docs/p1/mytext.txt
+docs/p1/sub/mytex2.tx
+docs/p1/sub/mymixcasetext2.txt
+ `)
+
+ // Page from markdown file.
+ b.AssertFileContent("public/docs/pfile/index.html", "Dates: Date: 2023-03-01|Lastmod: 2023-03-01|PublishDate: 2023-03-01|ExpiryDate: 0001-01-01|")
+ // Pages from gotmpl.
+ b.AssertFileContent("public/docs/p1/index.html",
+ "Single: p1:p1|",
+ "Path: /docs/p1|",
+ "<strong>Hello World</strong>",
+ "Params: param1v|",
+ "Len Resources: 7",
+ "RelPermalink: /mydata.yaml|Name: data1.yaml|Title: data1.yaml|Params: map[]|",
+ "RelPermalink: /mydata.yaml|Name: data2.yaml|Title: My data 2|Params: map[data2param1:data2Param1v]|",
+ "RelPermalink: /a/pixel.png|Name: featured.png|Title: My Featured Image|Params: map[data2param1:data2Param1v]|",
+ "RelPermalink: /docs/p1/sub/mytex2.txt|Name: sub/mytex2.txt|",
+ "RelPermalink: /docs/p1/sub/mymixcasetext2.txt|Name: sub/mymixcasetext2.txt|",
+ "RelPermalink: /mydata.yaml|Name: sub/data1.yaml|Title: Sub data|Params: map[]|",
+ "Featured Image: /a/pixel.png|featured.png|",
+ "Resized Featured Image: /a/pixel_hu8aa3346827e49d756ff4e630147c42b5_70_10x10_resize_box_3.png|10|",
+ // Resource from string
+ "RelPermalink: /docs/p1/mytext.txt|Name: textresource|Title: My Text Resource|Params: map[param1:param1v]|",
+ // Dates
+ "Dates: Date: 2023-03-01|Lastmod: 2023-03-01|PublishDate: 2023-03-01|ExpiryDate: 0001-01-01|",
+ )
+ b.AssertFileContent("public/docs/p2/index.html", "Single: p2title|", "<b>Hello World!</b> No **markdown** here.")
+ b.AssertFileContent("public/docs/p3/index.html", "<strong>Hello World Default</strong>")
+}
+
+func TestPagesFromGoTmplAsciidocAndSimilar(t *testing.T) {
+ files := `
+-- hugo.toml --
+disableKinds = ["taxonomy", "term", "rss", "sitemap"]
+baseURL = "https://example.com"
+[security]
+[security.exec]
+allow = ['asciidoctor', 'pandoc','rst2html', 'python']
+-- layouts/_default/single.html --
+|Content: {{ .Content }}|Title: {{ .Title }}|Path: {{ .Path }}|
+-- content/docs/_content.gotmpl --
+{{ $.AddPage (dict "path" "asciidoc" "content" (dict "value" "Mark my words, #automation is essential#." "mediaType" "text/asciidoc" )) }}
+{{ $.AddPage (dict "path" "pandoc" "content" (dict "value" "This ~~is deleted text.~~" "mediaType" "text/pandoc" )) }}
+{{ $.AddPage (dict "path" "rst" "content" (dict "value" "This is *bold*." "mediaType" "text/rst" )) }}
+{{ $.AddPage (dict "path" "org" "content" (dict "value" "the ability to use +strikethrough+ is a plus" "mediaType" "text/org" )) }}
+{{ $.AddPage (dict "path" "nocontent" "title" "No Content" ) }}
+
+ `
+
+ b := hugolib.Test(t, files)
+
+ if asciidocext.Supports() {
+ b.AssertFileContent("public/docs/asciidoc/index.html",
+ "Mark my words, <mark>automation is essential</mark>",
+ "Path: /docs/asciidoc|",
+ )
+ }
+ if pandoc.Supports() {
+ b.AssertFileContent("public/docs/pandoc/index.html",
+ "This <del>is deleted text.</del>",
+ "Path: /docs/pandoc|",
+ )
+ }
+
+ if rst.Supports() {
+ b.AssertFileContent("public/docs/rst/index.html",
+ "This is <em>bold</em>",
+ "Path: /docs/rst|",
+ )
+ }
+
+ b.AssertFileContent("public/docs/org/index.html",
+ "the ability to use <del>strikethrough</del> is a plus",
+ "Path: /docs/org|",
+ )
+
+ b.AssertFileContent("public/docs/nocontent/index.html", "|Content: |Title: No Content|Path: /docs/nocontent|")
+}
+
+func TestPagesFromGoTmplAddPageErrors(t *testing.T) {
+ filesTemplate := `
+-- hugo.toml --
+disableKinds = ["taxonomy", "term", "rss", "sitemap"]
+baseURL = "https://example.com"
+-- content/docs/_content.gotmpl --
+{{ $.AddPage DICT }}
+`
+
+ t.Run("AddPage, missing Path", func(t *testing.T) {
+ files := strings.ReplaceAll(filesTemplate, "DICT", `(dict "kind" "page" "title" "p1")`)
+ b, err := hugolib.TestE(t, files)
+ b.Assert(err, qt.IsNotNil)
+ b.Assert(err.Error(), qt.Contains, "_content.gotmpl:1:4")
+ b.Assert(err.Error(), qt.Contains, "error calling AddPage: path not set")
+ })
+
+ t.Run("AddPage, path starting with slash", func(t *testing.T) {
+ files := strings.ReplaceAll(filesTemplate, "DICT", `(dict "kind" "page" "title" "p1" "path" "/foo")`)
+ b, err := hugolib.TestE(t, files)
+ b.Assert(err, qt.IsNotNil)
+ b.Assert(err.Error(), qt.Contains, `path "/foo" must not start with a /`)
+ })
+
+ t.Run("AddPage, lang set", func(t *testing.T) {
+ files := strings.ReplaceAll(filesTemplate, "DICT", `(dict "kind" "page" "path" "p1" "lang" "en")`)
+ b, err := hugolib.TestE(t, files)
+ b.Assert(err, qt.IsNotNil)
+ b.Assert(err.Error(), qt.Contains, "_content.gotmpl:1:4")
+ b.Assert(err.Error(), qt.Contains, "error calling AddPage: lang must not be set")
+ })
+
+ t.Run("Site methods not ready", func(t *testing.T) {
+ filesTemplate := `
+-- hugo.toml --
+disableKinds = ["taxonomy", "term", "rss", "sitemap"]
+baseURL = "https://example.com"
+-- content/docs/_content.gotmpl --
+{{ .Site.METHOD }}
+`
+
+ for _, method := range []string{"RegularPages", "Pages", "AllPages", "AllRegularPages", "Home", "Sections", "GetPage", "Menus", "MainSections", "Taxonomies"} {
+ t.Run(method, func(t *testing.T) {
+ files := strings.ReplaceAll(filesTemplate, "METHOD", method)
+ b, err := hugolib.TestE(t, files)
+ b.Assert(err, qt.IsNotNil)
+ b.Assert(err.Error(), qt.Contains, fmt.Sprintf("error calling %s: this method cannot be called before the site is fully initialized", method))
+ })
+ }
+ })
+}
+
+func TestPagesFromGoTmplAddResourceErrors(t *testing.T) {
+ filesTemplate := `
+-- hugo.toml --
+disableKinds = ["taxonomy", "term", "rss", "sitemap"]
+baseURL = "https://example.com"
+-- content/docs/_content.gotmpl --
+{{ $.AddResource DICT }}
+`
+
+ t.Run("missing Path", func(t *testing.T) {
+ files := strings.ReplaceAll(filesTemplate, "DICT", `(dict "name" "r1")`)
+ b, err := hugolib.TestE(t, files)
+ b.Assert(err, qt.IsNotNil)
+ b.Assert(err.Error(), qt.Contains, "error calling AddResource: path not set")
+ })
+}
+
+func TestPagesFromGoTmplEditGoTmpl(t *testing.T) {
+ t.Parallel()
+ b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic)
+ b.EditFileReplaceAll("content/docs/_content.gotmpl", `"title" "p2title"`, `"title" "p2titleedited"`).Build()
+ b.AssertFileContent("public/docs/p2/index.html", "Single: p2titleedited|")
+ b.AssertFileContent("public/docs/index.html", "p2titleedited")
+}
+
+func TestPagesFromGoTmplEditDataResource(t *testing.T) {
+ t.Parallel()
+ b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic)
+ b.AssertRenderCountPage(7)
+ b.EditFileReplaceAll("assets/mydata.yaml", "p1: \"p1\"", "p1: \"p1edited\"").Build()
+ b.AssertFileContent("public/docs/p1/index.html", "Single: p1edited:p1|")
+ b.AssertFileContent("public/docs/index.html", "p1edited")
+ b.AssertRenderCountPage(3)
+}
+
+func TestPagesFromGoTmplEditPartial(t *testing.T) {
+ t.Parallel()
+ b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic)
+ b.EditFileReplaceAll("layouts/partials/get-value.html", "p1", "p1edited").Build()
+ b.AssertFileContent("public/docs/p1/index.html", "Single: p1:p1edited|")
+ b.AssertFileContent("public/docs/index.html", "p1edited")
+}
+
+func TestPagesFromGoTmplRemovePage(t *testing.T) {
+ t.Parallel()
+ b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic)
+ b.EditFileReplaceAll("content/docs/_content.gotmpl", `{{ $.AddPage (dict "kind" "page" "path" "p2" "title" "p2title" "dates" $dates "content" $contentHTML ) }}`, "").Build()
+ b.AssertFileContent("public/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p3title:/docs/p3|p4title:/docs/p4|pfile:/docs/pfile|$")
+}
+
+func TestPagesFromGoTmplDraftPage(t *testing.T) {
+ t.Parallel()
+ b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic)
+ b.EditFileReplaceAll("content/docs/_content.gotmpl", `"draft" false`, `"draft" true`).Build()
+ b.AssertFileContent("public/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p2title:/docs/p2|p4title:/docs/p4|pfile:/docs/pfile|$")
+}
+
+func TestPagesFromGoTmplDraftFlagFromResource(t *testing.T) {
+ t.Parallel()
+ b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic)
+ b.EditFileReplaceAll("assets/mydata.yaml", `draft: false`, `draft: true`).Build()
+ b.AssertFileContent("public/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p2title:/docs/p2|p3title:/docs/p3|pfile:/docs/pfile|$")
+ b.EditFileReplaceAll("assets/mydata.yaml", `draft: true`, `draft: false`).Build()
+ b.AssertFileContent("public/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p2title:/docs/p2|p3title:/docs/p3|p4title:/docs/p4|pfile:/docs/pfile|$")
+}
+
+func TestPagesFromGoTmplMovePage(t *testing.T) {
+ t.Parallel()
+ b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic)
+ b.AssertFileContent("public/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p2title:/docs/p2|p3title:/docs/p3|p4title:/docs/p4|pfile:/docs/pfile|$")
+ b.EditFileReplaceAll("content/docs/_content.gotmpl", `"path" "p2"`, `"path" "p2moved"`).Build()
+ b.AssertFileContent("public/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p2title:/docs/p2moved|p3title:/docs/p3|p4title:/docs/p4|pfile:/docs/pfile|$")
+}
+
+func TestPagesFromGoTmplRemoveGoTmpl(t *testing.T) {
+ t.Parallel()
+ b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic)
+ b.AssertFileContent("public/index.html",
+ "RegularPagesRecursive: p1:p1:/docs/p1|p2title:/docs/p2|p3title:/docs/p3|p4title:/docs/p4|pfile:/docs/pfile|$",
+ "Sections: Docs:/docs|",
+ )
+ b.AssertFileContent("public/docs/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p2title:/docs/p2|p3title:/docs/p3|p4title:/docs/p4|pfile:/docs/pfile|$")
+ b.RemoveFiles("content/docs/_content.gotmpl").Build()
+ // One regular page left.
+ b.AssertFileContent("public/index.html",
+ "RegularPagesRecursive: pfile:/docs/pfile|$",
+ "Sections: Docs:/docs|",
+ )
+ b.AssertFileContent("public/docs/index.html", "RegularPagesRecursive: pfile:/docs/pfile|$")
+}
+
+func TestPagesFromGoTmplLanguagePerFile(t *testing.T) {
+ filesTemplate := `
+-- hugo.toml --
+defaultContentLanguage = "en"
+defaultContentLanguageInSubdir = true
+[languages]
+[languages.en]
+weight = 1
+title = "Title"
+[languages.fr]
+weight = 2
+title = "Titre"
+disabled = DISABLE
+-- layouts/_default/single.html --
+Single: {{ .Title }}|{{ .Content }}|
+-- content/docs/_content.gotmpl --
+{{ $.AddPage (dict "kind" "page" "path" "p1" "title" "Title" ) }}
+-- content/docs/_content.fr.gotmpl --
+{{ $.AddPage (dict "kind" "page" "path" "p1" "title" "Titre" ) }}
+`
+
+ for _, disable := range []bool{false, true} {
+ t.Run(fmt.Sprintf("disable=%t", disable), func(t *testing.T) {
+ b := hugolib.Test(t, strings.ReplaceAll(filesTemplate, "DISABLE", fmt.Sprintf("%t", disable)))
+ b.AssertFileContent("public/en/docs/p1/index.html", "Single: Title||")
+ b.AssertFileExists("public/fr/docs/p1/index.html", !disable)
+ if !disable {
+ b.AssertFileContent("public/fr/docs/p1/index.html", "Single: Titre||")
+ }
+ })
+ }
+}
+
+func TestPagesFromGoTmplEnableAllLanguages(t *testing.T) {
+ t.Parallel()
+
+ filesTemplate := `
+-- hugo.toml --
+defaultContentLanguage = "en"
+defaultContentLanguageInSubdir = true
+[languages]
+[languages.en]
+weight = 1
+title = "Title"
+[languages.fr]
+title = "Titre"
+weight = 2
+disabled = DISABLE
+-- i18n/en.yaml --
+title: Title
+-- i18n/fr.yaml --
+title: Titre
+-- content/docs/_content.gotmpl --
+{{ .EnableAllLanguages }}
+{{ $titleFromStore := .Store.Get "title" }}
+{{ if not $titleFromStore }}
+ {{ $titleFromStore = "notfound"}}
+ {{ .Store.Set "title" site.Title }}
+{{ end }}
+{{ $title := printf "%s:%s:%s" site.Title (i18n "title") $titleFromStore }}
+{{ $.AddPage (dict "kind" "page" "path" "p1" "title" $title ) }}
+-- layouts/_default/single.html --
+Single: {{ .Title }}|{{ .Content }}|
+
+`
+
+ for _, disable := range []bool{false, true} {
+ t.Run(fmt.Sprintf("disable=%t", disable), func(t *testing.T) {
+ b := hugolib.Test(t, strings.ReplaceAll(filesTemplate, "DISABLE", fmt.Sprintf("%t", disable)))
+ b.AssertFileExists("public/fr/docs/p1/index.html", !disable)
+ if !disable {
+ b.AssertFileContent("public/en/docs/p1/index.html", "Single: Title:Title:notfound||")
+ b.AssertFileContent("public/fr/docs/p1/index.html", "Single: Titre:Titre:Title||")
+ }
+ })
+ }
+}
+
+func TestPagesFromGoTmplMarkdownify(t *testing.T) {
+ t.Parallel()
+
+ files := `
+-- hugo.toml --
+disableKinds = ["taxonomy", "term", "rss", "sitemap"]
+baseURL = "https://example.com"
+-- layouts/_default/single.html --
+|Content: {{ .Content }}|Title: {{ .Title }}|Path: {{ .Path }}|
+-- content/docs/_content.gotmpl --
+{{ $content := "**Hello World**" | markdownify }}
+{{ $.AddPage (dict "path" "p1" "content" (dict "value" $content "mediaType" "text/html" )) }}
+`
+
+ b, err := hugolib.TestE(t, files)
+
+ // This currently fails. We should fix this, but that is not a trivial task, so do it later.
+ b.Assert(err, qt.IsNotNil)
+ b.Assert(err.Error(), qt.Contains, "error calling markdownify: this method cannot be called before the site is fully initialized")
+}
+
+func TestPagesFromGoTmplResourceWithoutExtensionWithMediaTypeProvided(t *testing.T) {
+ t.Parallel()
+
+ files := `
+-- hugo.toml --
+disableKinds = ["taxonomy", "term", "rss", "sitemap"]
+baseURL = "https://example.com"
+-- layouts/_default/single.html --
+|Content: {{ .Content }}|Title: {{ .Title }}|Path: {{ .Path }}|
+{{ range .Resources }}
+|RelPermalink: {{ .RelPermalink }}|Name: {{ .Name }}|Title: {{ .Title }}|Params: {{ .Params }}|MediaType: {{ .MediaType }}|
+{{ end }}
+-- content/docs/_content.gotmpl --
+{{ $.AddPage (dict "path" "p1" "content" (dict "value" "**Hello World**" "mediaType" "text/markdown" )) }}
+{{ $.AddResource (dict "path" "p1/myresource" "content" (dict "value" "abcde" "mediaType" "text/plain" )) }}
+`
+
+ b := hugolib.Test(t, files)
+
+ b.AssertFileContent("public/docs/p1/index.html", "RelPermalink: /docs/p1/myresource|Name: myresource|Title: myresource|Params: map[]|MediaType: text/plain|")
+}
+
+func TestPagesFromGoTmplCascade(t *testing.T) {
+ t.Parallel()
+
+ files := `
+-- hugo.toml --
+disableKinds = ["taxonomy", "term", "rss", "sitemap"]
+baseURL = "https://example.com"
+-- layouts/_default/single.html --
+|Content: {{ .Content }}|Title: {{ .Title }}|Path: {{ .Path }}|Params: {{ .Params }}|
+-- content/_content.gotmpl --
+{{ $cascade := dict "params" (dict "cascadeparam1" "cascadeparam1value" ) }}
+{{ $.AddPage (dict "path" "docs" "kind" "section" "cascade" $cascade ) }}
+{{ $.AddPage (dict "path" "docs/p1" "content" (dict "value" "**Hello World**" "mediaType" "text/markdown" )) }}
+
+`
+ b := hugolib.Test(t, files)
+
+ b.AssertFileContent("public/docs/p1/index.html", "|Path: /docs/p1|Params: map[cascadeparam1:cascadeparam1value")
+}
+
+func TestPagesFromGoBuildOptions(t *testing.T) {
+ t.Parallel()
+
+ files := `
+-- hugo.toml --
+disableKinds = ["taxonomy", "term", "rss", "sitemap"]
+baseURL = "https://example.com"
+-- layouts/_default/single.html --
+|Content: {{ .Content }}|Title: {{ .Title }}|Path: {{ .Path }}|Params: {{ .Params }}|
+-- content/_content.gotmpl --
+{{ $.AddPage (dict "path" "docs/p1" "content" (dict "value" "**Hello World**" "mediaType" "text/markdown" )) }}
+{{ $never := dict "list" "never" "publishResources" false "render" "never" }}
+{{ $.AddPage (dict "path" "docs/p2" "content" (dict "value" "**Hello World**" "mediaType" "text/markdown" ) "build" $never ) }}
+
+
+`
+ b := hugolib.Test(t, files)
+
+ b.AssertFileExists("public/docs/p1/index.html", true)
+ b.AssertFileExists("public/docs/p2/index.html", false)
+}
--- /dev/null
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pagesfromdata
+
+import "testing"
+
+func BenchmarkHash(b *testing.B) {
+ m := map[string]any{
+ "foo": "bar",
+ "bar": "foo",
+ "stringSlice": []any{"a", "b", "c"},
+ "intSlice": []any{1, 2, 3},
+ "largeText": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris. Maecenas congue ligula ac quam viverra nec consectetur ante hendrerit.",
+ }
+
+ bs := BuildState{}
+
+ for i := 0; i < b.N; i++ {
+ bs.hash(m)
+ }
+}
// Allow the caller to delay the rendering of the shortcode if needed.
var fn shortcodeRenderFunc = func(ctx context.Context) ([]byte, bool, error) {
- if p.m.pageConfig.IsGoldmark && sc.doMarkup {
+ if p.m.pageConfig.ContentMediaType.IsMarkdown() && sc.doMarkup {
// Signal downwards that the content rendered will be
// parsed and rendered by Goldmark.
ctx = tpl.Context.IsInGoldmark.Set(ctx, true)
// unchanged.
// 2 If inner does not have a newline, strip the wrapping <p> block and
// the newline.
- switch p.m.pageConfig.Markup {
+ switch p.m.pageConfig.Content.Markup {
case "", "markdown":
if match, _ := regexp.MatchString(innerNewlineRegexp, inner); !match {
cleaner, err := regexp.Compile(innerCleanupRegexp)
)
func (s *Site) Taxonomies() page.TaxonomyList {
+ s.checkReady()
s.init.taxonomies.Do(context.Background())
return s.taxonomies
}
}
func (s *Site) Menus() navigation.Menus {
+ s.checkReady()
s.init.menus.Do(context.Background())
return s.menus
}
type whatChanged struct {
mu sync.Mutex
- contentChanged bool
- identitySet identity.Identities
+ needsPagesAssembly bool
+ identitySet identity.Identities
}
func (w *whatChanged) Add(ids ...identity.Identity) {
w.mu.Lock()
defer w.mu.Unlock()
+ if w.identitySet == nil {
+ w.identitySet = make(identity.Identities)
+ }
+
for _, id := range ids {
w.identitySet[id] = true
}
}
+func (w *whatChanged) Clear() {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ w.clear()
+}
+
+func (w *whatChanged) clear() {
+ w.identitySet = identity.Identities{}
+}
+
func (w *whatChanged) Changes() []identity.Identity {
if w == nil || w.identitySet == nil {
return nil
return w.identitySet.AsSlice()
}
+func (w *whatChanged) Drain() []identity.Identity {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ ids := w.identitySet.AsSlice()
+ w.clear()
+ return ids
+}
+
// RegisterMediaTypes will register the Site's media types in the mime
// package, so it will behave correctly with Hugo's built-in server.
func (s *Site) RegisterMediaTypes() {
// as possible for existing sites. Most sites will use {{ .Site.GetPage "section" "my/section" }},
// i.e. 2 arguments, so we test for that.
func (s *Site) GetPage(ref ...string) (page.Page, error) {
+ s.checkReady()
p, err := s.s.getPageForRefs(ref...)
if p == nil {
"github.com/gohugoio/hugo/config/allconfig"
"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/hugolib/doctree"
+ "github.com/gohugoio/hugo/hugolib/pagesfromdata"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/langs"
"github.com/gohugoio/hugo/langs/i18n"
var _ page.Site = (*Site)(nil)
+type siteState int
+
+const (
+ siteStateInit siteState = iota
+ siteStateReady
+)
+
type Site struct {
+ state siteState
conf *allconfig.Config
language *langs.Language
languagei int
treeResources: doctree.New(
treeConfig,
),
- treeTaxonomyEntries: doctree.NewTreeShiftTree[*weightedContentNode](doctree.DimensionLanguage.Index(), len(confm.Languages)),
+ treeTaxonomyEntries: doctree.NewTreeShiftTree[*weightedContentNode](doctree.DimensionLanguage.Index(), len(confm.Languages)),
+ treePagesFromTemplateAdapters: doctree.NewTreeShiftTree[*pagesfromdata.PagesFromTemplate](doctree.DimensionLanguage.Index(), len(confm.Languages)),
}
pageTrees.createMutableTrees()
// MainSections returns the list of main sections.
func (s *Site) MainSections() []string {
+ s.checkReady()
return s.conf.C.MainSections
}
// Deprecated: Use .Site.Lastmod instead.
func (s *Site) LastChange() time.Time {
+ s.checkReady()
hugo.Deprecate(".Site.LastChange", "Use .Site.Lastmod instead.", "v0.123.0")
return s.lastmod
}
// Pages returns all pages.
// This is for the current language only.
func (s *Site) Pages() page.Pages {
+ s.checkReady()
return s.pageMap.getPagesInSection(
pageMapQueryPagesInSection{
pageMapQueryPagesBelowPath: pageMapQueryPagesBelowPath{
// RegularPages returns all the regular pages.
// This is for the current language only.
func (s *Site) RegularPages() page.Pages {
+ s.checkReady()
return s.pageMap.getPagesInSection(
pageMapQueryPagesInSection{
pageMapQueryPagesBelowPath: pageMapQueryPagesBelowPath{
// AllPages returns all pages for all sites.
func (s *Site) AllPages() page.Pages {
+ s.checkReady()
return s.h.Pages()
}
// AllRegularPages returns all regular pages for all sites.
func (s *Site) AllRegularPages() page.Pages {
+ s.checkReady()
return s.h.RegularPages()
}
+
+func (s *Site) checkReady() {
+ if s.state != siteStateReady {
+ panic("this method cannot be called before the site is fully initialized")
+ }
+}
// Sections returns the top level sections.
func (s *Site) Sections() page.Pages {
+ s.checkReady()
return s.Home().Sections()
}
// Home is a shortcut to the home page, equivalent to .Site.GetPage "home".
func (s *Site) Home() page.Page {
+ s.checkReady()
return s.s.home
}
hugofs.WalkwayConfig{
Fs: dst.BaseFs.I18n.Fs,
IgnoreFile: dst.SourceSpec.IgnoreFile,
+ PathParser: dst.SourceSpec.Cfg.PathParser(),
WalkFn: func(path string, info hugofs.FileMetaInfo) error {
if info.IsDir() {
return nil
"strings"
"github.com/gohugoio/hugo/markup/highlight"
+ "github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/markup/markup_config"
defaultHandler := mcfg.DefaultMarkdownHandler
var defaultFound bool
- add := func(p converter.ProviderProvider, aliases ...string) error {
+ add := func(p converter.ProviderProvider, subType string, aliases ...string) error {
c, err := p.New(cfg)
if err != nil {
return err
name := c.Name()
aliases = append(aliases, name)
+ aliases = append(aliases, subType)
if strings.EqualFold(name, defaultHandler) {
aliases = append(aliases, "markdown")
return nil
}
- if err := add(goldmark.Provider); err != nil {
+ contentTypes := cfg.Conf.ContentTypes().(media.ContentTypes)
+
+ if err := add(goldmark.Provider, contentTypes.Markdown.SubType, contentTypes.Markdown.Suffixes()...); err != nil {
return nil, err
}
- if err := add(asciidocext.Provider, "ad", "adoc"); err != nil {
+ if err := add(asciidocext.Provider, contentTypes.AsciiDoc.SubType, contentTypes.AsciiDoc.Suffixes()...); err != nil {
return nil, err
}
- if err := add(rst.Provider); err != nil {
+ if err := add(rst.Provider, contentTypes.ReStructuredText.SubType, contentTypes.ReStructuredText.Suffixes()...); err != nil {
return nil, err
}
- if err := add(pandoc.Provider, "pdc"); err != nil {
+ if err := add(pandoc.Provider, contentTypes.Pandoc.SubType, contentTypes.Pandoc.Suffixes()...); err != nil {
return nil, err
}
- if err := add(org.Provider); err != nil {
+ if err := add(org.Provider, contentTypes.EmacsOrgMode.SubType, contentTypes.EmacsOrgMode.Suffixes()...); err != nil {
return nil, err
}
m[alias] = c
}
}
+
+// ResolveMarkup returns the markup type.
+func ResolveMarkup(s string) string {
+ s = strings.ToLower(s)
+ switch s {
+ case "goldmark":
+ return media.DefaultContentTypes.Markdown.SubType
+ case "asciidocext":
+ return media.DefaultContentTypes.AsciiDoc.SubType
+ default:
+ return s
+ }
+}
OpenTypeFontType Type
// Common document types
- PDFType Type
- MarkdownType Type
+ PDFType Type
+ MarkdownType Type
+ EmacsOrgModeType Type
+ AsciiDocType Type
+ PandocType Type
+ ReStructuredTextType Type
// Common video types
AVIType Type
OpenTypeFontType: Type{Type: "font/otf"},
// Common document types
- PDFType: Type{Type: "application/pdf"},
- MarkdownType: Type{Type: "text/markdown"},
+ PDFType: Type{Type: "application/pdf"},
+ MarkdownType: Type{Type: "text/markdown"},
+ AsciiDocType: Type{Type: "text/asciidoc"}, // https://github.com/asciidoctor/asciidoctor/issues/2502
+ PandocType: Type{Type: "text/pandoc"},
+ ReStructuredTextType: Type{Type: "text/rst"}, // https://docutils.sourceforge.io/FAQ.html#what-s-the-official-mime-type-for-restructuredtext-data
+ EmacsOrgModeType: Type{Type: "text/org"},
// Common video types
AVIType: Type{Type: "video/x-msvideo"},
"text/x-scss": map[string]any{"suffixes": []string{"scss"}},
"text/x-sass": map[string]any{"suffixes": []string{"sass"}},
"text/csv": map[string]any{"suffixes": []string{"csv"}},
- "text/html": map[string]any{"suffixes": []string{"html"}},
+ "text/html": map[string]any{"suffixes": []string{"html", "htm"}},
"text/javascript": map[string]any{"suffixes": []string{"js", "jsm", "mjs"}},
"text/typescript": map[string]any{"suffixes": []string{"ts"}},
"text/tsx": map[string]any{"suffixes": []string{"tsx"}},
// Common document types
"application/pdf": map[string]any{"suffixes": []string{"pdf"}},
- "text/markdown": map[string]any{"suffixes": []string{"md", "markdown"}},
+ "text/markdown": map[string]any{"suffixes": []string{"md", "mdown", "markdown"}},
+ "text/asciidoc": map[string]any{"suffixes": []string{"adoc", "asciidoc", "ad"}},
+ "text/pandoc": map[string]any{"suffixes": []string{"pandoc", "pdc"}},
+ "text/rst": map[string]any{"suffixes": []string{"rst"}},
+ "text/org": map[string]any{"suffixes": []string{"org"}},
// Common video types
"video/x-msvideo": map[string]any{"suffixes": []string{"avi"}},
"application/octet-stream": map[string]any{},
}
-
-func init() {
- // Apply delimiter to all.
- for _, m := range defaultMediaTypesConfig {
- m.(map[string]any)["delimiter"] = "."
- }
-}
package media
import (
- "errors"
"fmt"
+ "path/filepath"
"reflect"
"sort"
"strings"
"github.com/gohugoio/hugo/common/maps"
+ "github.com/gohugoio/hugo/common/paths"
"github.com/gohugoio/hugo/config"
"github.com/mitchellh/mapstructure"
var DefaultTypes Types
func init() {
+ // Apply delimiter to all.
+ for _, m := range defaultMediaTypesConfig {
+ m.(map[string]any)["delimiter"] = "."
+ }
+
ns, err := DecodeTypes(nil)
if err != nil {
panic(err)
// Initialize the Builtin types with values from DefaultTypes.
v := reflect.ValueOf(&Builtin).Elem()
+
for i := 0; i < v.NumField(); i++ {
f := v.Field(i)
+ fieldName := v.Type().Field(i).Name
builtinType := f.Interface().(Type)
+ if builtinType.Type == "" {
+ panic(fmt.Errorf("builtin type %q is empty", fieldName))
+ }
defaultType, found := DefaultTypes.GetByType(builtinType.Type)
if !found {
- panic(errors.New("missing default type for builtin type: " + builtinType.Type))
+ panic(fmt.Errorf("missing default type for field builtin type: %q", fieldName))
}
f.Set(reflect.ValueOf(defaultType))
}
}
+func init() {
+ DefaultContentTypes = ContentTypes{
+ HTML: Builtin.HTMLType,
+ Markdown: Builtin.MarkdownType,
+ AsciiDoc: Builtin.AsciiDocType,
+ Pandoc: Builtin.PandocType,
+ ReStructuredText: Builtin.ReStructuredTextType,
+ EmacsOrgMode: Builtin.EmacsOrgModeType,
+ }
+
+ DefaultContentTypes.init()
+}
+
+var DefaultContentTypes ContentTypes
+
+// ContentTypes holds the media types that are considered content in Hugo.
+type ContentTypes struct {
+ HTML Type
+ Markdown Type
+ AsciiDoc Type
+ Pandoc Type
+ ReStructuredText Type
+ EmacsOrgMode Type
+
+ // Created in init().
+ types Types
+ extensionSet map[string]bool
+}
+
+func (t *ContentTypes) init() {
+ t.types = Types{t.HTML, t.Markdown, t.AsciiDoc, t.Pandoc, t.ReStructuredText, t.EmacsOrgMode}
+ t.extensionSet = make(map[string]bool)
+ for _, mt := range t.types {
+ for _, suffix := range mt.Suffixes() {
+ t.extensionSet[suffix] = true
+ }
+ }
+}
+
+func (t ContentTypes) IsContentSuffix(suffix string) bool {
+ return t.extensionSet[suffix]
+}
+
+// IsContentFile returns whether the given filename is a content file.
+func (t ContentTypes) IsContentFile(filename string) bool {
+ return t.IsContentSuffix(strings.TrimPrefix(filepath.Ext(filename), "."))
+}
+
+// IsIndexContentFile returns whether the given filename is an index content file.
+func (t ContentTypes) IsIndexContentFile(filename string) bool {
+ if !t.IsContentFile(filename) {
+ return false
+ }
+
+ base := filepath.Base(filename)
+
+ return strings.HasPrefix(base, "index.") || strings.HasPrefix(base, "_index.")
+}
+
+// IsHTMLSuffix returns whether the given suffix is a HTML media type.
+func (t ContentTypes) IsHTMLSuffix(suffix string) bool {
+ for _, s := range t.HTML.Suffixes() {
+ if s == suffix {
+ return true
+ }
+ }
+ return false
+}
+
+// Types is a slice of media types.
+func (t ContentTypes) Types() Types {
+ return t.types
+}
+
+// FromTypes creates a new ContentTypes updated with the values from the given Types.
+func (t ContentTypes) FromTypes(types Types) ContentTypes {
+ if tt, ok := types.GetByType(t.HTML.Type); ok {
+ t.HTML = tt
+ }
+ if tt, ok := types.GetByType(t.Markdown.Type); ok {
+ t.Markdown = tt
+ }
+ if tt, ok := types.GetByType(t.AsciiDoc.Type); ok {
+ t.AsciiDoc = tt
+ }
+ if tt, ok := types.GetByType(t.Pandoc.Type); ok {
+ t.Pandoc = tt
+ }
+ if tt, ok := types.GetByType(t.ReStructuredText.Type); ok {
+ t.ReStructuredText = tt
+ }
+ if tt, ok := types.GetByType(t.EmacsOrgMode.Type); ok {
+ t.EmacsOrgMode = tt
+ }
+
+ t.init()
+
+ return t
+}
+
// Hold the configuration for a given media type.
type MediaTypeConfig struct {
// The file suffixes used for this media type.
}
return ns, nil
}
+
+// TODO(bep) get rid of this.
+var DefaultPathParser = &paths.PathParser{
+ IsContentExt: func(ext string) bool {
+ return DefaultContentTypes.IsContentSuffix(ext)
+ },
+}
tp Type
expectedMainType string
expectedSubType string
- expectedSuffix string
+ expectedSuffixes string
expectedType string
expectedString string
}{
{Builtin.CSSType, "text", "css", "css", "text/css", "text/css"},
{Builtin.SCSSType, "text", "x-scss", "scss", "text/x-scss", "text/x-scss"},
{Builtin.CSVType, "text", "csv", "csv", "text/csv", "text/csv"},
- {Builtin.HTMLType, "text", "html", "html", "text/html", "text/html"},
- {Builtin.JavascriptType, "text", "javascript", "js", "text/javascript", "text/javascript"},
+ {Builtin.HTMLType, "text", "html", "html,htm", "text/html", "text/html"},
+ {Builtin.MarkdownType, "text", "markdown", "md,mdown,markdown", "text/markdown", "text/markdown"},
+ {Builtin.EmacsOrgModeType, "text", "org", "org", "text/org", "text/org"},
+ {Builtin.PandocType, "text", "pandoc", "pandoc,pdc", "text/pandoc", "text/pandoc"},
+ {Builtin.ReStructuredTextType, "text", "rst", "rst", "text/rst", "text/rst"},
+ {Builtin.AsciiDocType, "text", "asciidoc", "adoc,asciidoc,ad", "text/asciidoc", "text/asciidoc"},
+ {Builtin.JavascriptType, "text", "javascript", "js,jsm,mjs", "text/javascript", "text/javascript"},
{Builtin.TypeScriptType, "text", "typescript", "ts", "text/typescript", "text/typescript"},
{Builtin.TSXType, "text", "tsx", "tsx", "text/tsx", "text/tsx"},
{Builtin.JSXType, "text", "jsx", "jsx", "text/jsx", "text/jsx"},
{Builtin.JSONType, "application", "json", "json", "application/json", "application/json"},
- {Builtin.RSSType, "application", "rss", "xml", "application/rss+xml", "application/rss+xml"},
+ {Builtin.RSSType, "application", "rss", "xml,rss", "application/rss+xml", "application/rss+xml"},
{Builtin.SVGType, "image", "svg", "svg", "image/svg+xml", "image/svg+xml"},
{Builtin.TextType, "text", "plain", "txt", "text/plain", "text/plain"},
{Builtin.XMLType, "application", "xml", "xml", "application/xml", "application/xml"},
{Builtin.TOMLType, "application", "toml", "toml", "application/toml", "application/toml"},
- {Builtin.YAMLType, "application", "yaml", "yaml", "application/yaml", "application/yaml"},
+ {Builtin.YAMLType, "application", "yaml", "yaml,yml", "application/yaml", "application/yaml"},
{Builtin.PDFType, "application", "pdf", "pdf", "application/pdf", "application/pdf"},
{Builtin.TrueTypeFontType, "font", "ttf", "ttf", "font/ttf", "font/ttf"},
{Builtin.OpenTypeFontType, "font", "otf", "otf", "font/otf", "font/otf"},
} {
c.Assert(test.tp.MainType, qt.Equals, test.expectedMainType)
c.Assert(test.tp.SubType, qt.Equals, test.expectedSubType)
-
+ c.Assert(test.tp.SuffixesCSV, qt.Equals, test.expectedSuffixes)
c.Assert(test.tp.Type, qt.Equals, test.expectedType)
c.Assert(test.tp.String(), qt.Equals, test.expectedString)
}
- c.Assert(len(DefaultTypes), qt.Equals, 36)
+ c.Assert(len(DefaultTypes), qt.Equals, 40)
}
return m
}
-// FromStringAndExt creates a Type from a MIME string and a given extension.
-func FromStringAndExt(t, ext string) (Type, error) {
+// FromStringAndExt creates a Type from a MIME string and a given extensions
+func FromStringAndExt(t string, ext ...string) (Type, error) {
tp, err := FromString(t)
if err != nil {
return tp, err
}
- tp.SuffixesCSV = strings.TrimPrefix(ext, ".")
+ for i, e := range ext {
+ ext[i] = strings.TrimPrefix(e, ".")
+ }
+ tp.SuffixesCSV = strings.Join(ext, ",")
tp.Delimiter = DefaultDelimiter
tp.init()
return tp, nil
return false
}
+// For internal use.
+func (m Type) IsHTML() bool {
+ return m.SubType == Builtin.HTMLType.SubType
+}
+
+// For internal use.
+func (m Type) IsMarkdown() bool {
+ return m.SubType == Builtin.MarkdownType.SubType
+}
+
func InitMediaType(m *Type) {
m.init()
}
func (t Types) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
func (t Types) Less(i, j int) bool { return t[i].Type < t[j].Type }
+// GetBestMatch returns the best match for the given media type string.
+func (t Types) GetBestMatch(s string) (Type, bool) {
+ // First try an exact match.
+ if mt, found := t.GetByType(s); found {
+ return mt, true
+ }
+
+ // Try main type.
+ if mt, found := t.GetBySubType(s); found {
+ return mt, true
+ }
+
+ // Try extension.
+ if mt, _, found := t.GetFirstBySuffix(s); found {
+ return mt, true
+ }
+
+ return Type{}, false
+}
+
// GetByType returns a media type for tp.
func (t Types) GetByType(tp string) (Type, bool) {
for _, tt := range t {
return
}
+// GetBySubType gets a media type given a sub type e.g. "plain".
+func (t Types) GetBySubType(subType string) (tp Type, found bool) {
+ for _, tt := range t {
+ if strings.EqualFold(subType, tt.SubType) {
+ if found {
+ // ambiguous
+ found = false
+ return
+ }
+ tp = tt
+ found = true
+ }
+ }
+ return
+}
+
// IsZero reports whether this Type represents a zero value.
// For internal use.
func (m Type) IsZero() bool {
func TestFromStringAndExt(t *testing.T) {
c := qt.New(t)
- f, err := FromStringAndExt("text/html", "html")
+ f, err := FromStringAndExt("text/html", "html", "htm")
c.Assert(err, qt.IsNil)
c.Assert(f, qt.Equals, Builtin.HTMLType)
- f, err = FromStringAndExt("text/html", ".html")
+ f, err = FromStringAndExt("text/html", ".html", ".htm")
c.Assert(err, qt.IsNil)
c.Assert(f, qt.Equals, Builtin.HTMLType)
}
}
}
+
+func TestIsContentFile(t *testing.T) {
+ c := qt.New(t)
+
+ c.Assert(DefaultContentTypes.IsContentFile(filepath.FromSlash("my/file.md")), qt.Equals, true)
+ c.Assert(DefaultContentTypes.IsContentFile(filepath.FromSlash("my/file.ad")), qt.Equals, true)
+ c.Assert(DefaultContentTypes.IsContentFile(filepath.FromSlash("textfile.txt")), qt.Equals, false)
+}
}
err = InterfaceToConfig(in, format, w)
-
if err != nil {
return err
}
// PageNop implements Page, but does nothing.
type nopPage int
-var noOpPathInfo = paths.Parse(files.ComponentFolderContent, "no-op.md")
+var noOpPathInfo = media.DefaultPathParser.Parse(files.ComponentFolderContent, "no-op.md")
func (p *nopPage) Err() resource.ResourceError {
return nil
package pagemeta
import (
+ "errors"
+ "fmt"
+ "path"
"strings"
"time"
+ "github.com/gohugoio/hugo/common/hreflect"
"github.com/gohugoio/hugo/common/htime"
+ "github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/paths"
+ "github.com/gohugoio/hugo/hugofs/files"
+ "github.com/gohugoio/hugo/markup"
+ "github.com/gohugoio/hugo/media"
+ "github.com/gohugoio/hugo/resources/kinds"
"github.com/gohugoio/hugo/resources/page"
+ "github.com/gohugoio/hugo/resources/resource"
"github.com/gohugoio/hugo/helpers"
"github.com/spf13/cast"
)
+type DatesStrings struct {
+ Date string `json:"date"`
+ Lastmod string `json:"lastMod"`
+ PublishDate string `json:"publishDate"`
+ ExpiryDate string `json:"expiryDate"`
+}
+
type Dates struct {
Date time.Time
Lastmod time.Time
// Note that all the top level fields are reserved Hugo keywords.
// Any custom configuration needs to be set in the Params map.
type PageConfig struct {
- Dates // Dates holds the four core dates for this page.
+ Dates Dates `json:"-"` // Dates holds the four core dates for this page.
+ DatesStrings
Title string // The title of the page.
LinkTitle string // The link title of the page.
Type string // The content type of the page.
Layout string // The layout to use for to render this page.
- Markup string // The markup used in the content file.
Weight int // The weight of the page, used in sorting if set to a non-zero value.
Kind string // The kind of page, e.g. "page", "section", "home" etc. This is usually derived from the content path.
Path string // The canonical path to the page, e.g. /sect/mypage. Note: Leading slash, no trailing slash, no extensions or language identifiers.
- URL string // The URL to the rendered page, e.g. /sect/mypage.html.
Lang string // The language code for this page. This is usually derived from the module mount or filename.
+ URL string // The URL to the rendered page, e.g. /sect/mypage.html.
Slug string // The slug for this page.
Description string // The description for this page.
Summary string // The summary for this page.
Draft bool // Whether or not the content is a draft.
- Headless bool // Whether or not the page should be rendered.
+ Headless bool `json:"-"` // Whether or not the page should be rendered.
IsCJKLanguage bool // Whether or not the content is in a CJK language.
TranslationKey string // The translation key for this page.
Keywords []string // The keywords for this page.
Aliases []string // The aliases for this page.
Outputs []string // The output formats to render this page in. If not set, the site's configured output formats for this page kind will be used.
- // These build options are set in the front matter,
- // but not passed on to .Params.
- Resources []map[string]any
- Cascade map[page.PageMatcher]maps.Params // Only relevant for branch nodes.
- Sitemap config.SitemapConfig
- Build BuildConfig
+ FrontMatterOnlyValues `mapstructure:"-" json:"-"`
+
+ Cascade []map[string]any
+ Sitemap config.SitemapConfig
+ Build BuildConfig
// User defined params.
Params maps.Params
+ // Content holds the content for this page.
+ Content Source
+
+ // Compiled values.
+ CascadeCompiled map[page.PageMatcher]maps.Params
+ ContentMediaType media.Type `mapstructure:"-" json:"-"`
+ IsFromContentAdapter bool `mapstructure:"-" json:"-"`
+}
+
+var DefaultPageConfig = PageConfig{
+ Build: DefaultBuildConfig,
+}
+
+func (p *PageConfig) Validate(pagesFromData bool) error {
+ if pagesFromData {
+ if p.Path == "" {
+ return errors.New("path must be set")
+ }
+ if strings.HasPrefix(p.Path, "/") {
+ return fmt.Errorf("path %q must not start with a /", p.Path)
+ }
+ if p.Lang != "" {
+ return errors.New("lang must not be set")
+ }
+
+ if p.Content.Markup != "" {
+ return errors.New("markup must not be set, use mediaType")
+ }
+ }
+
+ if p.Cascade != nil {
+ if !kinds.IsBranch(p.Kind) {
+ return errors.New("cascade is only supported for branch nodes")
+ }
+ }
+
+ return nil
+}
+
+// Compile sets up the page configuration after all fields have been set.
+func (p *PageConfig) Compile(basePath string, pagesFromData bool, ext string, logger loggers.Logger, mediaTypes media.Types) error {
+ // In content adapters, we always get relative paths.
+ if basePath != "" {
+ p.Path = path.Join(basePath, p.Path)
+ }
+
+ if pagesFromData {
+ // Note that NormalizePathStringBasic will make sure that we don't preserve the unnormalized path.
+ // We do that when we create pages from the file system; mostly for backward compatibility,
+ // but also because people tend to use use the filename to name their resources (with spaces and all),
+ // and this isn't relevant when creating resources from an API where it's easy to add textual meta data.
+ p.Path = paths.NormalizePathStringBasic(p.Path)
+ }
+
+ if p.Content.Markup == "" && p.Content.MediaType == "" {
+ if ext == "" {
+ ext = "md"
+ }
+ p.ContentMediaType = MarkupToMediaType(ext, mediaTypes)
+ if p.ContentMediaType.IsZero() {
+ return fmt.Errorf("failed to resolve media type for suffix %q", ext)
+ }
+ }
+
+ var s string
+ if p.ContentMediaType.IsZero() {
+ if p.Content.MediaType != "" {
+ s = p.Content.MediaType
+ p.ContentMediaType, _ = mediaTypes.GetByType(s)
+ }
+
+ if p.ContentMediaType.IsZero() && p.Content.Markup != "" {
+ s = p.Content.Markup
+ p.ContentMediaType = MarkupToMediaType(s, mediaTypes)
+ }
+ }
+
+ if p.ContentMediaType.IsZero() {
+ return fmt.Errorf("failed to resolve media type for %q", s)
+ }
+
+ if p.Content.Markup == "" {
+ p.Content.Markup = p.ContentMediaType.SubType
+ }
+
+ if p.Cascade != nil {
+ cascade, err := page.DecodeCascade(logger, p.Cascade)
+ if err != nil {
+ return fmt.Errorf("failed to decode cascade: %w", err)
+ }
+ p.CascadeCompiled = cascade
+ }
+
+ return nil
+}
+
+// MarkupToMediaType converts a markup string to a media type.
+func MarkupToMediaType(s string, mediaTypes media.Types) media.Type {
+ s = strings.ToLower(s)
+ mt, _ := mediaTypes.GetBestMatch(markup.ResolveMarkup(s))
+ return mt
+}
+
+type ResourceConfig struct {
+ Path string
+ Name string
+ Title string
+ Params maps.Params
+ Content Source
+
// Compiled values.
- IsGoldmark bool `json:"-"`
+ PathInfo *paths.Path `mapstructure:"-" json:"-"`
+ ContentMediaType media.Type
+}
+
+func (rc *ResourceConfig) Validate() error {
+ if rc.Path == "" {
+ return errors.New("path must be set")
+ }
+ if rc.Content.Markup != "" {
+ return errors.New("markup must not be set, use mediaType")
+ }
+ return nil
+}
+
+func (rc *ResourceConfig) Compile(basePath string, pathParser *paths.PathParser, mediaTypes media.Types) error {
+ if rc.Params != nil {
+ maps.PrepareParams(rc.Params)
+ }
+
+ // Note that NormalizePathStringBasic will make sure that we don't preserve the unnormalized path.
+ // We do that when we create resources from the file system; mostly for backward compatibility,
+ // but also because people tend to use use the filename to name their resources (with spaces and all),
+ // and this isn't relevant when creating resources from an API where it's easy to add textual meta data.
+ rc.Path = paths.NormalizePathStringBasic(path.Join(basePath, rc.Path))
+ rc.PathInfo = pathParser.Parse(files.ComponentFolderContent, rc.Path)
+ if rc.Content.MediaType != "" {
+ var found bool
+ rc.ContentMediaType, found = mediaTypes.GetByType(rc.Content.MediaType)
+ if !found {
+ return fmt.Errorf("media type %q not found", rc.Content.MediaType)
+ }
+ }
+ return nil
+}
+
+type Source struct {
+ // MediaType is the media type of the content.
+ MediaType string
+
+ // The markup used in Value. Only used in front matter.
+ Markup string
+
+ // The content.
+ Value any
+}
+
+func (s Source) IsZero() bool {
+ return !hreflect.IsTruthful(s.Value)
+}
+
+func (s Source) IsResourceValue() bool {
+ _, ok := s.Value.(resource.Resource)
+ return ok
+}
+
+func (s Source) ValueAsString() string {
+ if s.Value == nil {
+ return ""
+ }
+ ss, err := cast.ToStringE(s.Value)
+ if err != nil {
+ panic(fmt.Errorf("content source: failed to convert %T to string: %s", s.Value, err))
+ }
+ return ss
+}
+
+func (s Source) ValueAsOpenReadSeekCloser() hugio.OpenReadSeekCloser {
+ return hugio.NewOpenReadSeekCloser(hugio.NewReadSeekerNoOpCloserFromString(s.ValueAsString()))
+}
+
+// FrontMatterOnlyValues holds values that can only be set via front matter.
+type FrontMatterOnlyValues struct {
+ ResourcesMeta []map[string]any
}
// FrontMatterHandler maps front matter into Page fields and .Params.
type FrontMatterHandler struct {
fmConfig FrontmatterConfig
+ contentAdapterDatesHandler func(d *FrontMatterDescriptor) error
+
dateHandler frontMatterFieldHandler
lastModHandler frontMatterFieldHandler
publishDateHandler frontMatterFieldHandler
panic("missing pageConfig")
}
+ if d.PageConfig.IsFromContentAdapter {
+ if f.contentAdapterDatesHandler == nil {
+ panic("missing content adapter date handler")
+ }
+ return f.contentAdapterDatesHandler(d)
+ }
+
if f.dateHandler == nil {
panic("missing date handler")
}
func (f *FrontMatterHandler) createHandlers() error {
var err error
+ if f.contentAdapterDatesHandler, err = f.createContentAdapterDatesHandler(f.fmConfig); err != nil {
+ return err
+ }
+
if f.dateHandler, err = f.createDateHandler(f.fmConfig.Date,
func(d *FrontMatterDescriptor, t time.Time) {
- d.PageConfig.Date = t
+ d.PageConfig.Dates.Date = t
setParamIfNotSet(fmDate, t, d)
}); err != nil {
return err
if f.lastModHandler, err = f.createDateHandler(f.fmConfig.Lastmod,
func(d *FrontMatterDescriptor, t time.Time) {
setParamIfNotSet(fmLastmod, t, d)
- d.PageConfig.Lastmod = t
+ d.PageConfig.Dates.Lastmod = t
}); err != nil {
return err
}
if f.publishDateHandler, err = f.createDateHandler(f.fmConfig.PublishDate,
func(d *FrontMatterDescriptor, t time.Time) {
setParamIfNotSet(fmPubDate, t, d)
- d.PageConfig.PublishDate = t
+ d.PageConfig.Dates.PublishDate = t
}); err != nil {
return err
}
if f.expiryDateHandler, err = f.createDateHandler(f.fmConfig.ExpiryDate,
func(d *FrontMatterDescriptor, t time.Time) {
setParamIfNotSet(fmExpiryDate, t, d)
- d.PageConfig.ExpiryDate = t
+ d.PageConfig.Dates.ExpiryDate = t
}); err != nil {
return err
}
d.PageConfig.Params[key] = value
}
+func (f FrontMatterHandler) createContentAdapterDatesHandler(fmcfg FrontmatterConfig) (func(d *FrontMatterDescriptor) error, error) {
+ setTime := func(key string, value time.Time, in *PageConfig) {
+ switch key {
+ case fmDate:
+ in.Dates.Date = value
+ case fmLastmod:
+ in.Dates.Lastmod = value
+ case fmPubDate:
+ in.Dates.PublishDate = value
+ case fmExpiryDate:
+ in.Dates.ExpiryDate = value
+ }
+ }
+
+ getTime := func(key string, in *PageConfig) time.Time {
+ switch key {
+ case fmDate:
+ return in.Dates.Date
+ case fmLastmod:
+ return in.Dates.Lastmod
+ case fmPubDate:
+ return in.Dates.PublishDate
+ case fmExpiryDate:
+ return in.Dates.ExpiryDate
+ }
+ return time.Time{}
+ }
+
+ createSetter := func(identifiers []string, date string) func(pcfg *PageConfig) {
+ var getTimes []func(in *PageConfig) time.Time
+ for _, identifier := range identifiers {
+ if strings.HasPrefix(identifier, ":") {
+ continue
+ }
+ switch identifier {
+ case fmDate:
+ getTimes = append(getTimes, func(in *PageConfig) time.Time {
+ return getTime(fmDate, in)
+ })
+ case fmLastmod:
+ getTimes = append(getTimes, func(in *PageConfig) time.Time {
+ return getTime(fmLastmod, in)
+ })
+ case fmPubDate:
+ getTimes = append(getTimes, func(in *PageConfig) time.Time {
+ return getTime(fmPubDate, in)
+ })
+ case fmExpiryDate:
+ getTimes = append(getTimes, func(in *PageConfig) time.Time {
+ return getTime(fmExpiryDate, in)
+ })
+ }
+ }
+
+ return func(pcfg *PageConfig) {
+ for _, get := range getTimes {
+ if t := get(pcfg); !t.IsZero() {
+ setTime(date, t, pcfg)
+ return
+ }
+ }
+ }
+ }
+
+ setDate := createSetter(fmcfg.Date, fmDate)
+ setLastmod := createSetter(fmcfg.Lastmod, fmLastmod)
+ setPublishDate := createSetter(fmcfg.PublishDate, fmPubDate)
+ setExpiryDate := createSetter(fmcfg.ExpiryDate, fmExpiryDate)
+
+ fn := func(d *FrontMatterDescriptor) error {
+ pcfg := d.PageConfig
+ setDate(pcfg)
+ setLastmod(pcfg)
+ setPublishDate(pcfg)
+ setExpiryDate(pcfg)
+ return nil
+ }
+ return fn, nil
+}
+
func (f FrontMatterHandler) createDateHandler(identifiers []string, setter func(d *FrontMatterDescriptor, t time.Time)) (frontMatterFieldHandler, error) {
var h *frontmatterFieldHandlers
var handlers []frontMatterFieldHandler
"testing"
"time"
+ "github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/config/testconfig"
+ "github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/resources/page/pagemeta"
c.Assert(d.PageConfig.Dates.PublishDate.Day(), qt.Equals, 4)
c.Assert(d.PageConfig.Dates.ExpiryDate.IsZero(), qt.Equals, true)
}
+
+func TestContentMediaTypeFromMarkup(t *testing.T) {
+ c := qt.New(t)
+ logger := loggers.NewDefault()
+
+ for _, test := range []struct {
+ in string
+ expected string
+ }{
+ {"", "text/markdown"},
+ {"md", "text/markdown"},
+ {"markdown", "text/markdown"},
+ {"mdown", "text/markdown"},
+ {"goldmark", "text/markdown"},
+ {"html", "text/html"},
+ {"htm", "text/html"},
+ {"asciidoc", "text/asciidoc"},
+ {"asciidocext", "text/asciidoc"},
+ {"adoc", "text/asciidoc"},
+ {"pandoc", "text/pandoc"},
+ {"pdc", "text/pandoc"},
+ {"rst", "text/rst"},
+ } {
+ var pc pagemeta.PageConfig
+ pc.Content.Markup = test.in
+ c.Assert(pc.Compile("", true, "", logger, media.DefaultTypes), qt.IsNil)
+ c.Assert(pc.ContentMediaType.Type, qt.Equals, test.expected)
+ }
+}
Link = "link"
)
-var defaultBuildConfig = BuildConfig{
+var DefaultBuildConfig = BuildConfig{
List: Always,
Render: Always,
PublishResources: true,
}
func DecodeBuildConfig(m any) (BuildConfig, error) {
- b := defaultBuildConfig
+ b := DefaultBuildConfig
if m == nil {
return b, nil
}
"SuffixesCSV": "pre_foo.SuffixesCSV_post",
"Delimiter": "pre_foo.Delimiter_post",
"FirstSuffix": "pre_foo.FirstSuffix_post",
+ "IsHTML": "pre_foo.IsHTML_post",
+ "IsMarkdown": "pre_foo.IsMarkdown_post",
"IsText": "pre_foo.IsText_post",
"String": "pre_foo.String_post",
"Type": "pre_foo.Type_post",
// The name of the resource as it was read from the source.
NameOriginal string
+ // The title of the resource.
+ Title string
+
// Any base paths prepended to the target path. This will also typically be the
// language code, but setting it here means that it should not have any effect on
// the permalink.
// The Data to associate with this resource.
Data map[string]any
+ // The Params to associate with this resource.
+ Params maps.Params
+
// Delay publishing until either Permalink or RelPermalink is called. Maybe never.
LazyPublish bool
panic(errors.New("RelPath is empty"))
}
+ if fd.Params == nil {
+ fd.Params = make(maps.Params)
+ }
+
if fd.Path == nil {
- fd.Path = paths.Parse("", fd.TargetPath)
+ fd.Path = r.Cfg.PathParser().Parse("", fd.TargetPath)
}
if fd.TargetPath == "" {
fd.NameOriginal = fd.NameNormalized
}
+ if fd.Title == "" {
+ fd.Title = fd.NameOriginal
+ }
+
mediaType := fd.MediaType
if mediaType.IsZero() {
ext := fd.Path.Ext()
// Resource represents a linkable resource, i.e. a content page, image etc.
type Resource interface {
+ ResourceWithoutMeta
+ ResourceMetaProvider
+}
+
+type ResourceWithoutMeta interface {
ResourceTypeProvider
MediaTypeProvider
ResourceLinksProvider
- ResourceNameTitleProvider
- ResourceParamsProvider
ResourceDataProvider
ErrProvider
}
+type ResourceWrapper interface {
+ UnwrappedResource() Resource
+ WrapResource(Resource) ResourceWrapper
+}
+
type ResourceTypeProvider interface {
// ResourceType is the resource type. For most file types, this is the main
// part of the MIME type, e.g. "image", "application", "text" etc.
"github.com/gohugoio/hugo/hugofs/glob"
"github.com/gohugoio/hugo/media"
+ "github.com/gohugoio/hugo/resources/page/pagemeta"
"github.com/gohugoio/hugo/resources/resource"
"github.com/spf13/cast"
r.changed = true
}
-func CloneWithMetadataIfNeeded(m []map[string]any, r resource.Resource) resource.Resource {
+// cloneWithMetadataFromResourceConfigIfNeeded clones the given resource with the given metadata if the resource supports it.
+func cloneWithMetadataFromResourceConfigIfNeeded(rc *pagemeta.ResourceConfig, r resource.Resource) resource.Resource {
+ wmp, ok := r.(resource.WithResourceMetaProvider)
+ if !ok {
+ return r
+ }
+
+ if rc.Name == "" && rc.Title == "" && len(rc.Params) == 0 {
+ // No metadata.
+ return r
+ }
+
+ if rc.Title == "" {
+ rc.Title = rc.Name
+ }
+
+ wrapped := &metaResource{
+ name: rc.Name,
+ title: rc.Title,
+ params: rc.Params,
+ }
+
+ return wmp.WithResourceMeta(wrapped)
+}
+
+// CloneWithMetadataFromMapIfNeeded clones the given resource with the given metadata if the resource supports it.
+func CloneWithMetadataFromMapIfNeeded(m []map[string]any, r resource.Resource) resource.Resource {
wmp, ok := r.(resource.WithResourceMetaProvider)
if !ok {
return r
package resources
import (
+ "fmt"
"path"
"sync"
"github.com/gohugoio/hugo/output"
"github.com/gohugoio/hugo/resources/internal"
"github.com/gohugoio/hugo/resources/jsconfig"
+ "github.com/gohugoio/hugo/resources/page/pagemeta"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/hexec"
JSConfigBuilder *jsconfig.Builder
}
+func (r *Spec) NewResourceWrapperFromResourceConfig(rc *pagemeta.ResourceConfig) (resource.Resource, error) {
+ content := rc.Content
+ switch r := content.Value.(type) {
+ case resource.Resource:
+ return cloneWithMetadataFromResourceConfigIfNeeded(rc, r), nil
+ default:
+ return nil, fmt.Errorf("failed to create resource for path %q, expected a resource.Resource, got %T", rc.PathInfo.Path(), content.Value)
+ }
+}
+
// NewResource creates a new Resource from the given ResourceSourceDescriptor.
func (r *Spec) NewResource(rd ResourceSourceDescriptor) (resource.Resource, error) {
if err := rd.init(r); err != nil {
paths: rp,
spec: r,
sd: rd,
- params: make(map[string]any),
+ params: rd.Params,
name: rd.NameOriginal,
- title: rd.NameOriginal,
+ title: rd.Title,
}
if rd.MediaType.MainType == "image" {
return r.getImageOps().Filter(filters...)
}
+func (r *resourceAdapter) Resize(spec string) (images.ImageResource, error) {
+ return r.getImageOps().Resize(spec)
+}
+
func (r *resourceAdapter) Height() int {
return r.getImageOps().Height()
}
return r.target.RelPermalink()
}
-func (r *resourceAdapter) Resize(spec string) (images.ImageResource, error) {
- return r.getImageOps().Resize(spec)
-}
-
func (r *resourceAdapter) ResourceType() string {
r.init(false, false)
return r.target.ResourceType()
"github.com/bep/gitmap"
"github.com/gohugoio/hugo/common/hugo"
"github.com/gohugoio/hugo/common/paths"
+ "github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/common/hugio"
lazyInit sync.Once
}
+// IsContentAdapter returns whether the file represents a content adapter.
+// This means that there may be more than one Page associated with this file.
+func (fi *File) IsContentAdapter() bool {
+ return fi.fim.Meta().PathInfo.IsContentData()
+}
+
// Filename returns a file's absolute path and filename on disk.
func (fi *File) Filename() string { return fi.fim.Meta().Filename }
func NewFileInfoFrom(path, filename string) *File {
meta := &hugofs.FileMeta{
Filename: filename,
- PathInfo: paths.Parse("", filepath.ToSlash(path)),
+ PathInfo: media.DefaultPathParser.Parse("", filepath.ToSlash(path)),
}
return NewFileInfo(hugofs.NewFileMetaInfo(nil, meta))
TxtTmpl TemplateParseFinder
}
+type TemplateExecutor interface {
+ ExecuteWithContext(ctx context.Context, t Template, wr io.Writer, data any) error
+}
+
// TemplateHandler finds and executes templates.
type TemplateHandler interface {
TemplateFinder
- ExecuteWithContext(ctx context.Context, t Template, wr io.Writer, data any) error
+ TemplateExecutor
LookupLayout(d layouts.LayoutDescriptor, f output.Format) (Template, bool, error)
HasTemplate(name string) bool
GetIdentity(name string) (identity.Identity, bool)
projects / brevno-suite / hugo / commitdiff