From: Bjørn Erik Pedersen Date: Wed, 5 Nov 2025 14:52:47 +0000 (+0100) Subject: Add roles and versions as new dimensions (in addition to language) X-Git-Url: http://git.maquefel.me/?a=commitdiff_plain;h=264022a75a8e4d91d7461747eef4d06267cc519f;p=brevno-suite%2Fhugo Add roles and versions as new dimensions (in addition to language) See the main issue #13776 for details. Fixes #519 Fixes #13680 Fixes #13663 Fixes #13776 Fixes #13855 Fixes #13648 Fixes #13996 Fixes #14001 Fixes #14031 Fixes #13818 Fixes #13196 --- diff --git a/cache/dynacache/dynacache.go b/cache/dynacache/dynacache.go index 25d0f9b29..56070544c 100644 --- a/cache/dynacache/dynacache.go +++ b/cache/dynacache/dynacache.go @@ -340,7 +340,7 @@ func GetOrCreatePartition[K comparable, V any](c *Cache, name string, opts Optio panic("invalid Weight, must be between 1 and 100") } - if partitionNameRe.FindString(name) != name { + if !partitionNameRe.MatchString(name) { panic(fmt.Sprintf("invalid partition name %q", name)) } diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go index a30aaa50b..c1d79dfa8 100644 --- a/cache/filecache/filecache_test.go +++ b/cache/filecache/filecache_test.go @@ -270,7 +270,7 @@ func newPathsSpec(t *testing.T, fs afero.Fs, configStr string) *helpers.PathSpec cfg, err := config.FromConfigString(configStr, "toml") c.Assert(err, qt.IsNil) acfg := testconfig.GetTestConfig(fs, cfg) - p, err := helpers.NewPathSpec(hugofs.NewFrom(fs, acfg.BaseConfig()), acfg, nil) + p, err := helpers.NewPathSpec(hugofs.NewFrom(fs, acfg.BaseConfig()), acfg, nil, nil) c.Assert(err, qt.IsNil) return p } diff --git a/cache/httpcache/httpcache.go b/cache/httpcache/httpcache.go index 010b12e57..44f243bd3 100644 --- a/cache/httpcache/httpcache.go +++ b/cache/httpcache/httpcache.go @@ -173,16 +173,16 @@ func (gm *GlobMatcher) CompilePredicate() (func(string) bool, error) { if gm.IsZero() { panic("no includes or excludes") } - var p predicate.P[string] + var b predicate.PR[string] for _, include := range gm.Includes { g, err := glob.Compile(include, '/') if err != nil { return nil, err } - fn := func(s string) bool { - return g.Match(s) + fn := func(s string) predicate.Match { + return predicate.BoolMatch(g.Match(s)) } - p = p.Or(fn) + b = b.Or(fn) } for _, exclude := range gm.Excludes { @@ -190,13 +190,13 @@ func (gm *GlobMatcher) CompilePredicate() (func(string) bool, error) { if err != nil { return nil, err } - fn := func(s string) bool { - return !g.Match(s) + fn := func(s string) predicate.Match { + return predicate.BoolMatch(!g.Match(s)) } - p = p.And(fn) + b = b.And(fn) } - return p, nil + return b.BoolFunc(), nil } func DecodeConfig(_ config.BaseConfig, m map[string]any) (Config, error) { diff --git a/commands/config.go b/commands/config.go index 7d166b9b8..a44378b75 100644 --- a/commands/config.go +++ b/commands/config.go @@ -25,6 +25,7 @@ import ( "github.com/bep/simplecobra" "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/config/allconfig" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "github.com/gohugoio/hugo/modules" "github.com/gohugoio/hugo/parser" "github.com/gohugoio/hugo/parser/metadecoders" @@ -71,7 +72,7 @@ func (c *configCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, arg return fmt.Errorf("language %q not found", c.lang) } } else { - config = conf.configs.LanguageConfigSlice[0] + config = conf.configs.LanguageConfigMap[conf.configs.Base.DefaultContentLanguage] } var buf bytes.Buffer @@ -128,9 +129,9 @@ func (c *configCommand) PreRun(cd, runner *simplecobra.Commandeer) error { } type configModMount struct { - Source string `json:"source"` - Target string `json:"target"` - Lang string `json:"lang,omitempty"` + Source string `json:"source"` + Target string `json:"target"` + Sites sitesmatrix.Sites `json:"sites,omitzero"` } type configModMounts struct { @@ -146,7 +147,7 @@ func (m *configModMounts) MarshalJSON() ([]byte, error) { mounts = append(mounts, configModMount{ Source: mount.Source, Target: mount.Target, - Lang: mount.Lang, + Sites: mount.Sites, }) } diff --git a/commands/hugobuilder.go b/commands/hugobuilder.go index 3ca088985..ecca6ecc7 100644 --- a/commands/hugobuilder.go +++ b/commands/hugobuilder.go @@ -31,6 +31,7 @@ import ( "github.com/bep/simplecobra" "github.com/fsnotify/fsnotify" "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hstrings" "github.com/gohugoio/hugo/common/htime" "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/common/loggers" @@ -143,7 +144,7 @@ func (c *hugoBuilder) getDirList() ([]string, error) { return nil, err } - return helpers.UniqueStringsSorted(h.PathSpec.BaseFs.WatchFilenames()), nil + return hstrings.UniqueStringsSorted(h.PathSpec.BaseFs.WatchFilenames()), nil } func (c *hugoBuilder) initCPUProfile() (func(), error) { @@ -827,7 +828,7 @@ func (c *hugoBuilder) handleEvents(watcher *watcher.Batcher, continue } - walkAdder := func(path string, f hugofs.FileMetaInfo) error { + walkAdder := func(ctx context.Context, path string, f hugofs.FileMetaInfo) error { if f.IsDir() { c.r.logger.Println("adding created directory to watchlist", path) if err := watcher.Add(path); err != nil { diff --git a/commands/import.go b/commands/import.go index 37a6b0dbf..9da2b39e5 100644 --- a/commands/import.go +++ b/commands/import.go @@ -427,7 +427,7 @@ func (c *importCommand) importFromJekyll(args []string) error { c.r.Println("Importing...") fileCount := 0 - callback := func(path string, fi hugofs.FileMetaInfo) error { + callback := func(ctx context.Context, path string, fi hugofs.FileMetaInfo) error { if fi.IsDir() { return nil } diff --git a/commands/server.go b/commands/server.go index ddef4493f..82b18c608 100644 --- a/commands/server.go +++ b/commands/server.go @@ -49,6 +49,7 @@ import ( "github.com/fsnotify/fsnotify" "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/hugo" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/tpl/tplimpl" "github.com/gohugoio/hugo/common/types" @@ -627,7 +628,7 @@ func (c *serverCommand) setServerInfoInConfig() error { panic("no server ports set") } return c.withConfE(func(conf *commonConfig) error { - for i, language := range conf.configs.LanguagesDefaultFirst { + for i, language := range conf.configs.Languages { isMultihost := conf.configs.IsMultihost var serverPort int if isMultihost { @@ -879,7 +880,7 @@ func (c *serverCommand) serve() error { if isMultihost { for _, l := range conf.configs.ConfigLangs() { baseURLs = append(baseURLs, l.BaseURL()) - roots = append(roots, l.Language().Lang) + roots = append(roots, l.Language().(*langs.Language).Lang) } } else { l := conf.configs.GetFirstLanguageConfig() diff --git a/common/hashing/hashing.go b/common/hashing/hashing.go index e45356758..793e25a8c 100644 --- a/common/hashing/hashing.go +++ b/common/hashing/hashing.go @@ -23,6 +23,7 @@ import ( "github.com/cespare/xxhash/v2" "github.com/gohugoio/hashstructure" + "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/identity" ) @@ -38,6 +39,34 @@ func XXHashFromReader(r io.Reader) (uint64, int64, error) { return h.Sum64(), size, nil } +type Hasher interface { + io.StringWriter + io.Writer + io.ReaderFrom + Sum64() uint64 +} + +type HashCloser interface { + Hasher + io.Closer +} + +// XxHasher returns a Hasher that uses xxHash. +// Remember to call Close when done. +func XxHasher() HashCloser { + h := getXxHashReadFrom() + return struct { + Hasher + io.Closer + }{ + Hasher: h, + Closer: hugio.CloserFunc(func() error { + putXxHashReadFrom(h) + return nil + }), + } +} + // XxHashFromReaderHexEncoded calculates the xxHash for the given reader // and returns the hash as a hex encoded string. func XxHashFromReaderHexEncoded(r io.Reader) (string, error) { diff --git a/common/herrors/errors.go b/common/herrors/errors.go index 693ff024c..5e223f42f 100644 --- a/common/herrors/errors.go +++ b/common/herrors/errors.go @@ -19,7 +19,7 @@ import ( "fmt" "os" "regexp" - "runtime/debug" + "runtime" "strings" "time" ) @@ -36,7 +36,9 @@ type ErrorSender interface { func Recover(args ...any) { if r := recover(); r != nil { fmt.Println("ERR:", r) - args = append(args, "stacktrace from panic: \n"+string(debug.Stack()), "\n") + buf := make([]byte, 64<<10) + buf = buf[:runtime.Stack(buf, false)] + args = append(args, "stacktrace from panic: \n"+string(buf), "\n") fmt.Println(args...) } } diff --git a/common/hiter/iter.go b/common/hiter/iter.go new file mode 100644 index 000000000..dd221e18c --- /dev/null +++ b/common/hiter/iter.go @@ -0,0 +1,40 @@ +package hiter + +// Common iterator functions. +// Some of these are are based on this discsussion: https://github.com/golang/go/issues/61898 + +import "iter" + +// Concat returns an iterator over the concatenation of the sequences. +// Any nil sequences are ignored. +func Concat[V any](seqs ...iter.Seq[V]) iter.Seq[V] { + return func(yield func(V) bool) { + for _, seq := range seqs { + if seq == nil { + continue + } + for e := range seq { + if !yield(e) { + return + } + } + } + } +} + +// Concat2 returns an iterator over the concatenation of the sequences. +// Any nil sequences are ignored. +func Concat2[K, V any](seqs ...iter.Seq2[K, V]) iter.Seq2[K, V] { + return func(yield func(K, V) bool) { + for _, seq := range seqs { + if seq == nil { + continue + } + for k, v := range seq { + if !yield(k, v) { + return + } + } + } + } +} diff --git a/common/hstrings/strings.go b/common/hstrings/strings.go index 1de38678f..e4ffd64f6 100644 --- a/common/hstrings/strings.go +++ b/common/hstrings/strings.go @@ -17,6 +17,7 @@ import ( "fmt" "regexp" "slices" + "sort" "strings" "sync" @@ -128,7 +129,61 @@ func ToString(v any) (string, bool) { return "", false } -type ( - Strings2 [2]string - Strings3 [3]string -) +// UniqueStrings returns a new slice with any duplicates removed. +func UniqueStrings(s []string) []string { + unique := make([]string, 0, len(s)) + for i, val := range s { + var seen bool + for j := range i { + if s[j] == val { + seen = true + break + } + } + if !seen { + unique = append(unique, val) + } + } + return unique +} + +// UniqueStringsReuse returns a slice with any duplicates removed. +// It will modify the input slice. +func UniqueStringsReuse(s []string) []string { + result := s[:0] + for i, val := range s { + var seen bool + + for j := range i { + if s[j] == val { + seen = true + break + } + } + + if !seen { + result = append(result, val) + } + } + return result +} + +// UniqueStringsSorted returns a sorted slice with any duplicates removed. +// It will modify the input slice. +func UniqueStringsSorted(s []string) []string { + if len(s) == 0 { + return nil + } + ss := sort.StringSlice(s) + ss.Sort() + i := 0 + for j := 1; j < len(s); j++ { + if !ss.Less(i, j) { + continue + } + i++ + s[i] = s[j] + } + + return s[:i+1] +} diff --git a/common/hstrings/strings_test.go b/common/hstrings/strings_test.go index d8e9e204a..8c485130e 100644 --- a/common/hstrings/strings_test.go +++ b/common/hstrings/strings_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Hugo Authors. All rights reserved. +// Copyright 2025 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ package hstrings import ( + "reflect" "regexp" "testing" @@ -43,6 +44,84 @@ func TestGetOrCompileRegexp(t *testing.T) { c.Assert(re.MatchString("123"), qt.Equals, true) } +func TestUniqueStrings(t *testing.T) { + in := []string{"a", "b", "a", "b", "c", "", "a", "", "d"} + output := UniqueStrings(in) + expected := []string{"a", "b", "c", "", "d"} + if !reflect.DeepEqual(output, expected) { + t.Errorf("Expected %#v, got %#v\n", expected, output) + } +} + +func TestUniqueStringsReuse(t *testing.T) { + in := []string{"a", "b", "a", "b", "c", "", "a", "", "d"} + output := UniqueStringsReuse(in) + expected := []string{"a", "b", "c", "", "d"} + if !reflect.DeepEqual(output, expected) { + t.Errorf("Expected %#v, got %#v\n", expected, output) + } +} + +func TestUniqueStringsSorted(t *testing.T) { + c := qt.New(t) + in := []string{"a", "a", "b", "c", "b", "", "a", "", "d"} + output := UniqueStringsSorted(in) + expected := []string{"", "a", "b", "c", "d"} + c.Assert(output, qt.DeepEquals, expected) + c.Assert(UniqueStringsSorted(nil), qt.IsNil) +} + +func BenchmarkUniqueStrings(b *testing.B) { + input := []string{"a", "b", "d", "e", "d", "h", "a", "i"} + + b.Run("Safe", func(b *testing.B) { + for i := 0; i < b.N; i++ { + result := UniqueStrings(input) + if len(result) != 6 { + b.Fatalf("invalid count: %d", len(result)) + } + } + }) + + b.Run("Reuse slice", func(b *testing.B) { + b.StopTimer() + inputs := make([][]string, b.N) + for i := 0; i < b.N; i++ { + inputc := make([]string, len(input)) + copy(inputc, input) + inputs[i] = inputc + } + b.StartTimer() + for i := 0; i < b.N; i++ { + inputc := inputs[i] + + result := UniqueStringsReuse(inputc) + if len(result) != 6 { + b.Fatalf("invalid count: %d", len(result)) + } + } + }) + + b.Run("Reuse slice sorted", func(b *testing.B) { + b.StopTimer() + inputs := make([][]string, b.N) + for i := 0; i < b.N; i++ { + inputc := make([]string, len(input)) + copy(inputc, input) + inputs[i] = inputc + } + b.StartTimer() + for i := 0; i < b.N; i++ { + inputc := inputs[i] + + result := UniqueStringsSorted(inputc) + if len(result) != 6 { + b.Fatalf("invalid count: %d", len(result)) + } + } + }) +} + func BenchmarkGetOrCompileRegexp(b *testing.B) { for i := 0; i < b.N; i++ { GetOrCompileRegexp(`\d+`) diff --git a/common/hsync/oncemore.go b/common/hsync/oncemore.go new file mode 100644 index 000000000..cc951a215 --- /dev/null +++ b/common/hsync/oncemore.go @@ -0,0 +1,162 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hsync + +import ( + "context" + "sync" + "sync/atomic" +) + +// OnceMore is similar to sync.Once. +// +// Additional features are: +// * it can be reset, so the action can be repeated if needed +// * it has methods to check if it's done or in progress +type OnceMore struct { + _ doNotCopy + done atomic.Bool + mu sync.Mutex +} + +func (t *OnceMore) Do(f func()) { + if t.Done() { + return + } + + t.mu.Lock() + defer t.mu.Unlock() + + // Double check + if t.Done() { + return + } + + defer t.done.Store(true) + f() +} + +func (t *OnceMore) Done() bool { + return t.done.Load() +} + +func (t *OnceMore) Reset() { + t.mu.Lock() + t.done.Store(false) + t.mu.Unlock() +} + +type ValueResetter[T any] struct { + reset func() + f func(context.Context) T +} + +func (v *ValueResetter[T]) Value(ctx context.Context) T { + return v.f(ctx) +} + +func (v *ValueResetter[T]) Reset() { + v.reset() +} + +// OnceMoreValue returns a function that invokes f only once and returns the value +// returned by f. The returned function may be called concurrently. +// +// If f panics, the returned function will panic with the same value on every call. +func OnceMoreValue[T any](f func(context.Context) T) ValueResetter[T] { + v := struct { + f func(context.Context) T + once OnceMore + ok bool + p any + result T + }{ + f: f, + } + ff := func(ctx context.Context) T { + v.once.Do(func() { + v.ok = false + defer func() { + v.p = recover() + if !v.ok { + panic(v.p) + } + }() + v.result = v.f(ctx) + v.ok = true + }) + if !v.ok { + panic(v.p) + } + return v.result + } + + return ValueResetter[T]{ + reset: v.once.Reset, + f: ff, + } +} + +type FuncResetter struct { + f func(context.Context) error + reset func() +} + +func (v *FuncResetter) Do(ctx context.Context) error { + return v.f(ctx) +} + +func (v *FuncResetter) Reset() { + v.reset() +} + +func OnceMoreFunc(f func(context.Context) error) FuncResetter { + v := struct { + f func(context.Context) error + once OnceMore + ok bool + err error + p any + }{ + f: f, + } + ff := func(ctx context.Context) error { + v.once.Do(func() { + v.ok = false + defer func() { + v.p = recover() + if !v.ok { + panic(v.p) + } + }() + v.err = v.f(ctx) + v.ok = true + }) + if !v.ok { + panic(v.p) + } + return v.err + } + + return FuncResetter{ + f: ff, + reset: v.once.Reset, + } +} + +type doNotCopy struct{} + +// Lock is a no-op used by -copylocks checker from `go vet`. +func (*doNotCopy) Lock() {} +func (*doNotCopy) Unlock() {} diff --git a/common/hsync/oncemore_test.go b/common/hsync/oncemore_test.go new file mode 100644 index 000000000..e0b60418b --- /dev/null +++ b/common/hsync/oncemore_test.go @@ -0,0 +1,80 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hsync + +import ( + "context" + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestOnceMoreValue(t *testing.T) { + c := qt.New(t) + + var counter int + f := func(context.Context) int { + counter++ + return counter + } + + omf := OnceMoreValue(f) + for range 10 { + c.Assert(omf.Value(context.Background()), qt.Equals, 1) + } + omf.Reset() + for range 10 { + c.Assert(omf.Value(context.Background()), qt.Equals, 2) + } +} + +func TestOnceMoreFunc(t *testing.T) { + c := qt.New(t) + + var counter int + f := func(context.Context) error { + counter++ + return nil + } + + omf := OnceMoreFunc(f) + for range 10 { + c.Assert(omf.Do(context.Background()), qt.IsNil) + c.Assert(counter, qt.Equals, 1) + } + omf.Reset() + for range 10 { + c.Assert(omf.Do(context.Background()), qt.IsNil) + c.Assert(counter, qt.Equals, 2) + } +} + +func BenchmarkOnceMoreValue(b *testing.B) { + var counter int + f := func(context.Context) int { + counter++ + return counter + } + + for range b.N { + omf := OnceMoreValue(f) + for range 10 { + omf.Value(context.Background()) + } + omf.Reset() + for range 10 { + omf.Value(context.Background()) + } + } +} diff --git a/common/hugio/readers.go b/common/hugio/readers.go index c4304c84e..b1db09bac 100644 --- a/common/hugio/readers.go +++ b/common/hugio/readers.go @@ -32,6 +32,13 @@ type ReadSeekCloser interface { io.Closer } +// CloserFunc is an adapter to allow the use of ordinary functions as io.Closers. +type CloserFunc func() error + +func (f CloserFunc) Close() error { + return f() +} + // ReadSeekCloserProvider provides a ReadSeekCloser. type ReadSeekCloserProvider interface { ReadSeekCloser() (ReadSeekCloser, error) diff --git a/common/hugo/hugo.go b/common/hugo/hugo.go index d0345c3c4..173e9a66b 100644 --- a/common/hugo/hugo.go +++ b/common/hugo/hugo.go @@ -32,6 +32,7 @@ import ( "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/hstore" "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/common/version" "github.com/gohugoio/hugo/hugofs/files" "github.com/bep/helpers/contexthelpers" @@ -81,7 +82,7 @@ type HugoInfo struct { } // Version returns the current version as a comparable version string. -func (i HugoInfo) Version() VersionString { +func (i HugoInfo) Version() version.VersionString { return CurrentVersion.Version() } @@ -452,7 +453,7 @@ func deprecateLevelWithLogger(item, alternative, version string, level logg.Leve // We want people to run at least the current and previous version without any warnings. // We want people who don't update Hugo that often to see the warnings and errors before we remove the feature. func deprecationLogLevelFromVersion(ver string) logg.Level { - from := MustParseVersion(ver) + from := version.MustParseVersion(ver) to := CurrentVersion minorDiff := to.Minor - from.Minor switch { @@ -466,3 +467,46 @@ func deprecationLogLevelFromVersion(ver string) logg.Level { return logg.LevelInfo } } + +// BuildVersionString creates a version string. This is what you see when +// running "hugo version". +func BuildVersionString() string { + // program := "Hugo Static Site Generator" + program := "hugo" + + version := "v" + CurrentVersion.String() + + bi := getBuildInfo() + if bi == nil { + return version + } + if bi.Revision != "" { + version += "-" + bi.Revision + } + if IsExtended { + version += "+extended" + } + if IsWithdeploy { + version += "+withdeploy" + } + + osArch := bi.GoOS + "/" + bi.GoArch + + date := bi.RevisionTime + if date == "" { + // Accept vendor-specified build date if .git/ is unavailable. + date = buildDate + } + if date == "" { + date = "unknown" + } + + versionString := fmt.Sprintf("%s %s %s BuildDate=%s", + program, version, osArch, date) + + if vendorInfo != "" { + versionString += " VendorInfo=" + vendorInfo + } + + return versionString +} diff --git a/common/hugo/hugo_test.go b/common/hugo/hugo_test.go index f938073da..699142212 100644 --- a/common/hugo/hugo_test.go +++ b/common/hugo/hugo_test.go @@ -20,6 +20,7 @@ import ( "github.com/bep/logg" qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/common/version" ) func TestHugoInfo(t *testing.T) { @@ -29,7 +30,7 @@ func TestHugoInfo(t *testing.T) { hugoInfo := NewInfo(conf, nil) c.Assert(hugoInfo.Version(), qt.Equals, CurrentVersion.Version()) - c.Assert(fmt.Sprintf("%T", VersionString("")), qt.Equals, fmt.Sprintf("%T", hugoInfo.Version())) + c.Assert(fmt.Sprintf("%T", version.VersionString("")), qt.Equals, fmt.Sprintf("%T", hugoInfo.Version())) c.Assert(hugoInfo.WorkingDir(), qt.Equals, "/mywork") bi := getBuildInfo() diff --git a/common/hugo/version.go b/common/hugo/version.go deleted file mode 100644 index cf5988840..000000000 --- a/common/hugo/version.go +++ /dev/null @@ -1,305 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugo - -import ( - "fmt" - "io" - "math" - "runtime" - "strconv" - "strings" - - "github.com/gohugoio/hugo/compare" - "github.com/spf13/cast" -) - -// Version represents the Hugo build version. -type Version struct { - Major int - - Minor int - - // Increment this for bug releases - PatchLevel int - - // HugoVersionSuffix is the suffix used in the Hugo version string. - // It will be blank for release versions. - Suffix string -} - -var ( - _ compare.Eqer = (*VersionString)(nil) - _ compare.Comparer = (*VersionString)(nil) -) - -func (v Version) String() string { - return version(v.Major, v.Minor, v.PatchLevel, v.Suffix) -} - -// Version returns the Hugo version. -func (v Version) Version() VersionString { - return VersionString(v.String()) -} - -// Compare implements the compare.Comparer interface. -func (h Version) Compare(other any) int { - return compareVersions(h, other) -} - -// VersionString represents a Hugo version string. -type VersionString string - -func (h VersionString) String() string { - return string(h) -} - -// Compare implements the compare.Comparer interface. -func (h VersionString) Compare(other any) int { - return compareVersions(h.Version(), other) -} - -func (h VersionString) Version() Version { - return MustParseVersion(h.String()) -} - -// Eq implements the compare.Eqer interface. -func (h VersionString) Eq(other any) bool { - s, err := cast.ToStringE(other) - if err != nil { - return false - } - return s == h.String() -} - -var versionSuffixes = []string{"-test", "-DEV"} - -// ParseVersion parses a version string. -func ParseVersion(s string) (Version, error) { - var vv Version - for _, suffix := range versionSuffixes { - if strings.HasSuffix(s, suffix) { - vv.Suffix = suffix - s = strings.TrimSuffix(s, suffix) - } - } - - vv.Major, vv.Minor, vv.PatchLevel = parseVersion(s) - - return vv, nil -} - -// MustParseVersion parses a version string -// and panics if any error occurs. -func MustParseVersion(s string) Version { - vv, err := ParseVersion(s) - if err != nil { - panic(err) - } - return vv -} - -// ReleaseVersion represents the release version. -func (v Version) ReleaseVersion() Version { - v.Suffix = "" - return v -} - -// Next returns the next Hugo release version. -func (v Version) Next() Version { - return Version{Major: v.Major, Minor: v.Minor + 1} -} - -// Prev returns the previous Hugo release version. -func (v Version) Prev() Version { - return Version{Major: v.Major, Minor: v.Minor - 1} -} - -// NextPatchLevel returns the next patch/bugfix Hugo version. -// This will be a patch increment on the previous Hugo version. -func (v Version) NextPatchLevel(level int) Version { - prev := v.Prev() - prev.PatchLevel = level - return prev -} - -// BuildVersionString creates a version string. This is what you see when -// running "hugo version". -func BuildVersionString() string { - // program := "Hugo Static Site Generator" - program := "hugo" - - version := "v" + CurrentVersion.String() - - bi := getBuildInfo() - if bi == nil { - return version - } - if bi.Revision != "" { - version += "-" + bi.Revision - } - if IsExtended { - version += "+extended" - } - if IsWithdeploy { - version += "+withdeploy" - } - - osArch := bi.GoOS + "/" + bi.GoArch - - date := bi.RevisionTime - if date == "" { - // Accept vendor-specified build date if .git/ is unavailable. - date = buildDate - } - if date == "" { - date = "unknown" - } - - versionString := fmt.Sprintf("%s %s %s BuildDate=%s", - program, version, osArch, date) - - if vendorInfo != "" { - versionString += " VendorInfo=" + vendorInfo - } - - return versionString -} - -func version(major, minor, patch int, suffix string) string { - if patch > 0 || minor > 53 { - return fmt.Sprintf("%d.%d.%d%s", major, minor, patch, suffix) - } - return fmt.Sprintf("%d.%d%s", major, minor, suffix) -} - -// CompareVersion compares the given version string or number against the -// running Hugo version. -// It returns -1 if the given version is less than, 0 if equal and 1 if greater than -// the running version. -func CompareVersion(version any) int { - return compareVersions(CurrentVersion, version) -} - -func compareVersions(inVersion Version, in any) int { - var c int - switch d := in.(type) { - case float64: - c = compareFloatWithVersion(d, inVersion) - case float32: - c = compareFloatWithVersion(float64(d), inVersion) - case int: - c = compareFloatWithVersion(float64(d), inVersion) - case int32: - c = compareFloatWithVersion(float64(d), inVersion) - case int64: - c = compareFloatWithVersion(float64(d), inVersion) - case Version: - if d.Major == inVersion.Major && d.Minor == inVersion.Minor && d.PatchLevel == inVersion.PatchLevel { - return strings.Compare(inVersion.Suffix, d.Suffix) - } - if d.Major > inVersion.Major { - return 1 - } else if d.Major < inVersion.Major { - return -1 - } - if d.Minor > inVersion.Minor { - return 1 - } else if d.Minor < inVersion.Minor { - return -1 - } - if d.PatchLevel > inVersion.PatchLevel { - return 1 - } else if d.PatchLevel < inVersion.PatchLevel { - return -1 - } - default: - s, err := cast.ToStringE(in) - if err != nil { - return -1 - } - - v, err := ParseVersion(s) - if err != nil { - return -1 - } - return inVersion.Compare(v) - - } - - return c -} - -func parseVersion(s string) (int, int, int) { - var major, minor, patch int - parts := strings.Split(s, ".") - if len(parts) > 0 { - major, _ = strconv.Atoi(parts[0]) - } - if len(parts) > 1 { - minor, _ = strconv.Atoi(parts[1]) - } - if len(parts) > 2 { - patch, _ = strconv.Atoi(parts[2]) - } - - return major, minor, patch -} - -// compareFloatWithVersion compares v1 with v2. -// It returns -1 if v1 is less than v2, 0 if v1 is equal to v2 and 1 if v1 is greater than v2. -func compareFloatWithVersion(v1 float64, v2 Version) int { - mf, minf := math.Modf(v1) - v1maj := int(mf) - v1min := int(minf * 100) - - if v2.Major == v1maj && v2.Minor == v1min { - return 0 - } - - if v1maj > v2.Major { - return 1 - } - - if v1maj < v2.Major { - return -1 - } - - if v1min > v2.Minor { - return 1 - } - - return -1 -} - -func GoMinorVersion() int { - return goMinorVersion(runtime.Version()) -} - -func goMinorVersion(version string) int { - if strings.HasPrefix(version, "devel") { - return 9999 // magic - } - var major, minor int - var trailing string - n, err := fmt.Sscanf(version, "go%d.%d%s", &major, &minor, &trailing) - if n == 2 && err == io.EOF { - // Means there were no trailing characters (i.e., not an alpha/beta) - err = nil - } - if err != nil { - return 0 - } - return minor -} diff --git a/common/hugo/version_current.go b/common/hugo/version_current.go index 402697f46..4afa88fbb 100644 --- a/common/hugo/version_current.go +++ b/common/hugo/version_current.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2025 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,9 +13,11 @@ package hugo +import "github.com/gohugoio/hugo/common/version" + // CurrentVersion represents the current build version. // This should be the only one. -var CurrentVersion = Version{ +var CurrentVersion = version.Version{ Major: 0, Minor: 153, PatchLevel: 0, diff --git a/common/hugo/version_test.go b/common/hugo/version_test.go deleted file mode 100644 index 33e50ebf5..000000000 --- a/common/hugo/version_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugo - -import ( - "testing" - - qt "github.com/frankban/quicktest" -) - -func TestHugoVersion(t *testing.T) { - c := qt.New(t) - - c.Assert(version(0, 15, 0, "-DEV"), qt.Equals, "0.15-DEV") - c.Assert(version(0, 15, 2, "-DEV"), qt.Equals, "0.15.2-DEV") - - v := Version{Minor: 21, Suffix: "-DEV"} - - c.Assert(v.ReleaseVersion().String(), qt.Equals, "0.21") - c.Assert(v.String(), qt.Equals, "0.21-DEV") - c.Assert(v.Next().String(), qt.Equals, "0.22") - nextVersionString := v.Next().Version() - c.Assert(nextVersionString.String(), qt.Equals, "0.22") - c.Assert(nextVersionString.Eq("0.22"), qt.Equals, true) - c.Assert(nextVersionString.Eq("0.21"), qt.Equals, false) - c.Assert(nextVersionString.Eq(nextVersionString), qt.Equals, true) - c.Assert(v.NextPatchLevel(3).String(), qt.Equals, "0.20.3") - - // We started to use full semver versions even for main - // releases in v0.54.0 - v = Version{Minor: 53, PatchLevel: 0} - c.Assert(v.String(), qt.Equals, "0.53") - c.Assert(v.Next().String(), qt.Equals, "0.54.0") - c.Assert(v.Next().Next().String(), qt.Equals, "0.55.0") - v = Version{Minor: 54, PatchLevel: 0, Suffix: "-DEV"} - c.Assert(v.String(), qt.Equals, "0.54.0-DEV") -} - -func TestCompareVersions(t *testing.T) { - c := qt.New(t) - - c.Assert(compareVersions(MustParseVersion("0.20.0"), 0.20), qt.Equals, 0) - c.Assert(compareVersions(MustParseVersion("0.20.0"), float32(0.20)), qt.Equals, 0) - c.Assert(compareVersions(MustParseVersion("0.20.0"), float64(0.20)), qt.Equals, 0) - c.Assert(compareVersions(MustParseVersion("0.19.1"), 0.20), qt.Equals, 1) - c.Assert(compareVersions(MustParseVersion("0.19.3"), "0.20.2"), qt.Equals, 1) - c.Assert(compareVersions(MustParseVersion("0.1"), 3), qt.Equals, 1) - c.Assert(compareVersions(MustParseVersion("0.1"), int32(3)), qt.Equals, 1) - c.Assert(compareVersions(MustParseVersion("0.1"), int64(3)), qt.Equals, 1) - c.Assert(compareVersions(MustParseVersion("0.20"), "0.20"), qt.Equals, 0) - c.Assert(compareVersions(MustParseVersion("0.20.1"), "0.20.1"), qt.Equals, 0) - c.Assert(compareVersions(MustParseVersion("0.20.1"), "0.20"), qt.Equals, -1) - c.Assert(compareVersions(MustParseVersion("0.20.0"), "0.20.1"), qt.Equals, 1) - c.Assert(compareVersions(MustParseVersion("0.20.1"), "0.20.2"), qt.Equals, 1) - c.Assert(compareVersions(MustParseVersion("0.21.1"), "0.22.1"), qt.Equals, 1) - c.Assert(compareVersions(MustParseVersion("0.22.0"), "0.22-DEV"), qt.Equals, -1) - c.Assert(compareVersions(MustParseVersion("0.22.0"), "0.22.1-DEV"), qt.Equals, 1) - c.Assert(compareVersions(MustParseVersion("0.22.0-DEV"), "0.22"), qt.Equals, 1) - c.Assert(compareVersions(MustParseVersion("0.22.1-DEV"), "0.22"), qt.Equals, -1) - c.Assert(compareVersions(MustParseVersion("0.22.1-DEV"), "0.22.1-DEV"), qt.Equals, 0) -} - -func TestParseHugoVersion(t *testing.T) { - c := qt.New(t) - - c.Assert(MustParseVersion("0.25").String(), qt.Equals, "0.25") - c.Assert(MustParseVersion("0.25.2").String(), qt.Equals, "0.25.2") - c.Assert(MustParseVersion("0.25-test").String(), qt.Equals, "0.25-test") - c.Assert(MustParseVersion("0.25-DEV").String(), qt.Equals, "0.25-DEV") -} - -func TestGoMinorVersion(t *testing.T) { - c := qt.New(t) - c.Assert(goMinorVersion("go1.12.5"), qt.Equals, 12) - c.Assert(goMinorVersion("go1.14rc1"), qt.Equals, 14) - c.Assert(GoMinorVersion() >= 11, qt.Equals, true) -} diff --git a/common/maps/maps.go b/common/maps/maps.go index f9171ebf2..5b3a08b9c 100644 --- a/common/maps/maps.go +++ b/common/maps/maps.go @@ -98,6 +98,8 @@ func ToSliceStringMap(in any) ([]map[string]any, error) { return v, nil case Params: return []map[string]any{v}, nil + case map[string]any: + return []map[string]any{v}, nil case []any: var s []map[string]any for _, entry := range v { diff --git a/common/maps/orderedintset.go b/common/maps/orderedintset.go new file mode 100644 index 000000000..ed9babb2d --- /dev/null +++ b/common/maps/orderedintset.go @@ -0,0 +1,154 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maps + +import ( + "fmt" + "slices" + + "github.com/bits-and-blooms/bitset" +) + +type OrderedIntSet struct { + keys []int + values *bitset.BitSet +} + +// NewOrderedIntSet creates a new OrderedIntSet. +// Note that this is backed by https://github.com/bits-and-blooms/bitset +func NewOrderedIntSet(vals ...int) *OrderedIntSet { + m := &OrderedIntSet{ + keys: make([]int, 0, len(vals)), + values: bitset.New(uint(len(vals))), + } + for _, v := range vals { + m.Set(v) + } + return m +} + +// Set sets the value for the given key. +// Note that insertion order is not affected if a key is re-inserted into the set. +func (m *OrderedIntSet) Set(key int) { + if m == nil { + panic("nil OrderedIntSet") + } + keyu := uint(key) + if m.values.Test(keyu) { + return + } + m.values.Set(keyu) + m.keys = append(m.keys, key) +} + +// SetFrom sets the values from another OrderedIntSet. +func (m *OrderedIntSet) SetFrom(other *OrderedIntSet) { + if m == nil || other == nil { + return + } + for _, key := range other.keys { + m.Set(key) + } +} + +func (m *OrderedIntSet) Clone() *OrderedIntSet { + if m == nil { + return nil + } + newSet := &OrderedIntSet{ + keys: slices.Clone(m.keys), + values: m.values.Clone(), + } + return newSet +} + +// Next returns the next key in the set possibly including the given key. +// It returns -1 if the key is not found or if there are no keys greater than the given key. +func (m *OrderedIntSet) Next(i int) int { + n, ok := m.values.NextSet(uint(i)) + if !ok { + return -1 + } + return int(n) +} + +// The reason we don't use iter.Seq is https://github.com/golang/go/issues/69015 +// This is 70% faster than using iter.Seq2[int, int] for the keys. +// It returns false if the iteration was stopped early. +func (m *OrderedIntSet) ForEachKey(yield func(int) bool) bool { + if m == nil { + return true + } + for _, key := range m.keys { + if !yield(key) { + return false + } + } + return true +} + +func (m *OrderedIntSet) Has(key int) bool { + if m == nil { + return false + } + return m.values.Test(uint(key)) +} + +func (m *OrderedIntSet) Len() int { + if m == nil { + return 0 + } + return len(m.keys) +} + +// KeysSorted returns the keys in sorted order. +func (m *OrderedIntSet) KeysSorted() []int { + if m == nil { + return nil + } + keys := slices.Clone(m.keys) + slices.Sort(keys) + return m.keys +} + +func (m *OrderedIntSet) String() string { + if m == nil { + return "[]" + } + return fmt.Sprintf("%v", m.keys) +} + +func (m *OrderedIntSet) Values() *bitset.BitSet { + if m == nil { + return nil + } + return m.values +} + +func (m *OrderedIntSet) IsSuperSet(other *OrderedIntSet) bool { + if m == nil || other == nil { + return false + } + return m.values.IsSuperSet(other.values) +} + +// Words returns the bitset as array of 64-bit words, giving direct access to the internal representation. +// It is not a copy, so changes to the returned slice will affect the bitset. +// It is meant for advanced users. +func (m *OrderedIntSet) Words() []uint64 { + if m == nil { + return nil + } + return m.values.Words() +} diff --git a/common/maps/orderedintset_test.go b/common/maps/orderedintset_test.go new file mode 100644 index 000000000..243d0d617 --- /dev/null +++ b/common/maps/orderedintset_test.go @@ -0,0 +1,117 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maps + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestOrderedIntSet(t *testing.T) { + c := qt.New(t) + + m := NewOrderedIntSet(2, 1, 3, 7) + + c.Assert(m.Len(), qt.Equals, 4) + c.Assert(m.Has(1), qt.Equals, true) + c.Assert(m.Has(4), qt.Equals, false) + c.Assert(m.String(), qt.Equals, "[2 1 3 7]") + m.Set(4) + c.Assert(m.Len(), qt.Equals, 5) + c.Assert(m.Has(4), qt.Equals, true) + c.Assert(m.Next(0), qt.Equals, 1) + c.Assert(m.Next(1), qt.Equals, 1) + c.Assert(m.Next(2), qt.Equals, 2) + c.Assert(m.Next(3), qt.Equals, 3) + c.Assert(m.Next(4), qt.Equals, 4) + c.Assert(m.Next(7), qt.Equals, 7) + c.Assert(m.Next(8), qt.Equals, -1) + c.Assert(m.String(), qt.Equals, "[2 1 3 7 4]") + + var nilset *OrderedIntSet + c.Assert(nilset.Len(), qt.Equals, 0) + c.Assert(nilset.Has(1), qt.Equals, false) + c.Assert(nilset.String(), qt.Equals, "[]") + + var collected []int + m.ForEachKey(func(key int) bool { + collected = append(collected, key) + return true + }) + c.Assert(collected, qt.DeepEquals, []int{2, 1, 3, 7, 4}) +} + +func BenchmarkOrderedIntSet(b *testing.B) { + smallSet := NewOrderedIntSet() + for i := range 8 { + smallSet.Set(i) + } + mediumSet := NewOrderedIntSet() + for i := range 64 { + mediumSet.Set(i) + } + largeSet := NewOrderedIntSet() + for i := range 1024 { + largeSet.Set(i) + } + + b.Run("New", func(b *testing.B) { + for i := 0; i < b.N; i++ { + NewOrderedIntSet(1, 2, 3, 4, 5, 6, 7, 8) + } + }) + + b.Run("Has small", func(b *testing.B) { + for i := 0; i < b.N; i++ { + smallSet.Has(i % 32) + } + }) + + b.Run("Has medium", func(b *testing.B) { + for i := 0; i < b.N; i++ { + mediumSet.Has(i % 32) + } + }) + + b.Run("Next", func(b *testing.B) { + for i := 0; i < b.N; i++ { + mediumSet.Next(i % 32) + } + }) + + b.Run("ForEachKey small", func(b *testing.B) { + for i := 0; i < b.N; i++ { + smallSet.ForEachKey(func(key int) bool { + return true + }) + } + }) + + b.Run("ForEachKey medium", func(b *testing.B) { + for i := 0; i < b.N; i++ { + mediumSet.ForEachKey(func(key int) bool { + return true + }) + } + }) + + b.Run("ForEachKey large", func(b *testing.B) { + for i := 0; i < b.N; i++ { + largeSet.ForEachKey(func(key int) bool { + return true + }) + } + }) +} diff --git a/common/maps/params.go b/common/maps/params.go index ce03b73f1..e69d13a5b 100644 --- a/common/maps/params.go +++ b/common/maps/params.go @@ -187,6 +187,56 @@ func getNested(m map[string]any, indices []string) (any, string, map[string]any) } } +// CreateNestedParamsFromSegements creates empty nested maps for the given keySegments in the target map. +func CreateNestedParamsFromSegements(target Params, keySegments ...string) Params { + if len(keySegments) == 0 { + return target + } + + m := target + for i, key := range keySegments { + v, found := m[key] + if !found { + nm := Params{} + m[key] = nm + m = nm + if i == len(keySegments)-1 { + return nm + } + continue + } + m = v.(Params) + } + + return m +} + +// CreateNestedParamsSepString creates empty nested maps for the given keyStr in the target map +// It returns the last map created. +func CreateNestedParamsSepString(keyStr, separator string, target Params) Params { + keySegments := strings.Split(keyStr, separator) + return CreateNestedParamsFromSegements(target, keySegments...) +} + +// SetNestedParamIfNotSet sets the value for the given keyStr in the target map if it does not exist. +// It assumes that all but the last key in keyStr is a Params map or should be one. +func SetNestedParamIfNotSet(keyStr, separator string, value any, target Params) Params { + keySegments := strings.Split(keyStr, separator) + if len(keySegments) == 0 { + return target + } + base := keySegments[:len(keySegments)-1] + last := keySegments[len(keySegments)-1] + + m := CreateNestedParamsFromSegements(target, base...) + + if _, ok := m[last]; !ok { + m[last] = value + } + + return target +} + // GetNestedParam gets the first match of the keyStr in the candidates given. // It will first try the exact match and then try to find it as a nested map value, // using the given separator, e.g. "mymap.name". diff --git a/common/maps/params_test.go b/common/maps/params_test.go index 892c77175..4e27eae39 100644 --- a/common/maps/params_test.go +++ b/common/maps/params_test.go @@ -167,3 +167,35 @@ func TestParamsIsZero(t *testing.T) { c.Assert(Params{"_merge": "foo", "foo": "bar"}.IsZero(), qt.IsFalse) c.Assert(Params{"_merge": "foo"}.IsZero(), qt.IsTrue) } + +func TestSetNestedParamIfNotSet(t *testing.T) { + c := qt.New(t) + + m := Params{} + + SetNestedParamIfNotSet("a.b.c", ".", "value", m) + c.Assert(m, qt.DeepEquals, Params{ + "a": Params{ + "b": Params{ + "c": "value", + }, + }, + }) + + m = Params{ + "a": Params{ + "b": Params{ + "c": "existingValue", + }, + }, + } + + SetNestedParamIfNotSet("a.b.c", ".", "value", m) + c.Assert(m, qt.DeepEquals, Params{ + "a": Params{ + "b": Params{ + "c": "existingValue", + }, + }, + }) +} diff --git a/common/paths/path_test.go b/common/paths/path_test.go index bc27df6c6..0e0dfc2e2 100644 --- a/common/paths/path_test.go +++ b/common/paths/path_test.go @@ -311,3 +311,39 @@ func TestIsSameFilePath(t *testing.T) { c.Assert(IsSameFilePath(filepath.FromSlash(this.a), filepath.FromSlash(this.b)), qt.Equals, this.expected, qt.Commentf("a: %s b: %s", this.a, this.b)) } } + +func BenchmarkAddLeadingSlash(b *testing.B) { + const ( + noLeadingSlash = "a/b/c" + withLeadingSlash = "/a/b/c" + ) + + // This should not allocate any memory. + b.Run("With leading slash", func(b *testing.B) { + for i := 0; i < b.N; i++ { + got := AddLeadingSlash(withLeadingSlash) + if got != withLeadingSlash { + b.Fatal(got) + } + } + }) + + // This will allocate some memory. + b.Run("Without leading slash", func(b *testing.B) { + for i := 0; i < b.N; i++ { + got := AddLeadingSlash(noLeadingSlash) + if got != "/a/b/c" { + b.Fatal(got) + } + } + }) + + b.Run("Blank string", func(b *testing.B) { + for i := 0; i < b.N; i++ { + got := AddLeadingSlash("") + if got != "/" { + b.Fatal(got) + } + } + }) +} diff --git a/common/paths/pathparser.go b/common/paths/pathparser.go index 1cae710e8..1246694cc 100644 --- a/common/paths/pathparser.go +++ b/common/paths/pathparser.go @@ -14,23 +14,27 @@ package paths import ( + "fmt" "path" "path/filepath" "runtime" "strings" "sync" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/hugofs/files" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/resources/kinds" ) const ( - identifierBaseof = "baseof" + identifierBaseof = "baseof" + identifierCurstomWrapper = "_" ) -// PathParser parses a path into a Path. +// PathParser parses and manages paths. type PathParser struct { // Maps the language code to its index in the languages/sites slice. LanguageIndex map[string]int @@ -44,6 +48,19 @@ type PathParser struct { // Reports whether the given ext is a content file. IsContentExt func(string) bool + + // The configured sites matrix. + ConfiguredDimensions *sitesmatrix.ConfiguredDimensions + + // Below gets created on demand. + initOnce sync.Once + sitesMatrixCache *maps.Cache[string, sitesmatrix.VectorStore] // Maps language code to sites matrix vector store. +} + +func (pp *PathParser) init() { + pp.initOnce.Do(func() { + pp.sitesMatrixCache = maps.NewCache[string, sitesmatrix.VectorStore]() + }) } // NormalizePathString returns a normalized path string using the very basic Hugo rules. @@ -57,6 +74,32 @@ func NormalizePathStringBasic(s string) string { return s } +func (pp *PathParser) SitesMatrixFromPath(p *Path) sitesmatrix.VectorStore { + pp.init() + lang := p.Lang() + v, _ := pp.sitesMatrixCache.GetOrCreate(lang, func() (sitesmatrix.VectorStore, error) { + builder := sitesmatrix.NewIntSetsBuilder(pp.ConfiguredDimensions) + if lang != "" { + if idx, ok := pp.LanguageIndex[lang]; ok { + builder.WithLanguageIndices(idx) + } + } + + switch p.Component() { + case files.ComponentFolderContent: + builder.WithDefaultsIfNotSet() + case files.ComponentFolderLayouts: + builder.WithAllIfNotSet() + case files.ComponentFolderStatic: + builder.WithDefaultsAndAllLanguagesIfNotSet() + } + + return builder.Build(), nil + }) + + return v +} + // ParseIdentity parses component c with path s into a StringIdentity. func (pp *PathParser) ParseIdentity(c, s string) identity.StringIdentity { p := pp.parsePooled(c, s) @@ -145,6 +188,12 @@ func (pp *PathParser) parseIdentifier(component, s string, p *Path, i, lastDot, id := types.LowHigh[string]{Low: i + 1, High: high} sid := p.s[id.Low:id.High] + if strings.HasPrefix(sid, identifierCurstomWrapper) && strings.HasSuffix(sid, identifierCurstomWrapper) { + p.identifiersKnown = append(p.identifiersKnown, id) + p.posIdentifierCustom = len(p.identifiersKnown) - 1 + found = true + } + if len(p.identifiersKnown) == 0 { // The first is always the extension. p.identifiersKnown = append(p.identifiersKnown, id) @@ -155,12 +204,12 @@ func (pp *PathParser) parseIdentifier(component, s string, p *Path, i, lastDot, p.posIdentifierOutputFormat = 0 } } else { - var langFound bool if mayHaveLang { var disabled bool _, langFound = pp.LanguageIndex[sid] + if !langFound { disabled = pp.IsLangDisabled != nil && pp.IsLangDisabled(sid) if disabled { @@ -173,6 +222,7 @@ func (pp *PathParser) parseIdentifier(component, s string, p *Path, i, lastDot, p.identifiersKnown = append(p.identifiersKnown, id) p.posIdentifierLanguage = len(p.identifiersKnown) - 1 } + } if !found && mayHaveOutputFormat { @@ -202,8 +252,14 @@ func (pp *PathParser) parseIdentifier(component, s string, p *Path, i, lastDot, } if !found && mayHaveLayout { - p.identifiersKnown = append(p.identifiersKnown, id) - p.posIdentifierLayout = len(p.identifiersKnown) - 1 + if p.posIdentifierLayout != -1 { + // Move it to identifiersUnknown. + p.identifiersUnknown = append(p.identifiersUnknown, p.identifiersKnown[p.posIdentifierLayout]) + p.identifiersKnown[p.posIdentifierLayout] = id + } else { + p.identifiersKnown = append(p.identifiersKnown, id) + p.posIdentifierLayout = len(p.identifiersKnown) - 1 + } found = true } @@ -333,7 +389,7 @@ type Type int const ( - // A generic resource, e.g. a JSON file. + // A generic file, e.g. a JSON file. TypeFile Type = iota // All below are content files. @@ -380,6 +436,7 @@ type Path struct { posIdentifierKind int posIdentifierLayout int posIdentifierBaseof int + posIdentifierCustom int disabled bool trimLeadingSlash bool @@ -417,6 +474,7 @@ func (p *Path) reset() { p.posIdentifierKind = -1 p.posIdentifierLayout = -1 p.posIdentifierBaseof = -1 + p.posIdentifierCustom = -1 p.disabled = false p.trimLeadingSlash = false p.unnormalized = nil @@ -580,6 +638,9 @@ func (p *Path) Unnormalized() *Path { // PathNoLang returns the Path but with any language identifier removed. func (p *Path) PathNoLang() string { + if p.identifierIndex(p.posIdentifierLanguage) == -1 { + return p.Path() + } return p.base(true, false) } @@ -633,7 +694,12 @@ func (p *Path) BaseRel(owner *Path) string { // // For other files (Resources), any extension is kept. func (p *Path) Base() string { - return p.base(!p.isContentPage(), p.IsBundle()) + s := p.base(!p.isContentPage(), p.IsBundle()) + if s == "/" && p.isContentPage() { + // The content home page is represented as "". + s = "" + } + return s } // Used in template lookups. @@ -708,6 +774,10 @@ func (p *Path) Lang() string { return p.identifierAsString(p.posIdentifierLanguage) } +func (p *Path) Custom() string { + return strings.TrimSuffix(strings.TrimPrefix(p.identifierAsString(p.posIdentifierCustom), identifierCurstomWrapper), identifierCurstomWrapper) +} + func (p *Path) Identifier(i int) string { return p.identifierAsString(i) } @@ -786,3 +856,12 @@ func HasExt(p string) bool { } return false } + +// ValidateIdentifier returns true if the given string is a valid identifier according +// to Hugo's basic path normalization rules. +func ValidateIdentifier(s string) error { + if s == NormalizePathStringBasic(s) { + return nil + } + return fmt.Errorf("must be all lower case and no spaces") +} diff --git a/common/paths/pathparser_test.go b/common/paths/pathparser_test.go index b1734aef2..d7286d903 100644 --- a/common/paths/pathparser_test.go +++ b/common/paths/pathparser_test.go @@ -18,27 +18,33 @@ import ( "testing" "github.com/gohugoio/hugo/hugofs/files" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "github.com/gohugoio/hugo/resources/kinds" qt "github.com/frankban/quicktest" ) -var testParser = &PathParser{ - LanguageIndex: map[string]int{ - "no": 0, - "en": 1, - "fr": 2, - }, - IsContentExt: func(ext string) bool { - return ext == "md" - }, - IsOutputFormat: func(name, ext string) bool { - switch name { - case "html", "amp", "csv", "rss": - return true - } - return false - }, +func newTestParser() *PathParser { + dims := sitesmatrix.NewTestingDimensions([]string{"en", "no", "fr"}, []string{"v1", "v2", "v3"}, []string{"admin", "editor", "viewer", "guest"}) + + return &PathParser{ + LanguageIndex: map[string]int{ + "no": 0, + "en": 1, + "fr": 2, + }, + IsContentExt: func(ext string) bool { + return ext == "md" + }, + IsOutputFormat: func(name, ext string) bool { + switch name { + case "html", "amp", "csv", "rss": + return true + } + return false + }, + ConfiguredDimensions: dims, + } } func TestParse(t *testing.T) { @@ -190,7 +196,7 @@ func TestParse(t *testing.T) { c.Assert(p.Identifiers(), qt.DeepEquals, []string{"md"}) c.Assert(p.IsBranchBundle(), qt.IsTrue) c.Assert(p.IsBundle(), qt.IsTrue) - c.Assert(p.Base(), qt.Equals, "/") + c.Assert(p.Base(), qt.Equals, "") c.Assert(p.BaseReTyped("foo"), qt.Equals, "/foo") c.Assert(p.Path(), qt.Equals, "/_index.md") c.Assert(p.Container(), qt.Equals, "") @@ -266,7 +272,7 @@ func TestParse(t *testing.T) { "Index root no slash", "_index.md", func(c *qt.C, p *Path) { - c.Assert(p.Base(), qt.Equals, "/") + c.Assert(p.Base(), qt.Equals, "") c.Assert(p.Ext(), qt.Equals, "md") c.Assert(p.Name(), qt.Equals, "_index.md") }, @@ -275,7 +281,7 @@ func TestParse(t *testing.T) { "Index root", "/_index.md", func(c *qt.C, p *Path) { - c.Assert(p.Base(), qt.Equals, "/") + c.Assert(p.Base(), qt.Equals, "") c.Assert(p.Ext(), qt.Equals, "md") c.Assert(p.Name(), qt.Equals, "_index.md") }, @@ -373,13 +379,24 @@ func TestParse(t *testing.T) { c.Assert(p.IsContentData(), qt.IsFalse) }, }, + { + "Custom identifier", + "/a/b/p1._myid_.no.md", + func(c *qt.C, p *Path) { + c.Assert(p.Base(), qt.Equals, "/a/b/p1") + c.Assert(p.Lang(), qt.Equals, "no") + c.Assert(p.Ext(), qt.Equals, "md") + c.Assert(p.Custom(), qt.Equals, "myid") + }, + }, } + parser := newTestParser() for _, test := range tests { c.Run(test.name, func(c *qt.C) { - if test.name != "Home branch cundle" { - // return + if test.name != "Caret up identifier" { + // return } - test.assert(c, testParser.Parse(files.ComponentFolderContent, test.path)) + test.assert(c, parser.Parse(files.ComponentFolderContent, test.path)) }) } } @@ -437,8 +454,8 @@ func TestParseLayouts(t *testing.T) { "/mylayout.list.section.no.html", func(c *qt.C, p *Path) { c.Assert(p.Layout(), qt.Equals, "mylayout") - c.Assert(p.Identifiers(), qt.DeepEquals, []string{"html", "no", "section", "list", "mylayout"}) - c.Assert(p.IdentifiersUnknown(), qt.DeepEquals, []string{}) + c.Assert(p.Identifiers(), qt.DeepEquals, []string{"html", "no", "section", "mylayout"}) + c.Assert(p.IdentifiersUnknown(), qt.DeepEquals, []string{"list"}) c.Assert(p.Base(), qt.Equals, "/mylayout.html") c.Assert(p.Lang(), qt.Equals, "no") }, @@ -461,7 +478,8 @@ func TestParseLayouts(t *testing.T) { "Lang and output format", "/list.no.amp.not.html", func(c *qt.C, p *Path) { - c.Assert(p.Identifiers(), qt.DeepEquals, []string{"html", "not", "amp", "no", "list"}) + c.Assert(p.Identifiers(), qt.DeepEquals, []string{"html", "list", "amp", "no"}) + c.Assert(p.IdentifiersUnknown(), qt.DeepEquals, []string{"not"}) c.Assert(p.OutputFormat(), qt.Equals, "amp") c.Assert(p.Ext(), qt.Equals, "html") c.Assert(p.Lang(), qt.Equals, "no") @@ -583,14 +601,27 @@ func TestParseLayouts(t *testing.T) { c.Assert(p.NameNoIdentifier(), qt.Equals, "myshortcode") }, }, + { + "Not lang", + "/foo/index.xy.html", + func(c *qt.C, p *Path) { + c.Assert(p.Lang(), qt.Equals, "") + c.Assert(p.Layout(), qt.Equals, "index") + c.Assert(p.NameNoLang(), qt.Equals, "index.xy.html") + c.Assert(p.PathNoLang(), qt.Equals, "/foo/index.xy.html") + c.Assert(p.Identifiers(), qt.DeepEquals, []string{"html", "index"}) + c.Assert(p.IdentifiersUnknown(), qt.DeepEquals, []string{"xy"}) + }, + }, } + parser := newTestParser() for _, test := range tests { c.Run(test.name, func(c *qt.C) { - if test.name != "Shortcode lang layout" { - // return + if test.name != "Not lang" { + return } - test.assert(c, testParser.Parse(files.ComponentFolderLayouts, test.path)) + test.assert(c, parser.Parse(files.ComponentFolderLayouts, test.path)) }) } } @@ -605,7 +636,27 @@ func TestHasExt(t *testing.T) { } func BenchmarkParseIdentity(b *testing.B) { + parser := newTestParser() + for i := 0; i < b.N; i++ { + parser.ParseIdentity(files.ComponentFolderAssets, "/a/b.css") + } +} + +func TestSitesMatrixFromPath(t *testing.T) { + c := qt.New(t) + + parser := newTestParser() + p := parser.Parse(files.ComponentFolderContent, "/a/b/c.fr.md") + v := parser.SitesMatrixFromPath(p) + c.Assert(v.HasLanguage(2), qt.IsTrue) + c.Assert(v.LenVectors(), qt.Equals, 1) + c.Assert(v.VectorSample(), qt.Equals, sitesmatrix.Vector{2, 0, 0}) +} + +func BenchmarkSitesMatrixFromPath(b *testing.B) { + parser := newTestParser() + p := parser.Parse(files.ComponentFolderContent, "/a/b/c.fr.md") for i := 0; i < b.N; i++ { - testParser.ParseIdentity(files.ComponentFolderAssets, "/a/b.css") + parser.SitesMatrixFromPath(p) } } diff --git a/common/predicate/predicate.go b/common/predicate/predicate.go index f71536474..3b1c59502 100644 --- a/common/predicate/predicate.go +++ b/common/predicate/predicate.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Hugo Authors. All rights reserved. +// Copyright 2025 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,52 +13,128 @@ package predicate +import ( + "iter" + "strings" + + "github.com/gobwas/glob" + "github.com/gohugoio/hugo/hugofs/hglob" +) + +// Match represents the result of a predicate evaluation. +type Match interface { + OK() bool +} + +var ( + // Predefined Match values for common cases. + True = BoolMatch(true) + False = BoolMatch(false) +) + +// BoolMatch is a simple Match implementation based on a boolean value. +type BoolMatch bool + +func (b BoolMatch) OK() bool { + return bool(b) +} + +// breakMatch is a Match implementation that always returns false for OK() and signals to break evaluation. +type breakMatch struct{} + +func (b breakMatch) OK() bool { + return false +} + +var matchBreak = breakMatch{} + // P is a predicate function that tests whether a value of type T satisfies some condition. type P[T any] func(T) bool -// And returns a predicate that is a short-circuiting logical AND of this and the given predicates. -func (p P[T]) And(ps ...P[T]) P[T] { +// Or returns a predicate that is a short-circuiting logical OR of this and the given predicates. +// Note that P[T] only supports Or. For chained AND/OR logic, use PR[T]. +func (p P[T]) Or(ps ...P[T]) P[T] { return func(v T) bool { + if p != nil && p(v) { + return true + } for _, pp := range ps { - if !pp(v) { - return false + if pp(v) { + return true } } + return false + } +} + +// PR is a predicate function that tests whether a value of type T satisfies some condition and returns a Match result. +type PR[T any] func(T) Match + +// BoolFunc returns a P[T] version of this predicate. +func (p PR[T]) BoolFunc() P[T] { + return func(v T) bool { if p == nil { - return true + return false } - return p(v) + return p(v).OK() } } -// Or returns a predicate that is a short-circuiting logical OR of this and the given predicates. -func (p P[T]) Or(ps ...P[T]) P[T] { - return func(v T) bool { - for _, pp := range ps { - if pp(v) { - return true +// And returns a predicate that is a short-circuiting logical AND of this and the given predicates. +func (p PR[T]) And(ps ...PR[T]) PR[T] { + return func(v T) Match { + if p != nil { + m := p(v) + if !m.OK() || shouldBreak(m) { + return matchBreak } } - if p == nil { - return false + for _, pp := range ps { + m := pp(v) + if !m.OK() || shouldBreak(m) { + return matchBreak + } } - return p(v) + return BoolMatch(true) } } -// Negate returns a predicate that is a logical negation of this predicate. -func (p P[T]) Negate() P[T] { - return func(v T) bool { - return !p(v) +// Or returns a predicate that is a short-circuiting logical OR of this and the given predicates. +func (p PR[T]) Or(ps ...PR[T]) PR[T] { + return func(v T) Match { + if p != nil { + m := p(v) + if m.OK() { + return m + } + if shouldBreak(m) { + return matchBreak + } + } + for _, pp := range ps { + m := pp(v) + if m.OK() { + return m + } + if shouldBreak(m) { + return matchBreak + } + } + return BoolMatch(false) } } +func shouldBreak(m Match) bool { + _, ok := m.(breakMatch) + return ok +} + // Filter returns a new slice holding only the elements of s that satisfy p. // Filter modifies the contents of the slice s and returns the modified slice, which may have a smaller length. -func (p P[T]) Filter(s []T) []T { +func (p PR[T]) Filter(s []T) []T { var n int for _, v := range s { - if p(v) { + if p(v).OK() { s[n] = v n++ } @@ -67,12 +143,50 @@ func (p P[T]) Filter(s []T) []T { } // FilterCopy returns a new slice holding only the elements of s that satisfy p. -func (p P[T]) FilterCopy(s []T) []T { +func (p PR[T]) FilterCopy(s []T) []T { var result []T for _, v := range s { - if p(v) { + if p(v).OK() { result = append(result, v) } } return result } + +// NewStringPredicateFromGlobs creates a string predicate from the given glob patterns. +// A glob pattern starting with "!" is a negation pattern which will be ANDed with the rest. +func NewStringPredicateFromGlobs(patterns []string, getGlob func(pattern string) (glob.Glob, error)) (P[string], error) { + var p PR[string] + for _, pattern := range patterns { + pattern = strings.TrimSpace(pattern) + if pattern == "" { + continue + } + negate := strings.HasPrefix(pattern, hglob.NegationPrefix) + if negate { + pattern = pattern[2:] + g, err := getGlob(pattern) + if err != nil { + return nil, err + } + p = p.And(func(s string) Match { + return BoolMatch(!g.Match(s)) + }) + } else { + g, err := getGlob(pattern) + if err != nil { + return nil, err + } + p = p.Or(func(s string) Match { + return BoolMatch(g.Match(s)) + }) + + } + } + + return p.BoolFunc(), nil +} + +type IndexMatcher interface { + IndexMatch(match P[string]) (iter.Seq[int], error) +} diff --git a/common/predicate/predicate_test.go b/common/predicate/predicate_test.go index 1e1ec004b..5a9c8365b 100644 --- a/common/predicate/predicate_test.go +++ b/common/predicate/predicate_test.go @@ -17,38 +17,66 @@ import ( "testing" qt "github.com/frankban/quicktest" + "github.com/gobwas/glob" "github.com/gohugoio/hugo/common/predicate" ) -func TestAdd(t *testing.T) { +func TestPredicate(t *testing.T) { c := qt.New(t) - var p predicate.P[int] = intP1 + n := func() predicate.PR[int] { + var pr predicate.PR[int] + return pr + } - c.Assert(p(1), qt.IsTrue) + var pr predicate.PR[int] + p := pr.BoolFunc() + c.Assert(p(1), qt.IsFalse) + + pr = n().Or(intP1).Or(intP2) + p = pr.BoolFunc() + c.Assert(p(1), qt.IsTrue) // true || false + c.Assert(p(2), qt.IsTrue) // false || true + c.Assert(p(3), qt.IsFalse) // false || false + + pr = pr.And(intP3) + p = pr.BoolFunc() + c.Assert(p(2), qt.IsFalse) // true || true && false + c.Assert(pr(10), qt.IsTrue) // true || true && true + + pr = pr.And(intP4) + p = pr.BoolFunc() + c.Assert(p(10), qt.IsTrue) // true || true && true && true + c.Assert(p(2), qt.IsFalse) // true || true && false && false + c.Assert(p(1), qt.IsFalse) // true || false && false && false + c.Assert(p(3), qt.IsFalse) // false || false && false && false + c.Assert(p(4), qt.IsFalse) // false || false && false && false + c.Assert(p(42), qt.IsFalse) // false || false && false && false + + pr = n().And(intP1).And(intP2).And(intP3).And(intP4) + p = pr.BoolFunc() + c.Assert(p(1), qt.IsFalse) c.Assert(p(2), qt.IsFalse) + c.Assert(p(10), qt.IsTrue) - neg := p.Negate() - c.Assert(neg(1), qt.IsFalse) - c.Assert(neg(2), qt.IsTrue) - - and := p.And(intP2) - c.Assert(and(1), qt.IsFalse) - c.Assert(and(2), qt.IsFalse) - c.Assert(and(10), qt.IsTrue) + pr = n().And(intP1).And(intP2).And(intP3).And(intP4) + p = pr.BoolFunc() + c.Assert(p(1), qt.IsFalse) + c.Assert(p(2), qt.IsFalse) + c.Assert(p(10), qt.IsTrue) - or := p.Or(intP2) - c.Assert(or(1), qt.IsTrue) - c.Assert(or(2), qt.IsTrue) - c.Assert(or(10), qt.IsTrue) - c.Assert(or(11), qt.IsFalse) + pr = n().Or(intP1).Or(intP2).Or(intP3) + p = pr.BoolFunc() + c.Assert(p(1), qt.IsTrue) + c.Assert(p(10), qt.IsTrue) + c.Assert(p(4), qt.IsFalse) } func TestFilter(t *testing.T) { c := qt.New(t) - var p predicate.P[int] = intP1 - p = p.Or(intP2) + var p predicate.PR[int] + p = p.Or(intP1).Or(intP2) ints := []int{1, 2, 3, 4, 1, 6, 7, 8, 2} @@ -59,8 +87,8 @@ func TestFilter(t *testing.T) { func TestFilterCopy(t *testing.T) { c := qt.New(t) - var p predicate.P[int] = intP1 - p = p.Or(intP2) + var p predicate.PR[int] + p = p.Or(intP1).Or(intP2) ints := []int{1, 2, 3, 4, 1, 6, 7, 8, 2} @@ -68,16 +96,86 @@ func TestFilterCopy(t *testing.T) { c.Assert(ints, qt.DeepEquals, []int{1, 2, 3, 4, 1, 6, 7, 8, 2}) } -var intP1 = func(i int) bool { +var intP1 = func(i int) predicate.Match { + if i == 10 { + return predicate.True + } + return predicate.BoolMatch(i == 1) +} + +var intP2 = func(i int) predicate.Match { + if i == 10 { + return predicate.True + } + return predicate.BoolMatch(i == 2) +} + +var intP3 = func(i int) predicate.Match { if i == 10 { - return true + return predicate.True } - return i == 1 + return predicate.BoolMatch(i == 3) } -var intP2 = func(i int) bool { +var intP4 = func(i int) predicate.Match { if i == 10 { - return true + return predicate.True + } + return predicate.BoolMatch(i == 4) +} + +func TestNewStringPredicateFromGlobs(t *testing.T) { + c := qt.New(t) + + getGlob := func(pattern string) (glob.Glob, error) { + return glob.Compile(pattern) } - return i == 2 + + n := func(patterns ...string) predicate.P[string] { + p, err := predicate.NewStringPredicateFromGlobs(patterns, getGlob) + c.Assert(err, qt.IsNil) + return p + } + + m := n("a", "! ab*", "abc") + c.Assert(m("a"), qt.IsTrue) + c.Assert(m("ab"), qt.IsFalse) + c.Assert(m("abc"), qt.IsFalse) + + m = n() + c.Assert(m("anything"), qt.IsFalse) +} + +func BenchmarkPredicate(b *testing.B) { + b.Run("and or no match", func(b *testing.B) { + var p predicate.PR[int] = intP1 + p = p.And(intP2).Or(intP3) + for i := 0; i < b.N; i++ { + _ = p(3).OK() + } + }) + + b.Run("and and no match", func(b *testing.B) { + var p predicate.PR[int] = intP1 + p = p.And(intP2) + for range b.N { + _ = p(3).OK() + } + }) + + b.Run("and and match", func(b *testing.B) { + var p predicate.PR[int] = intP1 + p = p.And(intP2) + for range b.N { + _ = p(10).OK() + } + }) + + b.Run("or or match", func(b *testing.B) { + var p predicate.PR[int] = intP1 + p = p.Or(intP2).Or(intP3) + for range b.N { + _ = p(2).OK() + } + }) } diff --git a/common/types/types.go b/common/types/types.go index 7e94c1eea..3c9c3b60d 100644 --- a/common/types/types.go +++ b/common/types/types.go @@ -139,7 +139,25 @@ func NewBool(b bool) *bool { return &b } +// WeightProvider provides a weight. +type WeightProvider interface { + Weight() int +} + +// Weight0Provider provides a weight that's considered before the WeightProvider in sorting. +// This allows the weight set on a given term to win. +type Weight0Provider interface { + Weight0() int +} + // PrintableValueProvider is implemented by types that can provide a printable value. type PrintableValueProvider interface { PrintableValue() any } + +type ( + Strings2 [2]string + Strings3 [3]string + Ints2 [2]int + Ints3 [3]int +) diff --git a/common/version/version.go b/common/version/version.go new file mode 100644 index 000000000..de0f5ea6a --- /dev/null +++ b/common/version/version.go @@ -0,0 +1,268 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package version + +import ( + "fmt" + "io" + "math" + "runtime" + "strconv" + "strings" + + "github.com/gohugoio/hugo/compare" + "github.com/spf13/cast" +) + +// Version represents the Hugo build version. +type Version struct { + Major int + + Minor int + + // Increment this for bug releases + PatchLevel int + + // HugoVersionSuffix is the suffix used in the Hugo version string. + // It will be blank for release versions. + Suffix string +} + +var ( + _ compare.Eqer = (*VersionString)(nil) + _ compare.Comparer = (*VersionString)(nil) +) + +func (v Version) String() string { + return version(v.Major, v.Minor, v.PatchLevel, v.Suffix) +} + +// Version returns the Hugo version. +func (v Version) Version() VersionString { + return VersionString(v.String()) +} + +// Compare implements the compare.Comparer interface. +func (h Version) Compare(other any) int { + return CompareVersions(h, other) +} + +// VersionString represents a Hugo version string. +type VersionString string + +func (h VersionString) String() string { + return string(h) +} + +// Compare implements the compare.Comparer interface. +func (h VersionString) Compare(other any) int { + return CompareVersions(h.Version(), other) +} + +func (h VersionString) Version() Version { + return MustParseVersion(h.String()) +} + +// Eq implements the compare.Eqer interface. +func (h VersionString) Eq(other any) bool { + s, err := cast.ToStringE(other) + if err != nil { + return false + } + return s == h.String() +} + +// ParseVersion parses a version string. +func ParseVersion(s string) (Version, error) { + s = strings.TrimPrefix(strings.TrimSpace(s), "v") + var vv Version + hyphen := strings.Index(s, "-") + if hyphen > 0 { + suffix := s[hyphen:] + if len(suffix) > 1 { + if suffix[0] == '-' { + suffix = suffix[1:] + } + if len(suffix) > 0 { + vv.Suffix = suffix + s = s[:hyphen] + } + } + vv.Suffix = suffix + } + vv.Major, vv.Minor, vv.PatchLevel = parseVersion(s) + + return vv, nil +} + +// MustParseVersion parses a version string +// and panics if any error occurs. +func MustParseVersion(s string) Version { + vv, err := ParseVersion(s) + if err != nil { + panic(err) + } + return vv +} + +// ReleaseVersion represents the release version. +func (v Version) ReleaseVersion() Version { + v.Suffix = "" + return v +} + +// Next returns the next Hugo release version. +func (v Version) Next() Version { + return Version{Major: v.Major, Minor: v.Minor + 1} +} + +// Prev returns the previous Hugo release version. +func (v Version) Prev() Version { + return Version{Major: v.Major, Minor: v.Minor - 1} +} + +// NextPatchLevel returns the next patch/bugfix Hugo version. +// This will be a patch increment on the previous Hugo version. +func (v Version) NextPatchLevel(level int) Version { + prev := v.Prev() + prev.PatchLevel = level + return prev +} + +func version(major, minor, patch int, suffix string) string { + if suffix != "" { + if suffix[0] != '-' { + suffix = "-" + suffix + } + } + if patch > 0 || minor > 53 { + return fmt.Sprintf("%d.%d.%d%s", major, minor, patch, suffix) + } + return fmt.Sprintf("%d.%d%s", major, minor, suffix) +} + +// CompareVersion compares v1 with v2. +// It returns -1 if the v2 is less than, 0 if equal and 1 if greater than +// v1. +func CompareVersions(v1 Version, v2 any) int { + var c int + switch d := v2.(type) { + case float64: + c = compareFloatWithVersion(d, v1) + case float32: + c = compareFloatWithVersion(float64(d), v1) + case int: + c = compareFloatWithVersion(float64(d), v1) + case int32: + c = compareFloatWithVersion(float64(d), v1) + case int64: + c = compareFloatWithVersion(float64(d), v1) + case Version: + if d.Major == v1.Major && d.Minor == v1.Minor && d.PatchLevel == v1.PatchLevel { + return strings.Compare(v1.Suffix, d.Suffix) + } + if d.Major > v1.Major { + return 1 + } else if d.Major < v1.Major { + return -1 + } + if d.Minor > v1.Minor { + return 1 + } else if d.Minor < v1.Minor { + return -1 + } + if d.PatchLevel > v1.PatchLevel { + return 1 + } else if d.PatchLevel < v1.PatchLevel { + return -1 + } + default: + s, err := cast.ToStringE(v2) + if err != nil { + return -1 + } + + v, err := ParseVersion(s) + if err != nil { + return -1 + } + return v1.Compare(v) + + } + + return c +} + +func parseVersion(s string) (int, int, int) { + var major, minor, patch int + parts := strings.Split(s, ".") + if len(parts) > 0 { + major, _ = strconv.Atoi(parts[0]) + } + if len(parts) > 1 { + minor, _ = strconv.Atoi(parts[1]) + } + if len(parts) > 2 { + patch, _ = strconv.Atoi(parts[2]) + } + + return major, minor, patch +} + +// compareFloatWithVersion compares v1 with v2. +// It returns -1 if v1 is less than v2, 0 if v1 is equal to v2 and 1 if v1 is greater than v2. +func compareFloatWithVersion(v1 float64, v2 Version) int { + mf, minf := math.Modf(v1) + v1maj := int(mf) + v1min := int(minf * 100) + + if v2.Major == v1maj && v2.Minor == v1min { + return 0 + } + + if v1maj > v2.Major { + return 1 + } + + if v1maj < v2.Major { + return -1 + } + + if v1min > v2.Minor { + return 1 + } + + return -1 +} + +func GoMinorVersion() int { + return goMinorVersion(runtime.Version()) +} + +func goMinorVersion(version string) int { + if strings.HasPrefix(version, "devel") { + return 9999 // magic + } + var major, minor int + var trailing string + n, err := fmt.Sscanf(version, "go%d.%d%s", &major, &minor, &trailing) + if n == 2 && err == io.EOF { + // Means there were no trailing characters (i.e., not an alpha/beta) + err = nil + } + if err != nil { + return 0 + } + return minor +} diff --git a/common/version/version_test.go b/common/version/version_test.go new file mode 100644 index 000000000..f171e4d97 --- /dev/null +++ b/common/version/version_test.go @@ -0,0 +1,89 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package version + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestHugoVersion(t *testing.T) { + c := qt.New(t) + + c.Assert(version(0, 15, 0, "-DEV"), qt.Equals, "0.15-DEV") + c.Assert(version(0, 15, 2, "-DEV"), qt.Equals, "0.15.2-DEV") + + v := Version{Minor: 21, Suffix: "-DEV"} + + c.Assert(v.ReleaseVersion().String(), qt.Equals, "0.21") + c.Assert(v.String(), qt.Equals, "0.21-DEV") + c.Assert(v.Next().String(), qt.Equals, "0.22") + nextVersionString := v.Next().Version() + c.Assert(nextVersionString.String(), qt.Equals, "0.22") + c.Assert(nextVersionString.Eq("0.22"), qt.Equals, true) + c.Assert(nextVersionString.Eq("0.21"), qt.Equals, false) + c.Assert(nextVersionString.Eq(nextVersionString), qt.Equals, true) + c.Assert(v.NextPatchLevel(3).String(), qt.Equals, "0.20.3") + + // We started to use full semver versions even for main + // releases in v0.54.0 + v = Version{Minor: 53, PatchLevel: 0} + c.Assert(v.String(), qt.Equals, "0.53") + c.Assert(v.Next().String(), qt.Equals, "0.54.0") + c.Assert(v.Next().Next().String(), qt.Equals, "0.55.0") + v = Version{Minor: 54, PatchLevel: 0, Suffix: "-DEV"} + c.Assert(v.String(), qt.Equals, "0.54.0-DEV") +} + +func TestCompareVersions(t *testing.T) { + c := qt.New(t) + + c.Assert(CompareVersions(MustParseVersion("0.20.0"), 0.20), qt.Equals, 0) + c.Assert(CompareVersions(MustParseVersion("0.20.0"), float32(0.20)), qt.Equals, 0) + c.Assert(CompareVersions(MustParseVersion("0.20.0"), float64(0.20)), qt.Equals, 0) + c.Assert(CompareVersions(MustParseVersion("0.19.1"), 0.20), qt.Equals, 1) + c.Assert(CompareVersions(MustParseVersion("0.19.3"), "0.20.2"), qt.Equals, 1) + c.Assert(CompareVersions(MustParseVersion("0.1"), 3), qt.Equals, 1) + c.Assert(CompareVersions(MustParseVersion("0.1"), int32(3)), qt.Equals, 1) + c.Assert(CompareVersions(MustParseVersion("0.1"), int64(3)), qt.Equals, 1) + c.Assert(CompareVersions(MustParseVersion("0.20"), "0.20"), qt.Equals, 0) + c.Assert(CompareVersions(MustParseVersion("0.20.1"), "0.20.1"), qt.Equals, 0) + c.Assert(CompareVersions(MustParseVersion("0.20.1"), "0.20"), qt.Equals, -1) + c.Assert(CompareVersions(MustParseVersion("0.20.0"), "0.20.1"), qt.Equals, 1) + c.Assert(CompareVersions(MustParseVersion("0.20.1"), "0.20.2"), qt.Equals, 1) + c.Assert(CompareVersions(MustParseVersion("0.21.1"), "0.22.1"), qt.Equals, 1) + c.Assert(CompareVersions(MustParseVersion("0.22.0"), "0.22-DEV"), qt.Equals, -1) + c.Assert(CompareVersions(MustParseVersion("0.22.0"), "0.22.1-DEV"), qt.Equals, 1) + c.Assert(CompareVersions(MustParseVersion("0.22.0-DEV"), "0.22"), qt.Equals, 1) + c.Assert(CompareVersions(MustParseVersion("0.22.1-DEV"), "0.22"), qt.Equals, -1) + c.Assert(CompareVersions(MustParseVersion("0.22.1-DEV"), "0.22.1-DEV"), qt.Equals, 0) +} + +func TestParseHugoVersion(t *testing.T) { + c := qt.New(t) + + c.Assert(MustParseVersion("v2.3.2").String(), qt.Equals, "2.3.2") + c.Assert(MustParseVersion("0.25").String(), qt.Equals, "0.25") + c.Assert(MustParseVersion("0.25.2").String(), qt.Equals, "0.25.2") + c.Assert(MustParseVersion("0.25-test").String(), qt.Equals, "0.25-test") + c.Assert(MustParseVersion("0.25-DEV").String(), qt.Equals, "0.25-DEV") +} + +func TestGoMinorVersion(t *testing.T) { + c := qt.New(t) + c.Assert(goMinorVersion("go1.12.5"), qt.Equals, 12) + c.Assert(goMinorVersion("go1.14rc1"), qt.Equals, 14) + c.Assert(GoMinorVersion() >= 11, qt.Equals, true) +} diff --git a/compare/compare.go b/compare/compare.go index fd15bd087..5c6a4dae9 100644 --- a/compare/compare.go +++ b/compare/compare.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2025 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/compare/compare_test.go b/compare/compare_test.go new file mode 100644 index 000000000..674dc1660 --- /dev/null +++ b/compare/compare_test.go @@ -0,0 +1,14 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compare diff --git a/config/allconfig/allconfig.go b/config/allconfig/allconfig.go index 4e9cbcc5a..d5d00d679 100644 --- a/config/allconfig/allconfig.go +++ b/config/allconfig/allconfig.go @@ -29,6 +29,7 @@ import ( "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/cache/httpcache" + "github.com/gohugoio/hugo/common/hstrings" "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/common/maps" @@ -40,7 +41,10 @@ import ( "github.com/gohugoio/hugo/config/services" "github.com/gohugoio/hugo/deploy/deployconfig" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugolib/roles" "github.com/gohugoio/hugo/hugolib/segments" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" + "github.com/gohugoio/hugo/hugolib/versions" "github.com/gohugoio/hugo/langs" gc "github.com/gohugoio/hugo/markup/goldmark/goldmark_config" "github.com/gohugoio/hugo/markup/markup_config" @@ -99,7 +103,8 @@ type Config struct { // For internal use only. Internal InternalConfig `mapstructure:"-" json:"-"` // For internal use only. - C *ConfigCompiled `mapstructure:"-" json:"-"` + C *ConfigCompiled `mapstructure:"-" json:"-"` + isLanguageClone bool RootConfig @@ -139,16 +144,25 @@ type Config struct { // The outputformats configuration sections maps a format name (a string) to a configuration object for that format. OutputFormats *config.ConfigNamespace[map[string]output.OutputFormatConfig, output.Formats] `mapstructure:"-"` + // The languages configuration sections maps a language code (a string) to a configuration object for that language. + Languages *config.ConfigNamespace[map[string]langs.LanguageConfig, langs.LanguagesInternal] `mapstructure:"-"` + + // The versions configuration section contains the top level versions configuration options. + Versions *config.ConfigNamespace[map[string]versions.VersionConfig, versions.VersionsInternal] `mapstructure:"-"` + + // The roles configuration section contains the top level roles configuration options. + Roles *config.ConfigNamespace[map[string]roles.RoleConfig, roles.RolesInternal] `mapstructure:"-"` + // The outputs configuration section maps a Page Kind (a string) to a slice of output formats. // This can be overridden in the front matter. Outputs map[string][]string `mapstructure:"-"` // The cascade configuration section contains the top level front matter cascade configuration options, // a slice of page matcher and params to apply to those pages. - Cascade *config.ConfigNamespace[[]page.PageMatcherParamsConfig, *maps.Ordered[page.PageMatcher, page.PageMatcherParamsConfig]] `mapstructure:"-"` + Cascade *page.PageMatcherParamsConfigs `mapstructure:"-"` // The segments defines segments for the site. Used for partial/segmented builds. - Segments *config.ConfigNamespace[map[string]segments.SegmentConfig, segments.Segments] `mapstructure:"-"` + Segments *config.ConfigNamespace[map[string]segments.SegmentConfig, *segments.Segments] `mapstructure:"-"` // Menu configuration. // {"refs": ["config:languages:menus"] } @@ -200,19 +214,23 @@ type Config struct { // {"refs": ["config:languages:params"] } Params maps.Params `mapstructure:"-"` - // The languages configuration sections maps a language code (a string) to a configuration object for that language. - Languages map[string]langs.LanguageConfig `mapstructure:"-"` - // UglyURLs configuration. Either a boolean or a sections map. UglyURLs any `mapstructure:"-"` } +// Early initialization of config. type configCompiler interface { CompileConfig(logger loggers.Logger) error } +// Late initialization of config. +type configInitializer interface { + InitConfig(logger loggers.Logger, defaultSitesMatrix sitesmatrix.VectorStore, configuredDimensions *sitesmatrix.ConfiguredDimensions) error +} + func (c Config) cloneForLang() *Config { x := c + x.isLanguageClone = true x.C = nil copyStringSlice := func(in []string) []string { if in == nil { @@ -319,18 +337,6 @@ func (c *Config) CompileConfig(logger loggers.Logger) error { for _, lang := range c.DisableLanguages { disabledLangs[lang] = true } - for lang, language := range c.Languages { - if !language.Disabled && disabledLangs[lang] { - language.Disabled = true - c.Languages[lang] = language - } - if language.Disabled { - disabledLangs[lang] = true - if lang == c.DefaultContentLanguage { - return fmt.Errorf("cannot disable default content language %q", lang) - } - } - } for i, s := range c.IgnoreLogs { c.IgnoreLogs[i] = strings.ToLower(s) @@ -483,7 +489,6 @@ func (c *Config) CompileConfig(logger loggers.Logger) error { CreateTitle: helpers.GetTitleFunc(c.TitleCaseStyle), IsUglyURLSection: isUglyURL, IgnoreFile: ignoreFile, - SegmentFilter: c.Segments.Config.Get(func(s string) { logger.Warnf("Render segment %q not found in configuration", s) }, c.RootConfig.RenderSegments...), MainSections: c.MainSections, Clock: clock, HTTPCache: httpCache, @@ -523,7 +528,6 @@ type ConfigCompiled struct { CreateTitle func(s string) string IsUglyURLSection func(section string) bool IgnoreFile func(filename string) bool - SegmentFilter segments.SegmentFilter MainSections []string Clock time.Time HTTPCache httpcache.ConfigCompiled @@ -586,6 +590,18 @@ type RootConfig struct { // Set this to true to put all languages below their language ID. DefaultContentLanguageInSubdir bool + // The default content role to use for the site. + DefaultContentRole string + + // Set this to true to put the default role in a subdirectory. + DefaultContentRoleInSubdir bool + + // The default content version to use for the site. + DefaultContentVersion string + + // Set to true to render the default version in a subdirectory. + DefaultContentVersionInSubdir bool + // The default output format to use for the site. // If not set, we will use the first output format. DefaultOutputFormat string @@ -788,14 +804,13 @@ func (c RootConfig) staticDirs() []string { dirs = append(dirs, c.StaticDir8...) dirs = append(dirs, c.StaticDir9...) dirs = append(dirs, c.StaticDir10...) - return helpers.UniqueStringsReuse(dirs) + return hstrings.UniqueStringsReuse(dirs) } type Configs struct { - Base *Config - LoadingInfo config.LoadConfigResult - LanguageConfigMap map[string]*Config - LanguageConfigSlice []*Config + Base *Config + LoadingInfo config.LoadConfigResult + LanguageConfigMap map[string]*Config IsMultihost bool @@ -803,18 +818,16 @@ type Configs struct { ModulesClient *modules.Client // All below is set in Init. - Languages langs.Languages - LanguagesDefaultFirst langs.Languages - ContentPathParser *paths.PathParser + Languages langs.Languages + ContentPathParser *paths.PathParser + ConfiguredDimensions *sitesmatrix.ConfiguredDimensions + DefaultContentSitesMatrix *sitesmatrix.IntSets + AllSitesMatrix *sitesmatrix.IntSets configLangs []config.AllProvider } func (c *Configs) Validate(logger loggers.Logger) error { - c.Base.Cascade.Config.Range(func(p page.PageMatcher, cfg page.PageMatcherParamsConfig) bool { - page.CheckCascadePattern(logger, p) - return true - }) return nil } @@ -833,54 +846,15 @@ func (c *Configs) IsZero() bool { return c == nil || len(c.Languages) == 0 } -func (c *Configs) Init() error { +func (c *Configs) Init(logger loggers.Logger) error { var languages langs.Languages - var langKeys []string - var hasEn bool - - const en = "en" - - for k := range c.LanguageConfigMap { - langKeys = append(langKeys, k) - if k == en { - hasEn = true - } - } - - // Sort the LanguageConfigSlice by language weight (if set) or lang. - sort.Slice(langKeys, func(i, j int) bool { - ki := langKeys[i] - kj := langKeys[j] - lki := c.LanguageConfigMap[ki] - lkj := c.LanguageConfigMap[kj] - li := lki.Languages[ki] - lj := lkj.Languages[kj] - if li.Weight != lj.Weight { - return li.Weight < lj.Weight - } - return ki < kj - }) - - // See issue #13646. - defaultConfigLanguageFallback := en - if !hasEn { - // Pick the first one. - defaultConfigLanguageFallback = langKeys[0] - } - - if c.Base.DefaultContentLanguage == "" { - c.Base.DefaultContentLanguage = defaultConfigLanguageFallback - } - - for _, k := range langKeys { - v := c.LanguageConfigMap[k] - if v.DefaultContentLanguage == "" { - v.DefaultContentLanguage = defaultConfigLanguageFallback + for _, f := range c.Base.Languages.Config.Sorted { + v, found := c.LanguageConfigMap[f.Name] + if !found { + return fmt.Errorf("invalid language configuration for %q", f.Name) } - c.LanguageConfigSlice = append(c.LanguageConfigSlice, v) - languageConf := v.Languages[k] - language, err := langs.NewLanguage(k, c.Base.DefaultContentLanguage, v.TimeZone, languageConf) + language, err := langs.NewLanguage(f.Name, c.Base.DefaultContentLanguage, v.TimeZone, f.LanguageConfig) if err != nil { return err } @@ -897,25 +871,25 @@ func (c *Configs) Init() error { } languages = languages[:n] - var languagesDefaultFirst langs.Languages - for _, l := range languages { - if l.Lang == c.Base.DefaultContentLanguage { - languagesDefaultFirst = append(languagesDefaultFirst, l) - } - } - for _, l := range languages { - if l.Lang != c.Base.DefaultContentLanguage { - languagesDefaultFirst = append(languagesDefaultFirst, l) - } + c.Languages = languages + c.ConfiguredDimensions = &sitesmatrix.ConfiguredDimensions{ + ConfiguredLanguages: c.Base.Languages.Config, + ConfiguredVersions: c.Base.Versions.Config, + ConfiguredRoles: c.Base.Roles.Config, } - c.Languages = languages - c.LanguagesDefaultFirst = languagesDefaultFirst + intSetsCfg := sitesmatrix.IntSetsConfig{ + ApplyDefaults: sitesmatrix.IntSetsConfigApplyDefaultsIfNotSet, + } + matrix := sitesmatrix.NewIntSetsBuilder(c.ConfiguredDimensions).WithConfig(intSetsCfg) + c.DefaultContentSitesMatrix = matrix.Build() + c.AllSitesMatrix = sitesmatrix.NewIntSetsBuilder(c.ConfiguredDimensions).WithAllIfNotSet().Build() c.ContentPathParser = &paths.PathParser{ - LanguageIndex: languagesDefaultFirst.AsIndexSet(), - IsLangDisabled: c.Base.IsLangDisabled, - IsContentExt: c.Base.ContentTypes.Config.IsContentSuffix, + ConfiguredDimensions: c.ConfiguredDimensions, + LanguageIndex: languages.AsIndexSet(), + IsLangDisabled: c.Base.IsLangDisabled, + IsContentExt: c.Base.ContentTypes.Config.IsContentSuffix, IsOutputFormat: func(name, ext string) bool { if name == "" { return false @@ -932,12 +906,26 @@ func (c *Configs) Init() error { } c.configLangs = make([]config.AllProvider, len(c.Languages)) - for i, l := range c.LanguagesDefaultFirst { + + // Config can be shared between languages, + // avoid initializing the same config more than once. + for i, l := range c.Languages { + langConfig := c.LanguageConfigMap[l.Lang] + sitesMatrix := sitesmatrix.NewIntSetsBuilder(c.ConfiguredDimensions).WithLanguageIndices(i).WithAllIfNotSet().Build() + for _, s := range allDecoderSetups { + if getInitializer := s.getInitializer; getInitializer != nil { + if err := getInitializer(langConfig).InitConfig(logger, sitesMatrix, c.ConfiguredDimensions); err != nil { + return err + } + } + } + c.configLangs[i] = ConfigLanguage{ - m: c, - config: c.LanguageConfigMap[l.Lang], - baseConfig: c.LoadingInfo.BaseConfig, - language: l, + m: c, + config: langConfig, + baseConfig: c.LoadingInfo.BaseConfig, + language: l, + languageIndex: i, } } @@ -946,7 +934,7 @@ func (c *Configs) Init() error { } // Apply default project mounts. - if err := modules.ApplyProjectConfigDefaults(c.Modules[0], c.configLangs...); err != nil { + if err := modules.ApplyProjectConfigDefaults(logger.Logger(), c.Modules[0], c.configLangs...); err != nil { return err } @@ -955,7 +943,7 @@ func (c *Configs) Init() error { for _, m := range c.Modules[0].Mounts() { var found bool for _, cm := range c.Base.Module.Mounts { - if cm.Source == m.Source && cm.Target == m.Target && cm.Lang == m.Lang { + if cm.Equal(m) { found = true break } @@ -966,7 +954,7 @@ func (c *Configs) Init() error { } // Transfer the changed mounts to the language versions (all share the same mount set, but can be displayed in different languages). - for _, l := range c.LanguageConfigSlice { + for _, l := range c.LanguageConfigMap { l.Module.Mounts = c.Base.Module.Mounts } @@ -983,7 +971,7 @@ func (c Configs) GetFirstLanguageConfig() config.AllProvider { func (c Configs) GetByLang(lang string) config.AllProvider { for _, l := range c.configLangs { - if l.Language().Lang == lang { + if l.Language().(*langs.Language).Lang == lang { return l } } @@ -1060,7 +1048,6 @@ func fromLoadConfigResult(fs afero.Fs, logger loggers.Logger, res config.LoadCon maps.MergeStrategyKey: maps.ParamsMergeStrategyDeep, } } - for kk, vv := range x { if kk == "_merge" { continue @@ -1077,6 +1064,13 @@ func fromLoadConfigResult(fs afero.Fs, logger loggers.Logger, res config.LoadCon vv = maps.CloneParamsDeep(p) } + if kk == "cascade" { + // If not set, add the current language to make sure it does not get applied to other languages. + page.AddLangToCascadeTargetMap(k, vv.(maps.Params)) + // Always clone cascade config to get the sites matrix right. + differentRootKeys = append(differentRootKeys, kk) + } + mergedConfig.Set(kk, vv) rootv := cfg.Get(kk) if rootv != nil && cfg.IsSet(kk) { @@ -1108,7 +1102,7 @@ func fromLoadConfigResult(fs afero.Fs, logger loggers.Logger, res config.LoadCon } } } - differentRootKeys = helpers.UniqueStringsSorted(differentRootKeys) + differentRootKeys = hstrings.UniqueStringsSorted(differentRootKeys) if len(differentRootKeys) == 0 { langConfigMap[k] = all @@ -1188,7 +1182,7 @@ func decodeConfigFromParams(fs afero.Fs, logger loggers.Logger, bcfg config.Base }) for _, v := range decoderSetups { - p := decodeConfig{p: p, c: target, fs: fs, bcfg: bcfg} + p := decodeConfig{p: p, c: target, fs: fs, bcfg: bcfg, logger: logger} if err := v.decode(v, p); err != nil { return fmt.Errorf("failed to decode %q: %w", v.key, err) } diff --git a/config/allconfig/allconfig_integration_test.go b/config/allconfig/allconfig_integration_test.go index 79d2299de..54a2cc091 100644 --- a/config/allconfig/allconfig_integration_test.go +++ b/config/allconfig/allconfig_integration_test.go @@ -66,10 +66,12 @@ Title: {{ .Title }} b.Assert(modConf.Mounts, qt.HasLen, 8) b.Assert(modConf.Mounts[0].Source, qt.Equals, filepath.FromSlash("content/en")) b.Assert(modConf.Mounts[0].Target, qt.Equals, "content") - b.Assert(modConf.Mounts[0].Lang, qt.Equals, "en") + b.Assert(modConf.Mounts[0].Lang, qt.Equals, "") + b.Assert(modConf.Mounts[0].Sites.Matrix.Languages, qt.DeepEquals, []string{"en"}) b.Assert(modConf.Mounts[1].Source, qt.Equals, filepath.FromSlash("content/sv")) b.Assert(modConf.Mounts[1].Target, qt.Equals, "content") - b.Assert(modConf.Mounts[1].Lang, qt.Equals, "sv") + b.Assert(modConf.Mounts[1].Lang, qt.Equals, "") + b.Assert(modConf.Mounts[1].Sites.Matrix.Languages, qt.DeepEquals, []string{"sv"}) } func TestConfigAliases(t *testing.T) { @@ -198,28 +200,6 @@ x b.Assert(err.Error(), qt.Contains, `failed to create config: unknown output format "foo" for kind "home"`) } -// Issue 13201 -func TestLanguageConfigSlice(t *testing.T) { - t.Parallel() - - files := ` --- hugo.toml -- -disableKinds = ['page','rss','section','sitemap','taxonomy','term'] -[languages.en] -title = 'TITLE_EN' -weight = 2 -[languages.de] -title = 'TITLE_DE' -weight = 1 -[languages.fr] -title = 'TITLE_FR' -weight = 3 -` - - b := hugolib.Test(t, files) - b.Assert(b.H.Configs.LanguageConfigSlice[0].Title, qt.Equals, `TITLE_DE`) -} - func TestContentTypesDefault(t *testing.T) { files := ` -- hugo.toml -- diff --git a/config/allconfig/alldecoders.go b/config/allconfig/alldecoders.go index 035349790..81e31aa6e 100644 --- a/config/allconfig/alldecoders.go +++ b/config/allconfig/alldecoders.go @@ -15,11 +15,15 @@ package allconfig import ( "fmt" + "sort" "strings" "github.com/gohugoio/hugo/cache/filecache" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/cache/httpcache" + "github.com/gohugoio/hugo/common/hstrings" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/config" @@ -27,8 +31,9 @@ import ( "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/config/services" "github.com/gohugoio/hugo/deploy/deployconfig" + "github.com/gohugoio/hugo/hugolib/roles" "github.com/gohugoio/hugo/hugolib/segments" - "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/hugolib/versions" "github.com/gohugoio/hugo/markup/markup_config" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/minifiers" @@ -46,16 +51,18 @@ import ( ) type decodeConfig struct { - p config.Provider - c *Config - fs afero.Fs - bcfg config.BaseConfig + p config.Provider + c *Config + fs afero.Fs + bcfg config.BaseConfig + logger loggers.Logger } type decodeWeight struct { key string decode func(decodeWeight, decodeConfig) error getCompiler func(c *Config) configCompiler + getInitializer func(c *Config) configInitializer weight int internalOrDeprecated bool // Hide it from the docs. } @@ -69,8 +76,10 @@ var allDecoderSetups = map[string]decodeWeight{ return err } - // This need to match with Lang which is always lower case. + // This need to match with the map keys, always lower case. p.c.RootConfig.DefaultContentLanguage = strings.ToLower(p.c.RootConfig.DefaultContentLanguage) + p.c.RootConfig.DefaultContentRole = strings.ToLower(p.c.RootConfig.DefaultContentRole) + p.c.RootConfig.DefaultContentVersion = strings.ToLower(p.c.RootConfig.DefaultContentVersion) return nil }, @@ -140,9 +149,12 @@ var allDecoderSetups = map[string]decodeWeight{ key: "segments", decode: func(d decodeWeight, p decodeConfig) error { var err error - p.c.Segments, err = segments.DecodeSegments(p.p.GetStringMap(d.key)) + p.c.Segments, err = segments.DecodeSegments(p.p.GetStringMap(d.key), p.c.RenderSegments, p.logger) return err }, + getInitializer: func(c *Config) configInitializer { + return c.Segments.Config + }, }, "server": { key: "server", @@ -210,6 +222,70 @@ var allDecoderSetups = map[string]decodeWeight{ return err }, }, + "languages": { + key: "languages", + decode: func(d decodeWeight, p decodeConfig) error { + m := maps.CleanConfigStringMap(p.p.GetStringMap(d.key)) + if len(m) == 1 { + // In v0.112.4 we moved this to the language config, but it's very commmon for mono language sites to have this at the top level. + var first maps.Params + var ok bool + for _, v := range m { + first, ok = v.(maps.Params) + if ok { + break + } + } + if first != nil { + if _, found := first["languagecode"]; !found { + first["languagecode"] = p.p.GetString("languagecode") + } + } + } + var ( + err error + defaultContentLanguage string + ) + p.c.Languages, defaultContentLanguage, err = langs.DecodeConfig(p.c.RootConfig.DefaultContentLanguage, p.c.RootConfig.DisableLanguages, m) + if err != nil { + return fmt.Errorf("failed to decode languages config: %w", err) + } + for k, v := range p.c.Languages.Config.LanguageConfigs { + if v.Disabled { + p.c.RootConfig.DisableLanguages = append(p.c.RootConfig.DisableLanguages, k) + } + } + + p.c.RootConfig.DisableLanguages = hstrings.UniqueStringsReuse(p.c.RootConfig.DisableLanguages) + sort.Strings(p.c.RootConfig.DisableLanguages) + p.c.RootConfig.DefaultContentLanguage = defaultContentLanguage + return nil + }, + }, + "versions": { + key: "versions", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + m := maps.CleanConfigStringMap(p.p.GetStringMap(d.key)) + p.c.Versions, err = versions.DecodeConfig(p.c.RootConfig.DefaultContentVersion, m) + return err + }, + }, + "roles": { + key: "roles", + decode: func(d decodeWeight, p decodeConfig) error { + var ( + err error + defaultContentRole string + ) + m := maps.CleanConfigStringMap(p.p.GetStringMap(d.key)) + p.c.Roles, defaultContentRole, err = roles.DecodeConfig(p.c.RootConfig.DefaultContentRole, m) + p.c.RootConfig.DefaultContentRole = defaultContentRole + + return err + }, + }, + "params": { key: "params", decode: func(d decodeWeight, p decodeConfig) error { @@ -233,7 +309,7 @@ var allDecoderSetups = map[string]decodeWeight{ key: "module", decode: func(d decodeWeight, p decodeConfig) error { var err error - p.c.Module, err = modules.DecodeConfig(p.p) + p.c.Module, err = modules.DecodeConfig(p.logger.Logger(), p.p) return err }, }, @@ -283,56 +359,15 @@ var allDecoderSetups = map[string]decodeWeight{ return nil }, }, - "languages": { - key: "languages", - decode: func(d decodeWeight, p decodeConfig) error { - var err error - m := p.p.GetStringMap(d.key) - if len(m) == 1 { - // In v0.112.4 we moved this to the language config, but it's very commmon for mono language sites to have this at the top level. - var first maps.Params - var ok bool - for _, v := range m { - first, ok = v.(maps.Params) - if ok { - break - } - } - if first != nil { - if _, found := first["languagecode"]; !found { - first["languagecode"] = p.p.GetString("languagecode") - } - } - } - p.c.Languages, err = langs.DecodeConfig(m) - if err != nil { - return err - } - // Validate defaultContentLanguage. - if p.c.DefaultContentLanguage != "" { - var found bool - for lang := range p.c.Languages { - if lang == p.c.DefaultContentLanguage { - found = true - break - } - } - if !found { - return fmt.Errorf("config value %q for defaultContentLanguage does not match any language definition", p.c.DefaultContentLanguage) - } - } - - return nil - }, - }, "cascade": { key: "cascade", decode: func(d decodeWeight, p decodeConfig) error { var err error - p.c.Cascade, err = page.DecodeCascadeConfig(nil, true, p.p.Get(d.key)) + p.c.Cascade, err = page.DecodeCascadeConfig(p.p.Get(d.key)) return err }, + getInitializer: func(c *Config) configInitializer { return c.Cascade }, }, "menus": { key: "menus", diff --git a/config/allconfig/configlanguage.go b/config/allconfig/configlanguage.go index 6990a3590..9a7789efd 100644 --- a/config/allconfig/configlanguage.go +++ b/config/allconfig/configlanguage.go @@ -19,6 +19,7 @@ import ( "github.com/gohugoio/hugo/common/paths" "github.com/gohugoio/hugo/common/urls" "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/langs" ) @@ -27,20 +28,21 @@ type ConfigLanguage struct { config *Config baseConfig config.BaseConfig - m *Configs - language *langs.Language + m *Configs + language *langs.Language + languageIndex int } -func (c ConfigLanguage) Language() *langs.Language { +func (c ConfigLanguage) Language() any { return c.language } -func (c ConfigLanguage) Languages() langs.Languages { - return c.m.Languages +func (c ConfigLanguage) LanguageIndex() int { + return c.languageIndex } -func (c ConfigLanguage) LanguagesDefaultFirst() langs.Languages { - return c.m.LanguagesDefaultFirst +func (c ConfigLanguage) Languages() any { + return c.m.Languages } func (c ConfigLanguage) PathParser() *paths.PathParser { @@ -48,14 +50,14 @@ func (c ConfigLanguage) PathParser() *paths.PathParser { } func (c ConfigLanguage) LanguagePrefix() string { - if c.DefaultContentLanguageInSubdir() && c.DefaultContentLanguage() == c.Language().Lang { - return c.Language().Lang + if c.DefaultContentLanguageInSubdir() && c.DefaultContentLanguage() == c.language.Lang { + return c.language.Lang } - if !c.IsMultilingual() || c.DefaultContentLanguage() == c.Language().Lang { + if !c.IsMultilingual() || c.DefaultContentLanguage() == c.language.Lang { return "" } - return c.Language().Lang + return c.language.Lang } func (c ConfigLanguage) BaseURL() urls.BaseURL { @@ -97,6 +99,10 @@ func (c ConfigLanguage) IsLangDisabled(lang string) bool { return c.config.C.DisabledLanguages[lang] } +func (c ConfigLanguage) IsKindEnabled(kind string) bool { + return !c.config.C.DisabledKinds[kind] +} + func (c ConfigLanguage) IgnoredLogs() map[string]bool { return c.config.C.IgnoredLogs } @@ -137,11 +143,11 @@ func (c ConfigLanguage) Watching() bool { return c.m.Base.Internal.Watch } -func (c ConfigLanguage) NewIdentityManager(name string, opts ...identity.ManagerOption) identity.Manager { +func (c ConfigLanguage) NewIdentityManager(opts ...identity.ManagerOption) identity.Manager { if !c.Watching() { return identity.NopManager } - return identity.NewManager(name, opts...) + return identity.NewManager(opts...) } func (c ConfigLanguage) ContentTypes() config.ContentTypesProvider { @@ -155,16 +161,24 @@ func (c ConfigLanguage) GetConfigSection(s string) any { return c.config.Security case "build": return c.config.Build + case "cascade": + return c.config.Cascade case "frontmatter": return c.config.Frontmatter case "caches": return c.config.Caches case "markup": return c.config.Markup + case "module": + return c.config.Module case "mediaTypes": return c.config.MediaTypes.Config case "outputFormats": return c.config.OutputFormats.Config + case "roles": + return c.config.Roles.Config + case "versions": + return c.config.Versions.Config case "permalinks": return c.config.Permalinks case "minify": @@ -212,6 +226,14 @@ func (c ConfigLanguage) DefaultContentLanguageInSubdir() bool { return c.config.DefaultContentLanguageInSubdir } +func (c ConfigLanguage) DefaultContentRoleInSubdir() bool { + return c.config.DefaultContentRoleInSubdir +} + +func (c ConfigLanguage) DefaultContentVersionInSubdir() bool { + return c.config.DefaultContentVersionInSubdir +} + func (c ConfigLanguage) SummaryLength() int { return c.config.SummaryLength } @@ -259,3 +281,15 @@ func (c ConfigLanguage) StaticDirs() []string { func (c ConfigLanguage) EnableEmoji() bool { return c.config.EnableEmoji } + +func (c ConfigLanguage) ConfiguredDimensions() *sitesmatrix.ConfiguredDimensions { + return c.m.ConfiguredDimensions +} + +func (c ConfigLanguage) DefaultContentsitesMatrix() *sitesmatrix.IntSets { + return c.m.DefaultContentSitesMatrix +} + +func (c ConfigLanguage) AllSitesMatrix() *sitesmatrix.IntSets { + return c.m.AllSitesMatrix +} diff --git a/config/allconfig/load.go b/config/allconfig/load.go index b76c4abde..4527a4554 100644 --- a/config/allconfig/load.go +++ b/config/allconfig/load.go @@ -32,7 +32,7 @@ import ( "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" - hglob "github.com/gohugoio/hugo/hugofs/glob" + hglob "github.com/gohugoio/hugo/hugofs/hglob" "github.com/gohugoio/hugo/modules" "github.com/gohugoio/hugo/parser/metadecoders" "github.com/spf13/afero" @@ -95,7 +95,7 @@ func LoadConfig(d ConfigSourceDescriptor) (configs *Configs, err error) { configs.Modules = moduleConfig.AllModules configs.ModulesClient = modulesClient - if err := configs.Init(); err != nil { + if err := configs.Init(d.Logger); err != nil { return nil, fmt.Errorf("failed to init config: %w", err) } diff --git a/config/configProvider.go b/config/configProvider.go index c21342dce..34e7ea365 100644 --- a/config/configProvider.go +++ b/config/configProvider.go @@ -20,15 +20,15 @@ import ( "github.com/gohugoio/hugo/common/paths" "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/common/urls" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "github.com/gohugoio/hugo/identity" - "github.com/gohugoio/hugo/langs" ) // AllProvider is a sub set of all config settings. type AllProvider interface { - Language() *langs.Language - Languages() langs.Languages - LanguagesDefaultFirst() langs.Languages + Language() any + LanguageIndex() int + Languages() any LanguagePrefix() string BaseURL() urls.BaseURL BaseURLLiveReload() urls.BaseURL @@ -50,6 +50,11 @@ type AllProvider interface { IsUglyURLs(section string) bool DefaultContentLanguage() string DefaultContentLanguageInSubdir() bool + DefaultContentRoleInSubdir() bool + DefaultContentVersionInSubdir() bool + DefaultContentsitesMatrix() *sitesmatrix.IntSets + AllSitesMatrix() *sitesmatrix.IntSets + IsKindEnabled(string) bool IsLangDisabled(string) bool SummaryLength() int Pagination() Pagination @@ -58,7 +63,7 @@ type AllProvider interface { BuildDrafts() bool Running() bool Watching() bool - NewIdentityManager(name string, opts ...identity.ManagerOption) identity.Manager + NewIdentityManager(opts ...identity.ManagerOption) identity.Manager FastRenderMode() bool PrintUnusedTemplates() bool EnableMissingTranslationPlaceholders() bool @@ -73,6 +78,7 @@ type AllProvider interface { IgnoredLogs() map[string]bool WorkingDir() string EnableEmoji() bool + ConfiguredDimensions() *sitesmatrix.ConfiguredDimensions } // We cannot import the media package as that would create a circular dependency. @@ -110,3 +116,6 @@ func GetStringSlicePreserveString(cfg Provider, key string) []string { sd := cfg.Get(key) return types.ToStringSlicePreserveString(sd) } + +/*func (cd ConfiguredDimensions) Language(v sitesmatrix.Vector) ConfiguredDimension { +}*/ diff --git a/config/defaultConfigProvider.go b/config/defaultConfigProvider.go index 11cadbbba..8e0a294eb 100644 --- a/config/defaultConfigProvider.go +++ b/config/defaultConfigProvider.go @@ -16,12 +16,10 @@ package config import ( "errors" "fmt" - "slices" + "sort" "strings" "sync" - xmaps "maps" - "github.com/spf13/cast" "github.com/gohugoio/hugo/common/maps" @@ -239,7 +237,12 @@ func (c *defaultConfigProvider) Merge(k string, v any) { func (c *defaultConfigProvider) Keys() []string { c.mu.RLock() defer c.mu.RUnlock() - return slices.Collect(xmaps.Keys(c.root)) + var keys []string + for k := range c.root { + keys = append(keys, k) + } + sort.Strings(keys) + return keys } func (c *defaultConfigProvider) WalkParams(walkFn func(params ...maps.KeyParams) bool) { diff --git a/create/content.go b/create/content.go index a4661c1ba..89c20242d 100644 --- a/create/content.go +++ b/create/content.go @@ -16,6 +16,7 @@ package create import ( "bytes" + "context" "errors" "fmt" "io" @@ -23,11 +24,9 @@ import ( "path/filepath" "strings" - "github.com/gohugoio/hugo/hugofs/glob" - "github.com/gohugoio/hugo/common/hexec" - "github.com/gohugoio/hugo/common/hstrings" "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/hugofs/hglob" "github.com/gohugoio/hugo/hugofs" @@ -159,10 +158,10 @@ func (b *contentBuilder) buildDir() error { contentTargetFilenames = append(contentTargetFilenames, abs) } - var contentInclusionFilter *glob.FilenameFilter + var contentInclusionFilter *hglob.FilenameFilter if !b.dirMap.siteUsed { // We don't need to build everything. - contentInclusionFilter = glob.NewFilenameFilterForInclusionFunc(func(filename string) bool { + contentInclusionFilter = hglob.NewFilenameFilterForInclusionFunc(func(filename string) bool { filename = strings.TrimPrefix(filename, string(os.PathSeparator)) for _, cn := range contentTargetFilenames { if strings.Contains(cn, filename) { @@ -228,10 +227,10 @@ func (b *contentBuilder) buildFile() (string, error) { return "", err } - var contentInclusionFilter *glob.FilenameFilter + var contentInclusionFilter *hglob.FilenameFilter if !usesSite { // We don't need to build everything. - contentInclusionFilter = glob.NewFilenameFilterForInclusionFunc(func(filename string) bool { + contentInclusionFilter = hglob.NewFilenameFilterForInclusionFunc(func(filename string) bool { filename = strings.TrimPrefix(filename, string(os.PathSeparator)) return strings.Contains(contentPlaceholderAbsFilename, filename) }) @@ -291,9 +290,7 @@ func (b *contentBuilder) applyArcheType(contentFilename string, archetypeFi hugo func (b *contentBuilder) mapArcheTypeDir() error { var m archetypeMap - seen := map[hstrings.Strings2]bool{} - - walkFn := func(path string, fim hugofs.FileMetaInfo) error { + walkFn := func(ctx context.Context, path string, fim hugofs.FileMetaInfo) error { if fim.IsDir() { return nil } @@ -301,15 +298,6 @@ func (b *contentBuilder) mapArcheTypeDir() error { pi := fim.Meta().PathInfo if pi.IsContent() { - pathLang := hstrings.Strings2{pi.PathBeforeLangAndOutputFormatAndExt(), fim.Meta().Lang} - if seen[pathLang] { - // Duplicate content file, e.g. page.md and page.html. - // In the regular build, we will filter out the duplicates, but - // for archetype folders these are ambiguous and we need to - // fail. - return fmt.Errorf("duplicate content file found in archetype folder: %q; having both e.g. %s.md and %s.html is ambigous", path, pi.BaseNameNoIdentifier(), pi.BaseNameNoIdentifier()) - } - seen[pathLang] = true m.contentFiles = append(m.contentFiles, fim) if !m.siteUsed { var err error diff --git a/create/content_test.go b/create/content_test.go index 429edfc26..22629fdbc 100644 --- a/create/content_test.go +++ b/create/content_test.go @@ -136,23 +136,27 @@ site RegularPages: {{ len site.RegularPages }} cfg, fs := newTestCfg(c, mm) conf := testconfig.GetTestConfigs(fs.Source, cfg) - h, err := hugolib.NewHugoSites(deps.DepsCfg{Configs: conf, Fs: fs}) - c.Assert(err, qt.IsNil) - c.Assert(len(h.Sites), qt.Equals, 2) - c.Assert(create.NewContent(h, "my-bundle", "post/my-post", false), qt.IsNil) + h := func() *hugolib.HugoSites { + h, err := hugolib.NewHugoSites(deps.DepsCfg{Configs: conf, Fs: fs}) + c.Assert(err, qt.IsNil) + c.Assert(len(h.Sites), qt.Equals, 2) + return h + } + + c.Assert(create.NewContent(h(), "my-bundle", "post/my-post", false), qt.IsNil) cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.md")), `site RegularPages: 10`) // Default bundle archetype - c.Assert(create.NewContent(h, "", "post/my-post2", false), qt.IsNil) + c.Assert(create.NewContent(h(), "", "post/my-post2", false), qt.IsNil) cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post2/index.md")), `default archetype index.md`) // Regular file with bundle kind. - c.Assert(create.NewContent(h, "my-bundle", "post/foo.md", false), qt.IsNil) + c.Assert(create.NewContent(h(), "my-bundle", "post/foo.md", false), qt.IsNil) cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/foo.md")), `draft: true`) // Regular files should fall back to the default archetype (we have no regular file archetype). - c.Assert(create.NewContent(h, "my-bundle", "mypage.md", false), qt.IsNil) + c.Assert(create.NewContent(h(), "my-bundle", "mypage.md", false), qt.IsNil) cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "mypage.md")), `draft: true`) } diff --git a/deploy/deployconfig/deployConfig.go b/deploy/deployconfig/deployConfig.go index b16b7c627..94dbc0cf6 100644 --- a/deploy/deployconfig/deployConfig.go +++ b/deploy/deployconfig/deployConfig.go @@ -20,7 +20,7 @@ import ( "github.com/gobwas/glob" "github.com/gohugoio/hugo/config" - hglob "github.com/gohugoio/hugo/hugofs/glob" + hglob "github.com/gohugoio/hugo/hugofs/hglob" "github.com/mitchellh/mapstructure" ) diff --git a/deps/deps.go b/deps/deps.go index d0d6d95fc..acafb850e 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -217,14 +217,14 @@ func (d *Deps) Init() error { []byte(tpl.HugoDeferredTemplatePrefix), []byte(postpub.PostProcessPrefix)) - pathSpec, err := helpers.NewPathSpec(d.Fs, d.Conf, d.Log) + pathSpec, err := helpers.NewPathSpec(d.Fs, d.Conf, d.Log, nil) if err != nil { return err } d.PathSpec = pathSpec } else { var err error - d.PathSpec, err = helpers.NewPathSpecWithBaseBaseFsProvided(d.Fs, d.Conf, d.Log, d.PathSpec.BaseFs) + d.PathSpec, err = helpers.NewPathSpec(d.Fs, d.Conf, d.Log, d.PathSpec.BaseFs) if err != nil { return err } diff --git a/go.mod b/go.mod index ec1f40e4e..8adf07fba 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/bep/overlayfs v0.10.0 github.com/bep/simplecobra v0.6.1 github.com/bep/tmc v0.5.1 + github.com/bits-and-blooms/bitset v1.22.0 github.com/cespare/xxhash/v2 v2.3.0 github.com/clbanning/mxj/v2 v2.7.0 github.com/disintegration/gift v1.2.1 diff --git a/go.sum b/go.sum index 079c4da63..d8b367ceb 100644 --- a/go.sum +++ b/go.sum @@ -182,6 +182,8 @@ github.com/bep/tmc v0.5.1 h1:CsQnSC6MsomH64gw0cT5f+EwQDcvZz4AazKunFwTpuI= github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0= github.com/bep/workers v1.0.0 h1:U+H8YmEaBCEaFZBst7GcRVEoqeRC9dzH2dWOwGmOchg= github.com/bep/workers v1.0.0/go.mod h1:7kIESOB86HfR2379pwoMWNy8B50D7r99fRLUyPSNyCs= +github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4= +github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= diff --git a/helpers/general.go b/helpers/general.go index a9b0c0cd1..df62e947b 100644 --- a/helpers/general.go +++ b/helpers/general.go @@ -21,7 +21,6 @@ import ( "os" "path/filepath" "slices" - "sort" "strings" "unicode" "unicode/utf8" @@ -59,65 +58,6 @@ func FirstUpper(s string) string { return string(unicode.ToUpper(r)) + s[n:] } -// UniqueStrings returns a new slice with any duplicates removed. -func UniqueStrings(s []string) []string { - unique := make([]string, 0, len(s)) - for i, val := range s { - var seen bool - for j := range i { - if s[j] == val { - seen = true - break - } - } - if !seen { - unique = append(unique, val) - } - } - return unique -} - -// UniqueStringsReuse returns a slice with any duplicates removed. -// It will modify the input slice. -func UniqueStringsReuse(s []string) []string { - result := s[:0] - for i, val := range s { - var seen bool - - for j := range i { - if s[j] == val { - seen = true - break - } - } - - if !seen { - result = append(result, val) - } - } - return result -} - -// UniqueStringsSorted returns a sorted slice with any duplicates removed. -// It will modify the input slice. -func UniqueStringsSorted(s []string) []string { - if len(s) == 0 { - return nil - } - ss := sort.StringSlice(s) - ss.Sort() - i := 0 - for j := 1; j < len(s); j++ { - if !ss.Less(i, j) { - continue - } - i++ - s[i] = s[j] - } - - return s[:i+1] -} - // ReaderToBytes takes an io.Reader argument, reads from it // and returns bytes. func ReaderToBytes(lines io.Reader) []byte { diff --git a/helpers/general_test.go b/helpers/general_test.go index 6d2a78e02..f7bcb6642 100644 --- a/helpers/general_test.go +++ b/helpers/general_test.go @@ -14,7 +14,6 @@ package helpers_test import ( - "reflect" "strings" "testing" @@ -226,84 +225,6 @@ func BenchmarkReaderContains(b *testing.B) { } } -func TestUniqueStrings(t *testing.T) { - in := []string{"a", "b", "a", "b", "c", "", "a", "", "d"} - output := helpers.UniqueStrings(in) - expected := []string{"a", "b", "c", "", "d"} - if !reflect.DeepEqual(output, expected) { - t.Errorf("Expected %#v, got %#v\n", expected, output) - } -} - -func TestUniqueStringsReuse(t *testing.T) { - in := []string{"a", "b", "a", "b", "c", "", "a", "", "d"} - output := helpers.UniqueStringsReuse(in) - expected := []string{"a", "b", "c", "", "d"} - if !reflect.DeepEqual(output, expected) { - t.Errorf("Expected %#v, got %#v\n", expected, output) - } -} - -func TestUniqueStringsSorted(t *testing.T) { - c := qt.New(t) - in := []string{"a", "a", "b", "c", "b", "", "a", "", "d"} - output := helpers.UniqueStringsSorted(in) - expected := []string{"", "a", "b", "c", "d"} - c.Assert(output, qt.DeepEquals, expected) - c.Assert(helpers.UniqueStringsSorted(nil), qt.IsNil) -} - -func BenchmarkUniqueStrings(b *testing.B) { - input := []string{"a", "b", "d", "e", "d", "h", "a", "i"} - - b.Run("Safe", func(b *testing.B) { - for i := 0; i < b.N; i++ { - result := helpers.UniqueStrings(input) - if len(result) != 6 { - b.Fatalf("invalid count: %d", len(result)) - } - } - }) - - b.Run("Reuse slice", func(b *testing.B) { - b.StopTimer() - inputs := make([][]string, b.N) - for i := 0; i < b.N; i++ { - inputc := make([]string, len(input)) - copy(inputc, input) - inputs[i] = inputc - } - b.StartTimer() - for i := 0; i < b.N; i++ { - inputc := inputs[i] - - result := helpers.UniqueStringsReuse(inputc) - if len(result) != 6 { - b.Fatalf("invalid count: %d", len(result)) - } - } - }) - - b.Run("Reuse slice sorted", func(b *testing.B) { - b.StopTimer() - inputs := make([][]string, b.N) - for i := 0; i < b.N; i++ { - inputc := make([]string, len(input)) - copy(inputc, input) - inputs[i] = inputc - } - b.StartTimer() - for i := 0; i < b.N; i++ { - inputc := inputs[i] - - result := helpers.UniqueStringsSorted(inputc) - if len(result) != 6 { - b.Fatalf("invalid count: %d", len(result)) - } - } - }) -} - func TestStringSliceToList(t *testing.T) { for _, tt := range []struct { slice []string diff --git a/helpers/path.go b/helpers/path.go index ba95be605..aee617282 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -25,6 +25,7 @@ import ( "strings" "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hstrings" "github.com/gohugoio/hugo/common/text" "github.com/gohugoio/hugo/htesting" @@ -188,7 +189,7 @@ func ExtractAndGroupRootPaths(paths []string) []NamedSlice { groupsStr[i] = strings.Join(g, "/") } - groupsStr = UniqueStringsSorted(groupsStr) + groupsStr = hstrings.UniqueStringsSorted(groupsStr) var result []NamedSlice @@ -209,7 +210,7 @@ func ExtractAndGroupRootPaths(paths []string) []NamedSlice { } } - ns.Slice = UniqueStrings(ExtractRootPaths(ns.Slice)) + ns.Slice = hstrings.UniqueStrings(ExtractRootPaths(ns.Slice)) result = append(result, ns) } diff --git a/helpers/pathspec.go b/helpers/pathspec.go index 88571b93c..e01939f1b 100644 --- a/helpers/pathspec.go +++ b/helpers/pathspec.go @@ -32,19 +32,11 @@ type PathSpec struct { // The file systems to use Fs *hugofs.Fs - - // The config provider to use - Cfg config.AllProvider } // NewPathSpec creates a new PathSpec from the given filesystems and language. -func NewPathSpec(fs *hugofs.Fs, cfg config.AllProvider, logger loggers.Logger) (*PathSpec, error) { - return NewPathSpecWithBaseBaseFsProvided(fs, cfg, logger, nil) -} - -// NewPathSpecWithBaseBaseFsProvided creates a new PathSpec from the given filesystems and language. // If an existing BaseFs is provided, parts of that is reused. -func NewPathSpecWithBaseBaseFsProvided(fs *hugofs.Fs, cfg config.AllProvider, logger loggers.Logger, baseBaseFs *filesystems.BaseFs) (*PathSpec, error) { +func NewPathSpec(fs *hugofs.Fs, cfg config.AllProvider, logger loggers.Logger, baseBaseFs *filesystems.BaseFs) (*PathSpec, error) { p, err := paths.New(fs, cfg) if err != nil { return nil, err @@ -65,7 +57,6 @@ func NewPathSpecWithBaseBaseFsProvided(fs *hugofs.Fs, cfg config.AllProvider, lo Paths: p, BaseFs: bfs, Fs: fs, - Cfg: cfg, ProcessingStats: NewProcessingStats(p.Lang()), } diff --git a/helpers/testhelpers_test.go b/helpers/testhelpers_test.go index 6c5f62488..46c177974 100644 --- a/helpers/testhelpers_test.go +++ b/helpers/testhelpers_test.go @@ -23,7 +23,7 @@ func newTestPathSpecFromCfgAndLang(cfg config.Provider, lang string) *helpers.Pa } } fs := hugofs.NewFrom(mfs, conf.BaseConfig()) - ps, err := helpers.NewPathSpec(fs, conf, loggers.NewDefault()) + ps, err := helpers.NewPathSpec(fs, conf, loggers.NewDefault(), nil) if err != nil { panic(err) } diff --git a/hugofs/component_fs.go b/hugofs/component_fs.go index f491d4d0f..b4aaa2acd 100644 --- a/hugofs/component_fs.go +++ b/hugofs/component_fs.go @@ -14,15 +14,19 @@ package hugofs import ( + "context" iofs "io/fs" "os" "path" "runtime" "sort" + "github.com/bep/helpers/contexthelpers" "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/hugofs/files" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "github.com/spf13/afero" "golang.org/x/text/unicode/norm" ) @@ -39,7 +43,10 @@ func NewComponentFs(opts ComponentFsOptions) *componentFs { return &componentFs{Fs: bfs, opts: opts} } -var _ FilesystemUnwrapper = (*componentFs)(nil) +var ( + _ FilesystemUnwrapper = (*componentFs)(nil) + _ ReadDirWithContextDir = (*componentFsDir)(nil) +) // componentFs is a filesystem that holds one of the Hugo components, e.g. content, layouts etc. type componentFs struct { @@ -59,12 +66,24 @@ type componentFsDir struct { fs *componentFs } -// ReadDir reads count entries from this virtual directory and -// sorts the entries according to the component filesystem rules. -func (f *componentFsDir) ReadDir(count int) ([]iofs.DirEntry, error) { +type ( + contextKey uint8 +) + +const ( + contextKeyIsInLeafBundle contextKey = iota +) + +var componentFsContext = struct { + IsInLeafBundle contexthelpers.ContextDispatcher[bool] +}{ + IsInLeafBundle: contexthelpers.NewContextDispatcher[bool](contextKeyIsInLeafBundle), +} + +func (f *componentFsDir) ReadDirWithContext(ctx context.Context, count int) ([]iofs.DirEntry, context.Context, error) { fis, err := f.DirOnlyOps.(iofs.ReadDirFile).ReadDir(-1) if err != nil { - return nil, err + return nil, ctx, err } // Filter out any symlinks. @@ -79,7 +98,7 @@ func (f *componentFsDir) ReadDir(count int) ([]iofs.DirEntry, error) { if herrors.IsNotExist(err) { continue } - return nil, err + return nil, ctx, err } if info.Mode()&os.ModeSymlink == 0 { keep = true @@ -92,8 +111,8 @@ func (f *componentFsDir) ReadDir(count int) ([]iofs.DirEntry, error) { } fis = fis[:n] - n = 0 + for _, fi := range fis { s := path.Join(f.name, fi.Name()) if _, ok := f.fs.applyMeta(fi, s); ok { @@ -101,7 +120,9 @@ func (f *componentFsDir) ReadDir(count int) ([]iofs.DirEntry, error) { n++ } } + fis = fis[:n] + n = 0 sort.Slice(fis, func(i, j int) bool { fimi, fimj := fis[i].(FileMetaInfo), fis[j].(FileMetaInfo) @@ -110,6 +131,16 @@ func (f *componentFsDir) ReadDir(count int) ([]iofs.DirEntry, error) { } fimim, fimjm := fimi.Meta(), fimj.Meta() + bi, bj := fimim.PathInfo.Base(), fimjm.PathInfo.Base() + if bi == bj { + matrixi, matrixj := fimim.SitesMatrix, fimjm.SitesMatrix + l1, l2 := matrixi.LenVectors(), matrixj.LenVectors() + if l1 != l2 { + // Pull the ones with the least number of sites defined to the top. + return l1 < l2 + } + } + if fimim.ModuleOrdinal != fimjm.ModuleOrdinal { switch f.fs.opts.Component { case files.ComponentFolderI18n: @@ -133,7 +164,6 @@ func (f *componentFsDir) ReadDir(count int) ([]iofs.DirEntry, error) { } if exti != extj { - // This pulls .md above .html. return exti > extj } @@ -149,7 +179,108 @@ func (f *componentFsDir) ReadDir(count int) ([]iofs.DirEntry, error) { return fimi.Name() < fimj.Name() }) - return fis, nil + if f.fs.opts.Component == files.ComponentFolderContent { + isInLeafBundle := componentFsContext.IsInLeafBundle.Get(ctx) + var isCurrentLeafBundle bool + for _, fi := range fis { + if fi.IsDir() { + continue + } + + pi := fi.(FileMetaInfo).Meta().PathInfo + + if pi.IsLeafBundle() { + isCurrentLeafBundle = true + break + } + } + + if !isInLeafBundle && isCurrentLeafBundle { + ctx = componentFsContext.IsInLeafBundle.Set(ctx, true) + } + + if isInLeafBundle || isCurrentLeafBundle { + for _, fi := range fis { + if fi.IsDir() { + continue + } + pi := fi.(FileMetaInfo).Meta().PathInfo + + // Everything below a leaf bundle is a resource. + isResource := isInLeafBundle && pi.Type() > paths.TypeFile + // Every sibling of a leaf bundle is a resource. + isResource = isResource || (isCurrentLeafBundle && !pi.IsLeafBundle()) + + if isResource { + paths.ModifyPathBundleTypeResource(pi) + } + } + } + + } + + type typeBase struct { + Type paths.Type + Base string + } + + variants := make(map[typeBase][]sitesmatrix.VectorProvider) + + for _, fi := range fis { + + if !fi.IsDir() { + meta := fi.(FileMetaInfo).Meta() + + pi := meta.PathInfo + + if pi.Component() == files.ComponentFolderLayouts || pi.Component() == files.ComponentFolderContent { + + var base string + switch pi.Component() { + case files.ComponentFolderContent: + base = pi.Base() + pi.Custom() + default: + base = pi.PathNoLang() + } + + baseName := typeBase{pi.Type(), base} + + // There may be multiple languge/version/role combinations for the same file. + // The most important come early. + matrixes, found := variants[baseName] + + if found { + complement := meta.SitesMatrix.Complement(matrixes...) + if complement == nil || complement.LenVectors() == 0 { + continue + } + matrixes = append(matrixes, meta.SitesMatrix) + meta.SitesMatrix = complement + + variants[baseName] = matrixes + + } else { + matrixes = []sitesmatrix.VectorProvider{meta.SitesMatrix} + variants[baseName] = matrixes + + } + } + } + + fis[n] = fi + n++ + + } + fis = fis[:n] + + return fis, ctx, nil +} + +// ReadDir reads count entries from this virtual directory and +// sorts the entries according to the component filesystem rules. +func (f *componentFsDir) ReadDir(count int) ([]iofs.DirEntry, error) { + v, _, err := ReadDirWithContext(context.Background(), f.DirOnlyOps, count) + return v, err } func (f *componentFsDir) Stat() (iofs.FileInfo, error) { @@ -176,34 +307,36 @@ func (fs *componentFs) applyMeta(fi FileNameIsDir, name string) (FileMetaInfo, b } fim := fi.(FileMetaInfo) meta := fim.Meta() - pi := fs.opts.PathParser.Parse(fs.opts.Component, name) + pi := fs.opts.Cfg.PathParser().Parse(fs.opts.Component, name) if pi.Disabled() { return fim, false } - if meta.Lang != "" { - if isLangDisabled := fs.opts.PathParser.IsLangDisabled; isLangDisabled != nil && isLangDisabled(meta.Lang) { - return fim, false - } - } + meta.PathInfo = pi if !fim.IsDir() { if fileLang := meta.PathInfo.Lang(); fileLang != "" { - // A valid lang set in filename. - // Give priority to myfile.sv.txt inside the sv filesystem. - meta.Weight++ - meta.Lang = fileLang - } - } + if idx, ok := fs.opts.Cfg.PathParser().LanguageIndex[fileLang]; ok { + // A valid lang set in filename. + // Give priority to myfile.sv.txt inside the sv filesystem. + meta.Weight++ + meta.SitesMatrix = meta.SitesMatrix.WithLanguageIndices(idx) + if idx > 0 { + // Not the default language, add some weight. + meta.SitesMatrix = sitesmatrix.NewWeightedVectorStore(meta.SitesMatrix, 10) + } - if meta.Lang == "" { - meta.Lang = fs.opts.DefaultContentLanguage - } + } + } + switch meta.Component { + case files.ComponentFolderLayouts: + // Eg. index.fr.html when French isn't defined, + // we want e.g. index.html to be used instead. + if len(pi.IdentifiersUnknown()) > 0 { + meta.Weight-- + } + } - langIdx, found := fs.opts.PathParser.LanguageIndex[meta.Lang] - if !found { - panic("no language found for " + meta.Lang) } - meta.LangIndex = langIdx if fi.IsDir() { meta.OpenFunc = func() (afero.File, error) { @@ -238,10 +371,7 @@ type ComponentFsOptions struct { // The component name, e.g. "content", "layouts" etc. Component string - DefaultContentLanguage string - - // The parser used to parse paths provided by this filesystem. - PathParser *paths.PathParser + Cfg config.AllProvider } func (fs *componentFs) Open(name string) (afero.File, error) { diff --git a/hugofs/dirsmerger.go b/hugofs/dirsmerger.go index 9eedb5844..3e87829ff 100644 --- a/hugofs/dirsmerger.go +++ b/hugofs/dirsmerger.go @@ -19,31 +19,9 @@ import ( "github.com/bep/overlayfs" ) -// LanguageDirsMerger implements the overlayfs.DirsMerger func, which is used -// to merge two directories. -var LanguageDirsMerger overlayfs.DirsMerger = func(lofi, bofi []fs.DirEntry) []fs.DirEntry { - for _, fi1 := range bofi { - fim1 := fi1.(FileMetaInfo) - var found bool - for _, fi2 := range lofi { - fim2 := fi2.(FileMetaInfo) - if fi1.Name() == fi2.Name() && fim1.Meta().Lang == fim2.Meta().Lang { - found = true - break - } - } - if !found { - lofi = append(lofi, fi1) - } - } - - return lofi -} - // AppendDirsMerger merges two directories keeping all regular files // with the first slice as the base. // Duplicate directories in the second slice will be ignored. -// This strategy is used for the i18n and data fs where we need all entries. var AppendDirsMerger overlayfs.DirsMerger = func(lofi, bofi []fs.DirEntry) []fs.DirEntry { for _, fi1 := range bofi { var found bool diff --git a/hugofs/fileinfo.go b/hugofs/fileinfo.go index 5e6a87acc..f5cb865e1 100644 --- a/hugofs/fileinfo.go +++ b/hugofs/fileinfo.go @@ -27,7 +27,8 @@ import ( "sync" "time" - "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/hugofs/hglob" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "golang.org/x/text/unicode/norm" @@ -58,19 +59,17 @@ type FileMeta struct { IsProject bool Watch bool - // The lang associated with this file. This may be - // either the language set in the filename or - // the language defined in the source mount configuration. - Lang string - // The language index for the above lang. This is the index - // in the sorted list of languages/sites. - LangIndex int + // This file/directory will be built for these sites. + SitesMatrix sitesmatrix.VectorStore + + // This file/directory complements these other sites. + SitesComplements sitesmatrix.VectorStore OpenFunc func() (afero.File, error) JoinStatFunc func(name string) (FileMetaInfo, error) // Include only files or directories that match. - InclusionFilter *glob.FilenameFilter + InclusionFilter *hglob.FilenameFilter // Rename the name part of the file (not the directory). // Returns the new name and a boolean indicating if the file diff --git a/hugofs/filename_filter_fs.go b/hugofs/filename_filter_fs.go index 5bae4b876..344b50e95 100644 --- a/hugofs/filename_filter_fs.go +++ b/hugofs/filename_filter_fs.go @@ -20,13 +20,13 @@ import ( "syscall" "time" - "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/hugofs/hglob" "github.com/spf13/afero" ) var _ FilesystemUnwrapper = (*filenameFilterFs)(nil) -func newFilenameFilterFs(fs afero.Fs, base string, filter *glob.FilenameFilter) afero.Fs { +func newFilenameFilterFs(fs afero.Fs, base string, filter *hglob.FilenameFilter) afero.Fs { return &filenameFilterFs{ fs: fs, base: base, @@ -39,7 +39,7 @@ type filenameFilterFs struct { base string fs afero.Fs - filter *glob.FilenameFilter + filter *hglob.FilenameFilter } func (fs *filenameFilterFs) UnwrapFilesystem() afero.Fs { @@ -90,7 +90,7 @@ func (fs *filenameFilterFs) Stat(name string) (os.FileInfo, error) { type filenameFilterDir struct { afero.File base string - filter *glob.FilenameFilter + filter *hglob.FilenameFilter } func (f *filenameFilterDir) ReadDir(n int) ([]fs.DirEntry, error) { diff --git a/hugofs/filename_filter_fs_test.go b/hugofs/filename_filter_fs_test.go index 7b31f0f82..521d4a74d 100644 --- a/hugofs/filename_filter_fs_test.go +++ b/hugofs/filename_filter_fs_test.go @@ -20,8 +20,7 @@ import ( "path/filepath" "testing" - "github.com/gohugoio/hugo/hugofs/glob" - + "github.com/gohugoio/hugo/hugofs/hglob" "github.com/spf13/afero" qt "github.com/frankban/quicktest" @@ -43,7 +42,7 @@ func TestFilenameFilterFs(t *testing.T) { fs = NewBasePathFs(fs, base) - filter, err := glob.NewFilenameFilter(nil, []string{"/b/**.txt"}) + filter, err := hglob.NewFilenameFilter(nil, []string{"/b/**.txt"}) c.Assert(err, qt.IsNil) fs = newFilenameFilterFs(fs, base, filter) diff --git a/hugofs/fs.go b/hugofs/fs.go index aecf72a7f..ecc62470e 100644 --- a/hugofs/fs.go +++ b/hugofs/fs.go @@ -15,7 +15,9 @@ package hugofs import ( + "context" "fmt" + iofs "io/fs" "os" "strings" @@ -250,3 +252,18 @@ type filesystemsWrapper struct { func (w filesystemsWrapper) UnwrapFilesystem() afero.Fs { return w.content } + +type ReadDirWithContextDir interface { + ReadDirWithContext(context context.Context, count int) ([]iofs.DirEntry, context.Context, error) +} + +func ReadDirWithContext(ctx context.Context, f DirOnlyOps, count int) ([]iofs.DirEntry, context.Context, error) { + if ff, ok := f.(ReadDirWithContextDir); ok { + return ff.ReadDirWithContext(ctx, count) + } + v, err := f.(iofs.ReadDirFile).ReadDir(count) + if err != nil { + return nil, ctx, err + } + return v, ctx, nil +} diff --git a/hugofs/glob.go b/hugofs/glob.go index 6a6d999ce..9e6e0b71b 100644 --- a/hugofs/glob.go +++ b/hugofs/glob.go @@ -14,29 +14,29 @@ package hugofs import ( + "context" "errors" "path/filepath" "strings" - "github.com/gohugoio/hugo/hugofs/glob" - + "github.com/gohugoio/hugo/hugofs/hglob" "github.com/spf13/afero" ) // Glob walks the fs and passes all matches to the handle func. // The handle func can return true to signal a stop. func Glob(fs afero.Fs, pattern string, handle func(fi FileMetaInfo) (bool, error)) error { - pattern = glob.NormalizePathNoLower(pattern) + pattern = hglob.NormalizePathNoLower(pattern) if pattern == "" { return nil } - root := glob.ResolveRootDir(pattern) + root := hglob.ResolveRootDir(pattern) if !strings.HasPrefix(root, "/") { root = "/" + root } pattern = strings.ToLower(pattern) - g, err := glob.GetGlob(pattern) + g, err := hglob.GetGlob(pattern) if err != nil { return err } @@ -47,8 +47,8 @@ func Glob(fs afero.Fs, pattern string, handle func(fi FileMetaInfo) (bool, error // Signals that we're done. done := errors.New("done") - wfn := func(p string, info FileMetaInfo) error { - p = glob.NormalizePath(p) + wfn := func(ctx context.Context, p string, info FileMetaInfo) error { + p = hglob.NormalizePath(p) if info.IsDir() { if !hasSuperAsterisk { // Avoid walking to the bottom if we can avoid it. diff --git a/hugofs/glob/filename_filter.go b/hugofs/glob/filename_filter.go deleted file mode 100644 index 6f283de48..000000000 --- a/hugofs/glob/filename_filter.go +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright 2021 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package glob - -import ( - "path" - "path/filepath" - "strings" - - "github.com/gobwas/glob" -) - -type FilenameFilter struct { - shouldInclude func(filename string) bool - inclusions []glob.Glob - dirInclusions []glob.Glob - exclusions []glob.Glob - isWindows bool - - nested []*FilenameFilter -} - -func normalizeFilenameGlobPattern(s string) string { - // Use Unix separators even on Windows. - s = filepath.ToSlash(s) - if !strings.HasPrefix(s, "/") { - s = "/" + s - } - return s -} - -// NewFilenameFilter creates a new Glob where the Match method will -// return true if the file should be included. -// Note that the inclusions will be checked first. -func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error) { - if inclusions == nil && exclusions == nil { - return nil, nil - } - filter := &FilenameFilter{isWindows: isWindows} - - for _, include := range inclusions { - include = normalizeFilenameGlobPattern(include) - g, err := GetGlob(include) - if err != nil { - return nil, err - } - filter.inclusions = append(filter.inclusions, g) - - // For mounts that do directory walking (e.g. content) we - // must make sure that all directories up to this inclusion also - // gets included. - dir := path.Dir(include) - parts := strings.Split(dir, "/") - for i := range parts { - pattern := "/" + filepath.Join(parts[:i+1]...) - g, err := GetGlob(pattern) - if err != nil { - return nil, err - } - filter.dirInclusions = append(filter.dirInclusions, g) - } - } - - for _, exclude := range exclusions { - exclude = normalizeFilenameGlobPattern(exclude) - g, err := GetGlob(exclude) - if err != nil { - return nil, err - } - filter.exclusions = append(filter.exclusions, g) - } - - return filter, nil -} - -// MustNewFilenameFilter invokes NewFilenameFilter and panics on error. -func MustNewFilenameFilter(inclusions, exclusions []string) *FilenameFilter { - filter, err := NewFilenameFilter(inclusions, exclusions) - if err != nil { - panic(err) - } - return filter -} - -// NewFilenameFilterForInclusionFunc create a new filter using the provided inclusion func. -func NewFilenameFilterForInclusionFunc(shouldInclude func(filename string) bool) *FilenameFilter { - return &FilenameFilter{shouldInclude: shouldInclude, isWindows: isWindows} -} - -// Match returns whether filename should be included. -func (f *FilenameFilter) Match(filename string, isDir bool) bool { - if f == nil { - return true - } - if !f.doMatch(filename, isDir) { - return false - } - - for _, nested := range f.nested { - if !nested.Match(filename, isDir) { - return false - } - } - - return true -} - -// Append appends a filter to the chain. The receiver will be copied if needed. -func (f *FilenameFilter) Append(other *FilenameFilter) *FilenameFilter { - if f == nil { - return other - } - - clone := *f - nested := make([]*FilenameFilter, len(clone.nested)+1) - copy(nested, clone.nested) - nested[len(nested)-1] = other - clone.nested = nested - - return &clone -} - -func (f *FilenameFilter) doMatch(filename string, isDir bool) bool { - if f == nil { - return true - } - - if !strings.HasPrefix(filename, filepathSeparator) { - filename = filepathSeparator + filename - } - - if f.shouldInclude != nil { - if f.shouldInclude(filename) { - return true - } - if f.isWindows { - // The Glob matchers below handles this by themselves, - // for the shouldInclude we need to take some extra steps - // to make this robust. - winFilename := filepath.FromSlash(filename) - if filename != winFilename { - if f.shouldInclude(winFilename) { - return true - } - } - } - - } - - for _, inclusion := range f.inclusions { - if inclusion.Match(filename) { - return true - } - } - - if isDir && f.inclusions != nil { - for _, inclusion := range f.dirInclusions { - if inclusion.Match(filename) { - return true - } - } - } - - for _, exclusion := range f.exclusions { - if exclusion.Match(filename) { - return false - } - } - - return f.inclusions == nil && f.shouldInclude == nil -} diff --git a/hugofs/glob/filename_filter_test.go b/hugofs/glob/filename_filter_test.go deleted file mode 100644 index 6398e8a1e..000000000 --- a/hugofs/glob/filename_filter_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2021 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package glob - -import ( - "path/filepath" - "strings" - "testing" - - qt "github.com/frankban/quicktest" -) - -func TestFilenameFilter(t *testing.T) { - c := qt.New(t) - - excludeAlmostAllJSON, err := NewFilenameFilter([]string{"/a/b/c/foo.json"}, []string{"**.json"}) - c.Assert(err, qt.IsNil) - c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/data/my.json"), false), qt.Equals, false) - c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c/foo.json"), false), qt.Equals, true) - c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c/foo.bar"), false), qt.Equals, false) - c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c"), true), qt.Equals, true) - c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b"), true), qt.Equals, true) - c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a"), true), qt.Equals, true) - c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/"), true), qt.Equals, true) - c.Assert(excludeAlmostAllJSON.Match("", true), qt.Equals, true) - - excludeAllButFooJSON, err := NewFilenameFilter([]string{"/a/**/foo.json"}, []string{"**.json"}) - c.Assert(err, qt.IsNil) - c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/data/my.json"), false), qt.Equals, false) - c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/c/d/e/foo.json"), false), qt.Equals, true) - c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/c"), true), qt.Equals, true) - c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/"), true), qt.Equals, true) - c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/"), true), qt.Equals, true) - c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/b"), true), qt.Equals, false) - - excludeAllButFooJSONMixedCasePattern, err := NewFilenameFilter([]string{"/**/Foo.json"}, nil) - c.Assert(excludeAllButFooJSONMixedCasePattern.Match(filepath.FromSlash("/a/b/c/d/e/foo.json"), false), qt.Equals, true) - c.Assert(excludeAllButFooJSONMixedCasePattern.Match(filepath.FromSlash("/a/b/c/d/e/FOO.json"), false), qt.Equals, true) - - c.Assert(err, qt.IsNil) - - nopFilter, err := NewFilenameFilter(nil, nil) - c.Assert(err, qt.IsNil) - c.Assert(nopFilter.Match("ab.txt", false), qt.Equals, true) - - includeOnlyFilter, err := NewFilenameFilter([]string{"**.json", "**.jpg"}, nil) - c.Assert(err, qt.IsNil) - c.Assert(includeOnlyFilter.Match("ab.json", false), qt.Equals, true) - c.Assert(includeOnlyFilter.Match("ab.jpg", false), qt.Equals, true) - c.Assert(includeOnlyFilter.Match("ab.gif", false), qt.Equals, false) - - excludeOnlyFilter, err := NewFilenameFilter(nil, []string{"**.json", "**.jpg"}) - c.Assert(err, qt.IsNil) - c.Assert(excludeOnlyFilter.Match("ab.json", false), qt.Equals, false) - c.Assert(excludeOnlyFilter.Match("ab.jpg", false), qt.Equals, false) - c.Assert(excludeOnlyFilter.Match("ab.gif", false), qt.Equals, true) - - var nilFilter *FilenameFilter - c.Assert(nilFilter.Match("ab.gif", false), qt.Equals, true) - - funcFilter := NewFilenameFilterForInclusionFunc(func(s string) bool { return strings.HasSuffix(s, ".json") }) - c.Assert(funcFilter.Match("ab.json", false), qt.Equals, true) - c.Assert(funcFilter.Match("ab.bson", false), qt.Equals, false) -} diff --git a/hugofs/glob/glob.go b/hugofs/glob/glob.go deleted file mode 100644 index a4e5c49e8..000000000 --- a/hugofs/glob/glob.go +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright 2021 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package glob - -import ( - "os" - "path" - "path/filepath" - "runtime" - "strings" - "sync" - - "github.com/gobwas/glob" - "github.com/gobwas/glob/syntax" -) - -const filepathSeparator = string(os.PathSeparator) - -var ( - isWindows = runtime.GOOS == "windows" - defaultGlobCache = &globCache{ - isWindows: isWindows, - cache: make(map[string]globErr), - } -) - -type globErr struct { - glob glob.Glob - err error -} - -type globCache struct { - // Config - isWindows bool - - // Cache - sync.RWMutex - cache map[string]globErr -} - -func (gc *globCache) GetGlob(pattern string) (glob.Glob, error) { - var eg globErr - - gc.RLock() - var found bool - eg, found = gc.cache[pattern] - gc.RUnlock() - if found { - return eg.glob, eg.err - } - - var g glob.Glob - var err error - - pattern = filepath.ToSlash(pattern) - g, err = glob.Compile(strings.ToLower(pattern), '/') - - eg = globErr{ - globDecorator{ - g: g, - isWindows: gc.isWindows, - }, - err, - } - - gc.Lock() - gc.cache[pattern] = eg - gc.Unlock() - - return eg.glob, eg.err -} - -// Or creates a new Glob from the given globs. -func Or(globs ...glob.Glob) glob.Glob { - return globSlice{globs: globs} -} - -// MatchesFunc is a convenience type to create a glob.Glob from a function. -type MatchesFunc func(s string) bool - -func (m MatchesFunc) Match(s string) bool { - return m(s) -} - -type globSlice struct { - globs []glob.Glob -} - -func (g globSlice) Match(s string) bool { - for _, g := range g.globs { - if g.Match(s) { - return true - } - } - return false -} - -type globDecorator struct { - // On Windows we may get filenames with Windows slashes to match, - // which we need to normalize. - isWindows bool - - g glob.Glob -} - -func (g globDecorator) Match(s string) bool { - if g.isWindows { - s = filepath.ToSlash(s) - } - s = strings.ToLower(s) - return g.g.Match(s) -} - -func GetGlob(pattern string) (glob.Glob, error) { - return defaultGlobCache.GetGlob(pattern) -} - -func NormalizePath(p string) string { - return strings.ToLower(NormalizePathNoLower(p)) -} - -func NormalizePathNoLower(p string) string { - return strings.Trim(path.Clean(filepath.ToSlash(p)), "/.") -} - -// ResolveRootDir takes a normalized path on the form "assets/**.json" and -// determines any root dir, i.e. any start path without any wildcards. -func ResolveRootDir(p string) string { - parts := strings.Split(path.Dir(p), "/") - var roots []string - for _, part := range parts { - if HasGlobChar(part) { - break - } - roots = append(roots, part) - } - - if len(roots) == 0 { - return "" - } - - return strings.Join(roots, "/") -} - -// FilterGlobParts removes any string with glob wildcard. -func FilterGlobParts(a []string) []string { - b := a[:0] - for _, x := range a { - if !HasGlobChar(x) { - b = append(b, x) - } - } - return b -} - -// HasGlobChar returns whether s contains any glob wildcards. -func HasGlobChar(s string) bool { - for i := range len(s) { - if syntax.Special(s[i]) { - return true - } - } - return false -} diff --git a/hugofs/glob/glob_test.go b/hugofs/glob/glob_test.go deleted file mode 100644 index b63854e29..000000000 --- a/hugofs/glob/glob_test.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2021 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package glob - -import ( - "path/filepath" - "testing" - - qt "github.com/frankban/quicktest" -) - -func TestResolveRootDir(t *testing.T) { - c := qt.New(t) - - for _, test := range []struct { - input string - expected string - }{ - {"data/foo.json", "data"}, - {"a/b/**/foo.json", "a/b"}, - {"dat?a/foo.json", ""}, - {"a/b[a-c]/foo.json", "a"}, - } { - c.Assert(ResolveRootDir(test.input), qt.Equals, test.expected) - } -} - -func TestFilterGlobParts(t *testing.T) { - c := qt.New(t) - - for _, test := range []struct { - input []string - expected []string - }{ - {[]string{"a", "*", "c"}, []string{"a", "c"}}, - } { - c.Assert(FilterGlobParts(test.input), qt.DeepEquals, test.expected) - } -} - -func TestNormalizePath(t *testing.T) { - c := qt.New(t) - - for _, test := range []struct { - input string - expected string - }{ - {filepath.FromSlash("data/FOO.json"), "data/foo.json"}, - {filepath.FromSlash("/data/FOO.json"), "data/foo.json"}, - {filepath.FromSlash("./FOO.json"), "foo.json"}, - {"//", ""}, - } { - c.Assert(NormalizePath(test.input), qt.Equals, test.expected) - } -} - -func TestGetGlob(t *testing.T) { - for _, cache := range []*globCache{defaultGlobCache} { - c := qt.New(t) - g, err := cache.GetGlob("**.JSON") - c.Assert(err, qt.IsNil) - c.Assert(g.Match("data/my.jSon"), qt.Equals, true) - } -} - -func BenchmarkGetGlob(b *testing.B) { - runBench := func(name string, cache *globCache, search string) { - b.Run(name, func(b *testing.B) { - g, err := GetGlob("**/foo") - if err != nil { - b.Fatal(err) - } - for i := 0; i < b.N; i++ { - _ = g.Match(search) - } - }) - } - - runBench("Default cache", defaultGlobCache, "abcde") - runBench("Filenames cache, lowercase searches", defaultGlobCache, "abcde") - runBench("Filenames cache, mixed case searches", defaultGlobCache, "abCDe") - - b.Run("GetGlob", func(b *testing.B) { - for i := 0; i < b.N; i++ { - _, err := GetGlob("**/foo") - if err != nil { - b.Fatal(err) - } - } - }) -} diff --git a/hugofs/hglob/filename_filter.go b/hugofs/hglob/filename_filter.go new file mode 100644 index 000000000..86f53ea0a --- /dev/null +++ b/hugofs/hglob/filename_filter.go @@ -0,0 +1,208 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hglob + +import ( + "path" + "path/filepath" + "slices" + "strings" + + "github.com/gobwas/glob" +) + +type FilenameFilter struct { + shouldInclude func(filename string) bool + + entries []globFilenameFilterEntry + + isWindows bool + + nested []*FilenameFilter +} + +type globFilenameFilterEntryType int + +const ( + globFilenameFilterEntryTypeInclusion globFilenameFilterEntryType = iota + globFilenameFilterEntryTypeInclusionDir + globFilenameFilterEntryTypeExclusion +) + +// NegationPrefix is the prefix that makes a pattern an exclusion. +const NegationPrefix = "! " + +type globFilenameFilterEntry struct { + g glob.Glob + t globFilenameFilterEntryType +} + +func normalizeFilenameGlobPattern(s string) string { + // Use Unix separators even on Windows. + s = filepath.ToSlash(s) + if !strings.HasPrefix(s, "/") { + s = "/" + s + } + return s +} + +func NewFilenameFilterV2(patterns []string) (*FilenameFilter, error) { + if len(patterns) == 0 { + return nil, nil + } + filter := &FilenameFilter{isWindows: isWindows} + for _, p := range patterns { + var t globFilenameFilterEntryType + if strings.HasPrefix(p, NegationPrefix) { + t = globFilenameFilterEntryTypeExclusion + p = strings.TrimPrefix(p, NegationPrefix) + } else { + t = globFilenameFilterEntryTypeInclusion + } + p = normalizeFilenameGlobPattern(p) + g, err := GetGlob(p) + if err != nil { + return nil, err + } + filter.entries = append(filter.entries, globFilenameFilterEntry{t: t, g: g}) + if t == globFilenameFilterEntryTypeInclusion { + // For mounts that do directory walking (e.g. content) we + // must make sure that all directories up to this inclusion also + // gets included. + dir := path.Dir(p) + parts := strings.Split(dir, "/") + for i := range parts { + pattern := "/" + filepath.Join(parts[:i+1]...) + g, err := GetGlob(pattern) + if err != nil { + return nil, err + } + filter.entries = append(filter.entries, globFilenameFilterEntry{t: globFilenameFilterEntryTypeInclusionDir, g: g}) + } + } + + } + + return filter, nil +} + +// NewFilenameFilter creates a new Glob where the Match method will +// return true if the file should be included. +// Note that the exclusions will be checked first. +// Deprecated: Use NewFilenameFilterV2. +func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error) { + for i, p := range exclusions { + if !strings.HasPrefix(p, NegationPrefix) { + exclusions[i] = NegationPrefix + p + } + } + all := slices.Concat(inclusions, exclusions) + return NewFilenameFilterV2(all) +} + +// MustNewFilenameFilter invokes NewFilenameFilter and panics on error. +// Deprecated: Use NewFilenameFilterV2. +func MustNewFilenameFilter(inclusions, exclusions []string) *FilenameFilter { + filter, err := NewFilenameFilter(inclusions, exclusions) + if err != nil { + panic(err) + } + return filter +} + +// NewFilenameFilterForInclusionFunc create a new filter using the provided inclusion func. +func NewFilenameFilterForInclusionFunc(shouldInclude func(filename string) bool) *FilenameFilter { + return &FilenameFilter{shouldInclude: shouldInclude, isWindows: isWindows} +} + +// Match returns whether filename should be included. +func (f *FilenameFilter) Match(filename string, isDir bool) bool { + if f == nil { + return true + } + if !f.doMatch(filename, isDir) { + return false + } + + for _, nested := range f.nested { + if !nested.Match(filename, isDir) { + return false + } + } + + return true +} + +// Append appends a filter to the chain. The receiver will be copied if needed. +func (f *FilenameFilter) Append(other *FilenameFilter) *FilenameFilter { + if f == nil { + return other + } + + clone := *f + nested := make([]*FilenameFilter, len(clone.nested)+1) + copy(nested, clone.nested) + nested[len(nested)-1] = other + clone.nested = nested + + return &clone +} + +func (f *FilenameFilter) doMatch(filename string, isDir bool) bool { + if f == nil { + return true + } + + if !strings.HasPrefix(filename, filepathSeparator) { + filename = filepathSeparator + filename + } + + if f.shouldInclude != nil { + if f.shouldInclude(filename) { + return true + } + if f.isWindows { + // The Glob matchers below handles this by themselves, + // for the shouldInclude we need to take some extra steps + // to make this robust. + winFilename := filepath.FromSlash(filename) + if filename != winFilename { + if f.shouldInclude(winFilename) { + return true + } + } + } + } + + var hasInclude bool + for _, entry := range f.entries { + switch entry.t { + case globFilenameFilterEntryTypeExclusion: + if entry.g.Match(filename) { + return false + } + case globFilenameFilterEntryTypeInclusion: + if entry.g.Match(filename) { + return true + } + hasInclude = true + case globFilenameFilterEntryTypeInclusionDir: + if isDir && entry.g.Match(filename) { + return true + } + } + } + + return !hasInclude && f.shouldInclude == nil +} diff --git a/hugofs/hglob/filename_filter_test.go b/hugofs/hglob/filename_filter_test.go new file mode 100644 index 000000000..9d366234c --- /dev/null +++ b/hugofs/hglob/filename_filter_test.go @@ -0,0 +1,75 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hglob + +import ( + "path/filepath" + "strings" + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestFilenameFilter(t *testing.T) { + c := qt.New(t) + + excludeAlmostAllJSON, err := NewFilenameFilter([]string{"/a/b/c/foo.json"}, []string{"**.json"}) + c.Assert(err, qt.IsNil) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/data/my.json"), false), qt.Equals, false) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c/foo.json"), false), qt.Equals, true) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c/foo.bar"), false), qt.Equals, false) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c"), true), qt.Equals, true) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b"), true), qt.Equals, true) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a"), true), qt.Equals, true) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/"), true), qt.Equals, true) + c.Assert(excludeAlmostAllJSON.Match("", true), qt.Equals, true) + + excludeAllButFooJSON, err := NewFilenameFilter([]string{"/a/**/foo.json"}, []string{"**.json"}) + c.Assert(err, qt.IsNil) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/data/my.json"), false), qt.Equals, false) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/c/d/e/foo.json"), false), qt.Equals, true) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/c"), true), qt.Equals, true) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/"), true), qt.Equals, true) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/"), true), qt.Equals, true) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/b"), true), qt.Equals, false) + + excludeAllButFooJSONMixedCasePattern, err := NewFilenameFilter([]string{"/**/Foo.json"}, nil) + c.Assert(excludeAllButFooJSONMixedCasePattern.Match(filepath.FromSlash("/a/b/c/d/e/foo.json"), false), qt.Equals, true) + c.Assert(excludeAllButFooJSONMixedCasePattern.Match(filepath.FromSlash("/a/b/c/d/e/FOO.json"), false), qt.Equals, true) + + c.Assert(err, qt.IsNil) + + nopFilter, err := NewFilenameFilter(nil, nil) + c.Assert(err, qt.IsNil) + c.Assert(nopFilter.Match("ab.txt", false), qt.Equals, true) + + includeOnlyFilter, err := NewFilenameFilter([]string{"**.json", "**.jpg"}, nil) + c.Assert(err, qt.IsNil) + c.Assert(includeOnlyFilter.Match("ab.json", false), qt.Equals, true) + c.Assert(includeOnlyFilter.Match("ab.jpg", false), qt.Equals, true) + c.Assert(includeOnlyFilter.Match("ab.gif", false), qt.Equals, false) + + excludeOnlyFilter, err := NewFilenameFilter(nil, []string{"**.json", "**.jpg"}) + c.Assert(err, qt.IsNil) + c.Assert(excludeOnlyFilter.Match("ab.json", false), qt.Equals, false) + c.Assert(excludeOnlyFilter.Match("ab.jpg", false), qt.Equals, false) + c.Assert(excludeOnlyFilter.Match("ab.gif", false), qt.Equals, true) + + var nilFilter *FilenameFilter + c.Assert(nilFilter.Match("ab.gif", false), qt.Equals, true) + + funcFilter := NewFilenameFilterForInclusionFunc(func(s string) bool { return strings.HasSuffix(s, ".json") }) + c.Assert(funcFilter.Match("ab.json", false), qt.Equals, true) + c.Assert(funcFilter.Match("ab.bson", false), qt.Equals, false) +} diff --git a/hugofs/hglob/glob.go b/hugofs/hglob/glob.go new file mode 100644 index 000000000..15d3ba2cb --- /dev/null +++ b/hugofs/hglob/glob.go @@ -0,0 +1,195 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hglob + +import ( + "os" + "path" + "path/filepath" + "runtime" + "strings" + + "github.com/gobwas/glob" + "github.com/gobwas/glob/syntax" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/identity" +) + +const filepathSeparator = string(os.PathSeparator) + +var ( + isWindows = runtime.GOOS == "windows" + defaultGlobCache = &pathGlobCache{ + isWindows: isWindows, + cache: maps.NewCache[string, globErr](), + } + dotGlobCache = maps.NewCache[string, globErr]() +) + +type globErr struct { + glob glob.Glob + err error +} + +type pathGlobCache struct { + // Config + isWindows bool + + // Cache + cache *maps.Cache[string, globErr] +} + +// GetGlobDot returns a glob.Glob that matches the given pattern, using '.' as the path separator. +func GetGlobDot(pattern string) (glob.Glob, error) { + v, err := dotGlobCache.GetOrCreate(pattern, func() (globErr, error) { + g, err := glob.Compile(pattern, '.') + return globErr{ + glob: g, + err: err, + }, nil + }) + if err != nil { + return nil, err + } + + return v.glob, v.err +} + +func (gc *pathGlobCache) GetGlob(pattern string) (glob.Glob, error) { + v, err := gc.cache.GetOrCreate(pattern, func() (globErr, error) { + pattern = filepath.ToSlash(pattern) + g, err := glob.Compile(strings.ToLower(pattern), '/') + return globErr{ + glob: globDecorator{ + isWindows: gc.isWindows, + g: g, + }, + err: err, + }, nil + }) + if err != nil { + return nil, err + } + + return v.glob, v.err +} + +// Or creates a new Glob from the given globs. +func Or(globs ...glob.Glob) glob.Glob { + return globSlice{globs: globs} +} + +// MatchesFunc is a convenience type to create a glob.Glob from a function. +type MatchesFunc func(s string) bool + +func (m MatchesFunc) Match(s string) bool { + return m(s) +} + +type globSlice struct { + globs []glob.Glob +} + +func (g globSlice) Match(s string) bool { + for _, g := range g.globs { + if g.Match(s) { + return true + } + } + return false +} + +type globDecorator struct { + // On Windows we may get filenames with Windows slashes to match, + // which we need to normalize. + isWindows bool + + g glob.Glob +} + +func (g globDecorator) Match(s string) bool { + if g.isWindows { + s = filepath.ToSlash(s) + } + s = strings.ToLower(s) + return g.g.Match(s) +} + +func GetGlob(pattern string) (glob.Glob, error) { + return defaultGlobCache.GetGlob(pattern) +} + +func NormalizePath(p string) string { + return strings.ToLower(NormalizePathNoLower(p)) +} + +func NormalizePathNoLower(p string) string { + return strings.Trim(path.Clean(filepath.ToSlash(p)), "/.") +} + +// ResolveRootDir takes a normalized path on the form "assets/**.json" and +// determines any root dir, i.e. any start path without any wildcards. +func ResolveRootDir(p string) string { + parts := strings.Split(path.Dir(p), "/") + var roots []string + for _, part := range parts { + if HasGlobChar(part) { + break + } + roots = append(roots, part) + } + + if len(roots) == 0 { + return "" + } + + return strings.Join(roots, "/") +} + +// FilterGlobParts removes any string with glob wildcard. +func FilterGlobParts(a []string) []string { + b := a[:0] + for _, x := range a { + if !HasGlobChar(x) { + b = append(b, x) + } + } + return b +} + +// HasGlobChar returns whether s contains any glob wildcards. +func HasGlobChar(s string) bool { + for i := range len(s) { + if syntax.Special(s[i]) { + return true + } + } + return false +} + +// NewGlobIdentity creates a new Identity that +// is probably dependent on any other Identity +// that matches the given pattern. +func NewGlobIdentity(pattern string) identity.Identity { + glob, err := GetGlob(pattern) + if err != nil { + panic(err) + } + + predicate := func(other identity.Identity) bool { + return glob.Match(other.IdentifierBase()) + } + + return identity.NewPredicateIdentity(predicate, nil) +} diff --git a/hugofs/hglob/glob_test.go b/hugofs/hglob/glob_test.go new file mode 100644 index 000000000..e69e0e622 --- /dev/null +++ b/hugofs/hglob/glob_test.go @@ -0,0 +1,102 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hglob + +import ( + "path/filepath" + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestResolveRootDir(t *testing.T) { + c := qt.New(t) + + for _, test := range []struct { + input string + expected string + }{ + {"data/foo.json", "data"}, + {"a/b/**/foo.json", "a/b"}, + {"dat?a/foo.json", ""}, + {"a/b[a-c]/foo.json", "a"}, + } { + c.Assert(ResolveRootDir(test.input), qt.Equals, test.expected) + } +} + +func TestFilterGlobParts(t *testing.T) { + c := qt.New(t) + + for _, test := range []struct { + input []string + expected []string + }{ + {[]string{"a", "*", "c"}, []string{"a", "c"}}, + } { + c.Assert(FilterGlobParts(test.input), qt.DeepEquals, test.expected) + } +} + +func TestNormalizePath(t *testing.T) { + c := qt.New(t) + + for _, test := range []struct { + input string + expected string + }{ + {filepath.FromSlash("data/FOO.json"), "data/foo.json"}, + {filepath.FromSlash("/data/FOO.json"), "data/foo.json"}, + {filepath.FromSlash("./FOO.json"), "foo.json"}, + {"//", ""}, + } { + c.Assert(NormalizePath(test.input), qt.Equals, test.expected) + } +} + +func TestGetGlob(t *testing.T) { + for _, cache := range []*pathGlobCache{defaultGlobCache} { + c := qt.New(t) + g, err := cache.GetGlob("**.JSON") + c.Assert(err, qt.IsNil) + c.Assert(g.Match("data/my.jSon"), qt.Equals, true) + } +} + +func BenchmarkGetGlob(b *testing.B) { + runBench := func(name string, cache *pathGlobCache, search string) { + b.Run(name, func(b *testing.B) { + g, err := GetGlob("**/foo") + if err != nil { + b.Fatal(err) + } + for i := 0; i < b.N; i++ { + _ = g.Match(search) + } + }) + } + + runBench("Default cache", defaultGlobCache, "abcde") + runBench("Filenames cache, lowercase searches", defaultGlobCache, "abcde") + runBench("Filenames cache, mixed case searches", defaultGlobCache, "abCDe") + + b.Run("GetGlob", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := GetGlob("**/foo") + if err != nil { + b.Fatal(err) + } + } + }) +} diff --git a/hugofs/hugofs_integration_test.go b/hugofs/hugofs_integration_test.go index e3fcfdcb0..e9592a4e1 100644 --- a/hugofs/hugofs_integration_test.go +++ b/hugofs/hugofs_integration_test.go @@ -84,3 +84,37 @@ foo theme. b = runFiles(files) b.AssertFileContent("public/index.html", "Foo: foo theme.") } + +func TestMultipleMountsOfTheSameContentDirectoryIssue13818(t *testing.T) { + files := ` +-- hugo.toml -- + +[[module.mounts]] +source = 'shared/tutorials' +target = 'content/product1/tutorials' + +[[module.mounts]] +source = 'shared/tutorials' +target = 'content/product2/tutorials' + +-- shared/tutorials/tutorial1.md -- +--- +title: "Tutorial 1" +--- +-- shared/tutorials/tutorial2.md -- +--- +title: "Tutorial 2" +--- +-- layouts/all.html -- +Title: {{ .Title }}| + +` + b := hugolib.Test(t, files) + + b.AssertPublishDir(` +product1/tutorials/tutorial1/index.html +product1/tutorials/tutorial2/index.html +product2/tutorials/tutorial1/index.html +product2/tutorials/tutorial2/index.html +`) +} diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index 388493174..63b65dae3 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -28,7 +28,7 @@ import ( "github.com/bep/overlayfs" "github.com/gohugoio/hugo/hugofs/files" - "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/hugofs/hglob" radix "github.com/armon/go-radix" "github.com/spf13/afero" @@ -41,24 +41,24 @@ var _ ReverseLookupProvder = (*RootMappingFs)(nil) // NewRootMappingFs creates a new RootMappingFs on top of the provided with // root mappings with some optional metadata about the root. // Note that From represents a virtual root that maps to the actual filename in To. -func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { +func NewRootMappingFs(fs afero.Fs, rms ...*RootMapping) (*RootMappingFs, error) { rootMapToReal := radix.New() realMapToRoot := radix.New() id := fmt.Sprintf("rfs-%d", rootMappingFsCounter.Add(1)) - addMapping := func(key string, rm RootMapping, to *radix.Tree) { - var mappings []RootMapping + addMapping := func(key string, rm *RootMapping, to *radix.Tree) { + var mappings []*RootMapping v, found := to.Get(key) if found { // There may be more than one language pointing to the same root. - mappings = v.([]RootMapping) + mappings = v.([]*RootMapping) } mappings = append(mappings, rm) to.Insert(key, mappings) } for _, rm := range rms { - (&rm).clean() + rm.clean() rm.FromBase = files.ResolveComponentFolder(rm.From) @@ -118,7 +118,7 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { } nameToFilename := filepathSeparator + nameTo - rm.Meta.InclusionFilter = rm.Meta.InclusionFilter.Append(glob.NewFilenameFilterForInclusionFunc( + rm.Meta.InclusionFilter = rm.Meta.InclusionFilter.Append(hglob.NewFilenameFilterForInclusionFunc( func(filename string) bool { return nameToFilename == filename }, @@ -172,9 +172,9 @@ func newRootMappingFsFromFromTo( fs afero.Fs, fromTo ...string, ) (*RootMappingFs, error) { - rms := make([]RootMapping, len(fromTo)/2) + rms := make([]*RootMapping, len(fromTo)/2) for i, j := 0, 0; j < len(fromTo); i, j = i+1, j+2 { - rms[i] = RootMapping{ + rms[i] = &RootMapping{ From: fromTo[j], To: fromTo[j+1], ToBase: baseDir, @@ -202,7 +202,7 @@ type RootMapping struct { type keyRootMappings struct { key string - roots []RootMapping + roots []*RootMapping } func (rm *RootMapping) clean() { @@ -278,11 +278,11 @@ func (fs *RootMappingFs) UnwrapFilesystem() afero.Fs { } // Filter creates a copy of this filesystem with only mappings matching a filter. -func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs { +func (fs RootMappingFs) Filter(f func(m *RootMapping) bool) *RootMappingFs { rootMapToReal := radix.New() fs.rootMapToReal.Walk(func(b string, v any) bool { - rms := v.([]RootMapping) - var nrms []RootMapping + rms := v.([]*RootMapping) + var nrms []*RootMapping for _, rm := range rms { if f(rm) { nrms = append(nrms, rm) @@ -323,7 +323,6 @@ func (fs *RootMappingFs) Stat(name string) (os.FileInfo, error) { type ComponentPath struct { Component string Path string - Lang string Watch bool } @@ -377,7 +376,6 @@ func (fs *RootMappingFs) ReverseLookupComponent(component, filename string) ([]C cps = append(cps, ComponentPath{ Component: first.FromBase, Path: paths.ToSlashTrimLeading(filename), - Lang: first.Meta.Lang, Watch: first.Meta.Watch, }) } @@ -395,21 +393,21 @@ func (fs *RootMappingFs) hasPrefix(prefix string) bool { return hasPrefix } -func (fs *RootMappingFs) getRoot(key string) []RootMapping { +func (fs *RootMappingFs) getRoot(key string) []*RootMapping { v, found := fs.rootMapToReal.Get(key) if !found { return nil } - return v.([]RootMapping) + return v.([]*RootMapping) } -func (fs *RootMappingFs) getRoots(key string) (string, []RootMapping) { +func (fs *RootMappingFs) getRoots(key string) (string, []*RootMapping) { tree := fs.rootMapToReal levels := strings.Count(key, filepathSeparator) - seen := make(map[RootMapping]bool) + seen := make(map[*RootMapping]bool) - var roots []RootMapping + var roots []*RootMapping var s string for { @@ -420,7 +418,7 @@ func (fs *RootMappingFs) getRoots(key string) (string, []RootMapping) { break } - for _, rm := range vv.([]RootMapping) { + for _, rm := range vv.([]*RootMapping) { if !seen[rm] { seen[rm] = true roots = append(roots, rm) @@ -439,19 +437,19 @@ func (fs *RootMappingFs) getRoots(key string) (string, []RootMapping) { return s, roots } -func (fs *RootMappingFs) getRootsReverse(key string) (string, []RootMapping) { +func (fs *RootMappingFs) getRootsReverse(key string) (string, []*RootMapping) { tree := fs.realMapToRoot s, v, found := tree.LongestPrefix(key) if !found { return "", nil } - return s, v.([]RootMapping) + return s, v.([]*RootMapping) } -func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []RootMapping { - var roots []RootMapping +func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []*RootMapping { + var roots []*RootMapping fs.rootMapToReal.WalkPrefix(prefix, func(b string, v any) bool { - roots = append(roots, v.([]RootMapping)...) + roots = append(roots, v.([]*RootMapping)...) return false }) @@ -464,7 +462,7 @@ func (fs *RootMappingFs) getAncestors(prefix string) []keyRootMappings { if strings.HasPrefix(prefix, s+filepathSeparator) { roots = append(roots, keyRootMappings{ key: s, - roots: v.([]RootMapping), + roots: v.([]*RootMapping), }) } return false @@ -540,7 +538,7 @@ func (rfs *RootMappingFs) collectDirEntries(prefix string) ([]iofs.DirEntry, err seen := make(map[string]bool) // Prevent duplicate directories level := strings.Count(prefix, filepathSeparator) - collectDir := func(rm RootMapping, fi FileMetaInfo) error { + collectDir := func(rm *RootMapping, fi FileMetaInfo) error { f, err := fi.Meta().Open() if err != nil { return err @@ -618,7 +616,7 @@ func (rfs *RootMappingFs) collectDirEntries(prefix string) ([]iofs.DirEntry, err return false } - rms := v.([]RootMapping) + rms := v.([]*RootMapping) for _, rm := range rms { name := filepath.Base(rm.From) if seen[name] { @@ -720,7 +718,7 @@ func (fs *RootMappingFs) doDoStat(name string) ([]FileMetaInfo, error) { return []FileMetaInfo{newDirNameOnlyFileInfo(name, roots[0].Meta, fs.virtualDirOpener(name))}, nil } -func (fs *RootMappingFs) statRoot(root RootMapping, filename string) (FileMetaInfo, error) { +func (fs *RootMappingFs) statRoot(root *RootMapping, filename string) (FileMetaInfo, error) { dir, name := filepath.Split(filename) if root.Meta.Rename != nil { n, ok := root.Meta.Rename(name, false) diff --git a/hugofs/rootmapping_fs_test.go b/hugofs/rootmapping_fs_test.go index 83a95d648..59470d6d0 100644 --- a/hugofs/rootmapping_fs_test.go +++ b/hugofs/rootmapping_fs_test.go @@ -21,14 +21,30 @@ import ( iofs "io/fs" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/hugofs/hglob" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/htesting" "github.com/spf13/afero" ) +const ( + eni = iota + svi + noi + fri +) + +var ( + en = sitesMatrixForLangs(eni) + sv = sitesMatrixForLangs(svi) + no = sitesMatrixForLangs(noi) + fr = sitesMatrixForLangs(fri) +) + func TestLanguageRootMapping(t *testing.T) { c := qt.New(t) v := config.New() @@ -48,30 +64,30 @@ func TestLanguageRootMapping(t *testing.T) { c.Assert(afero.WriteFile(fs, filepath.Join("themes/b/myenblogcontent", "en-b-f.txt"), []byte("some en content"), 0o755), qt.IsNil) rfs, err := NewRootMappingFs(fs, - RootMapping{ + &RootMapping{ From: "content/blog", // Virtual path, first element is one of content, static, layouts etc. To: "themes/a/mysvblogcontent", // Real path - Meta: &FileMeta{Lang: "sv"}, + Meta: &FileMeta{SitesMatrix: sv}, }, - RootMapping{ + &RootMapping{ From: "content/blog", To: "themes/a/myenblogcontent", - Meta: &FileMeta{Lang: "en"}, + Meta: &FileMeta{SitesMatrix: en}, }, - RootMapping{ + &RootMapping{ From: "content/blog", To: "content/sv", - Meta: &FileMeta{Lang: "sv"}, + Meta: &FileMeta{SitesMatrix: sv}, }, - RootMapping{ + &RootMapping{ From: "content/blog", To: "themes/a/myotherenblogcontent", - Meta: &FileMeta{Lang: "en"}, + Meta: &FileMeta{SitesMatrix: en}, }, - RootMapping{ + &RootMapping{ From: "content/docs", To: "themes/a/mysvdocs", - Meta: &FileMeta{Lang: "sv"}, + Meta: &FileMeta{SitesMatrix: sv}, }, ) @@ -123,15 +139,14 @@ func TestLanguageRootMapping(t *testing.T) { return names } - - rfsEn := rfs.Filter(func(rm RootMapping) bool { - return rm.Meta.Lang == "en" + rfsEn := rfs.Filter(func(rm *RootMapping) bool { + return rm.Meta.SitesMatrix.HasLanguage(eni) }) c.Assert(getDirnames("content/blog", rfsEn), qt.DeepEquals, []string{"d1", "en-f.txt", "en-f2.txt"}) - rfsSv := rfs.Filter(func(rm RootMapping) bool { - return rm.Meta.Lang == "sv" + rfsSv := rfs.Filter(func(rm *RootMapping) bool { + return rm.Meta.SitesMatrix.HasLanguage(svi) }) c.Assert(getDirnames("content/blog", rfsSv), qt.DeepEquals, []string{"d1", "sv-f.txt", "svdir"}) @@ -207,35 +222,35 @@ func TestRootMappingFsMount(t *testing.T) { c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/singlefiles", "sv.txt"), []byte("sv text"), 0o755), qt.IsNil) bfs := NewBasePathFs(fs, "themes/a") - rm := []RootMapping{ + rm := []*RootMapping{ // Directories { From: "content/blog", To: "mynoblogcontent", - Meta: &FileMeta{Lang: "no"}, + Meta: &FileMeta{SitesMatrix: no}, }, { From: "content/blog", To: "myenblogcontent", - Meta: &FileMeta{Lang: "en"}, + Meta: &FileMeta{SitesMatrix: en}, }, { From: "content/blog", To: "mysvblogcontent", - Meta: &FileMeta{Lang: "sv"}, + Meta: &FileMeta{SitesMatrix: sv}, }, // Files { From: "content/singles/p1.md", To: "singlefiles/no.txt", ToBase: "singlefiles", - Meta: &FileMeta{Lang: "no"}, + Meta: &FileMeta{SitesMatrix: no}, }, { From: "content/singles/p1.md", To: "singlefiles/sv.txt", ToBase: "singlefiles", - Meta: &FileMeta{Lang: "sv"}, + Meta: &FileMeta{SitesMatrix: sv}, }, } @@ -246,7 +261,7 @@ func TestRootMappingFsMount(t *testing.T) { c.Assert(err, qt.IsNil) c.Assert(blog.IsDir(), qt.Equals, true) blogm := blog.(FileMetaInfo).Meta() - c.Assert(blogm.Lang, qt.Equals, "no") // First match + c.Assert(blogm.SitesMatrix.HasLanguage(noi), qt.IsTrue) // First match f, err := blogm.Open() c.Assert(err, qt.IsNil) @@ -268,32 +283,11 @@ func TestRootMappingFsMount(t *testing.T) { singles, err := singlesDir.(iofs.ReadDirFile).ReadDir(-1) c.Assert(err, qt.IsNil) c.Assert(singles, qt.HasLen, 2) - for i, lang := range []string{"no", "sv"} { + for i, langi := range []int{noi, svi} { fi := singles[i].(FileMetaInfo) - c.Assert(fi.Meta().Lang, qt.Equals, lang) + c.Assert(fi.Meta().SitesMatrix.HasLanguage(langi), qt.IsTrue) c.Assert(fi.Name(), qt.Equals, "p1.md") } - - // Test ReverseLookup. - // Single file mounts. - cps, err := rfs.ReverseLookup(filepath.FromSlash("singlefiles/no.txt")) - c.Assert(err, qt.IsNil) - c.Assert(cps, qt.DeepEquals, []ComponentPath{ - {Component: "content", Path: "singles/p1.md", Lang: "no"}, - }) - - cps, err = rfs.ReverseLookup(filepath.FromSlash("singlefiles/sv.txt")) - c.Assert(err, qt.IsNil) - c.Assert(cps, qt.DeepEquals, []ComponentPath{ - {Component: "content", Path: "singles/p1.md", Lang: "sv"}, - }) - - // File inside directory mount. - cps, err = rfs.ReverseLookup(filepath.FromSlash("mynoblogcontent/test.txt")) - c.Assert(err, qt.IsNil) - c.Assert(cps, qt.DeepEquals, []ComponentPath{ - {Component: "content", Path: "blog/test.txt", Lang: "no"}, - }) } func TestRootMappingFsMountOverlap(t *testing.T) { @@ -305,7 +299,7 @@ func TestRootMappingFsMountOverlap(t *testing.T) { c.Assert(afero.WriteFile(fs, filepath.FromSlash("dc/c.txt"), []byte("some no content"), 0o755), qt.IsNil) c.Assert(afero.WriteFile(fs, filepath.FromSlash("de/e.txt"), []byte("some no content"), 0o755), qt.IsNil) - rm := []RootMapping{ + rm := []*RootMapping{ { From: "static", To: "da", @@ -506,21 +500,21 @@ func TestRootMappingFileFilter(t *testing.T) { } } - rm := []RootMapping{ + rm := []*RootMapping{ { From: "content", To: "no", - Meta: &FileMeta{Lang: "no", InclusionFilter: glob.MustNewFilenameFilter(nil, []string{"**.txt"})}, + Meta: &FileMeta{SitesMatrix: no, InclusionFilter: hglob.MustNewFilenameFilter(nil, []string{"**.txt"})}, }, { From: "content", To: "en", - Meta: &FileMeta{Lang: "en"}, + Meta: &FileMeta{SitesMatrix: en}, }, { From: "content", To: "fr", - Meta: &FileMeta{Lang: "fr", InclusionFilter: glob.MustNewFilenameFilter(nil, []string{"**.txt"})}, + Meta: &FileMeta{SitesMatrix: fr, InclusionFilter: hglob.MustNewFilenameFilter(nil, []string{"**.txt"})}, }, } @@ -558,3 +552,9 @@ func TestRootMappingFileFilter(t *testing.T) { c.Assert(err, qt.IsNil) c.Assert(len(dirEntries), qt.Equals, 4) } + +var testDims = sitesmatrix.NewTestingDimensions([]string{"en", "no"}, []string{"v1", "v2", "v3"}, []string{"admin", "editor", "viewer", "guest"}) + +func sitesMatrixForLangs(langs ...int) *sitesmatrix.IntSets { + return sitesmatrix.NewIntSetsBuilder(testDims).WithSets(maps.NewOrderedIntSet(langs...), nil, nil).Build() +} diff --git a/hugofs/walk.go b/hugofs/walk.go index 4af46d89e..85026534a 100644 --- a/hugofs/walk.go +++ b/hugofs/walk.go @@ -14,8 +14,8 @@ package hugofs import ( + "context" "fmt" - "io/fs" "path/filepath" "sort" "strings" @@ -29,8 +29,8 @@ import ( ) type ( - WalkFunc func(path string, info FileMetaInfo) error - WalkHook func(dir FileMetaInfo, path string, readdir []FileMetaInfo) ([]FileMetaInfo, error) + WalkFunc func(ctx context.Context, path string, info FileMetaInfo) error + WalkHook func(ctx context.Context, dir FileMetaInfo, path string, readdir []FileMetaInfo) ([]FileMetaInfo, error) ) type Walkway struct { @@ -55,8 +55,10 @@ type WalkwayConfig struct { PathParser *paths.PathParser // One or both of these may be pre-set. - Info FileMetaInfo // The start info. - DirEntries []FileMetaInfo // The start info's dir entries. + Info FileMetaInfo // The start info. + DirEntries []FileMetaInfo // The start info's dir entries. + Ctx context.Context // Optional starting context. + IgnoreFile func(filename string) bool // Optional // Will be called in order. @@ -99,7 +101,12 @@ func (w *Walkway) Walk() error { return nil } - return w.walk(w.cfg.Root, w.cfg.Info, w.cfg.DirEntries) + ctx := w.cfg.Ctx + if ctx == nil { + ctx = context.Background() + } + + return w.walk(ctx, w.cfg.Root, w.cfg.Info, w.cfg.DirEntries) } // checkErr returns true if the error is handled. @@ -116,7 +123,7 @@ func (w *Walkway) checkErr(filename string, err error) bool { } // walk recursively descends path, calling walkFn. -func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo) error { +func (w *Walkway) walk(ctx context.Context, path string, info FileMetaInfo, dirEntries []FileMetaInfo) error { pathRel := strings.TrimPrefix(path, w.cfg.Root) if info == nil { @@ -134,7 +141,7 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo info = fi.(FileMetaInfo) } - err := w.cfg.WalkFn(path, info) + err := w.cfg.WalkFn(ctx, path, info) if err != nil { if info.IsDir() && err == filepath.SkipDir { return nil @@ -154,7 +161,8 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo } return fmt.Errorf("walk: open: path: %q filename: %q: %s", path, info.Meta().Filename, err) } - fis, err := f.(fs.ReadDirFile).ReadDir(-1) + fis, newCtx, err := ReadDirWithContext(ctx, f, -1) + ctx = newCtx f.Close() if err != nil { @@ -192,7 +200,7 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo if w.cfg.HookPre != nil { var err error - dirEntries, err = w.cfg.HookPre(info, path, dirEntries) + dirEntries, err = w.cfg.HookPre(ctx, info, path, dirEntries) if err != nil { if err == filepath.SkipDir { return nil @@ -203,7 +211,7 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo for _, fim := range dirEntries { nextPath := filepath.Join(path, fim.Name()) - err := w.walk(nextPath, fim, nil) + err := w.walk(ctx, nextPath, fim, nil) if err != nil { if !fim.IsDir() || err != filepath.SkipDir { return err @@ -213,7 +221,7 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo if w.cfg.HookPost != nil { var err error - dirEntries, err = w.cfg.HookPost(info, path, dirEntries) + dirEntries, err = w.cfg.HookPost(ctx, info, path, dirEntries) if err != nil { if err == filepath.SkipDir { return nil diff --git a/hugofs/walk_test.go b/hugofs/walk_test.go index 03b808533..d8dc0d9b0 100644 --- a/hugofs/walk_test.go +++ b/hugofs/walk_test.go @@ -55,7 +55,7 @@ func TestWalkRootMappingFs(t *testing.T) { c.Assert(afero.WriteFile(fs, filepath.Join("c/d", testfile), []byte("some content"), 0o755), qt.IsNil) c.Assert(afero.WriteFile(fs, filepath.Join("e/f", testfile), []byte("some content"), 0o755), qt.IsNil) - rm := []RootMapping{ + rm := []*RootMapping{ { From: "static/b", To: "e/f", @@ -116,7 +116,7 @@ func TestWalkRootMappingFs(t *testing.T) { func collectPaths(fs afero.Fs, root string) ([]string, error) { var names []string - walkFn := func(path string, info FileMetaInfo) error { + walkFn := func(ctx context.Context, path string, info FileMetaInfo) error { if info.IsDir() { return nil } @@ -135,7 +135,7 @@ func collectPaths(fs afero.Fs, root string) ([]string, error) { func collectFileinfos(fs afero.Fs, root string) ([]FileMetaInfo, error) { var fis []FileMetaInfo - walkFn := func(path string, info FileMetaInfo) error { + walkFn := func(ctx context.Context, path string, info FileMetaInfo) error { fis = append(fis, info) return nil @@ -169,7 +169,7 @@ func BenchmarkWalk(b *testing.B) { writeFiles("root/l1_2/l2_1", numFilesPerDir) writeFiles("root/l1_3", numFilesPerDir) - walkFn := func(path string, info FileMetaInfo) error { + walkFn := func(ctx context.Context, path string, info FileMetaInfo) error { if info.IsDir() { return nil } diff --git a/hugolib/404_test.go b/hugolib/404_test.go index 78cf0e70c..ed953339c 100644 --- a/hugolib/404_test.go +++ b/hugolib/404_test.go @@ -23,11 +23,14 @@ func Test404(t *testing.T) { files := ` -- hugo.toml -- +disableKinds = ["rss", "sitemap", "taxonomy", "term"] baseURL = "http://example.com/" +-- layouts/all.html -- +All. {{ .Kind }}. {{ .Title }}|Lastmod: {{ .Lastmod.Format "2006-01-02" }}| -- layouts/404.html -- {{ $home := site.Home }} 404: -Parent: {{ .Parent.Kind }} +Parent: {{ .Parent.Kind }}|{{ .Parent.Path }}| IsAncestor: {{ .IsAncestor $home }}/{{ $home.IsAncestor . }} IsDescendant: {{ .IsDescendant $home }}/{{ $home.IsDescendant . }} CurrentSection: {{ .CurrentSection.Kind }}| @@ -42,11 +45,11 @@ Data: {{ len .Data }}| IntegrationTestConfig{ T: t, TxtarString: files, - // LogLevel: logg.LevelTrace, - // Verbose: true, }, ).Build() + b.AssertFileContent("public/index.html", "All. home. |") + // Note: We currently have only 1 404 page. One might think that we should have // multiple, to follow the Custom Output scheme, but I don't see how that would work // right now. diff --git a/hugolib/alias.go b/hugolib/alias.go index 7b252c613..f5e7d3fad 100644 --- a/hugolib/alias.go +++ b/hugolib/alias.go @@ -25,6 +25,7 @@ import ( "strings" "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/publisher" "github.com/gohugoio/hugo/resources/page" @@ -47,7 +48,7 @@ type aliasPage struct { page.Page } -func (a aliasHandler) renderAlias(permalink string, p page.Page) (io.Reader, error) { +func (a aliasHandler) renderAlias(permalink string, p page.Page, matrix sitesmatrix.VectorProvider) (io.Reader, error) { var templateDesc tplimpl.TemplateDescriptor var base string = "" if ps, ok := p.(*pageState); ok { @@ -62,6 +63,7 @@ func (a aliasHandler) renderAlias(permalink string, p page.Page) (io.Reader, err Path: base, Category: tplimpl.CategoryLayout, Desc: templateDesc, + Sites: matrix, } t := a.ts.LookupPagesLayout(q) @@ -96,7 +98,7 @@ func (s *Site) publishDestAlias(allowRoot bool, path, permalink string, outputFo return err } - aliasContent, err := handler.renderAlias(permalink, p) + aliasContent, err := handler.renderAlias(permalink, p, s.siteVector) if err != nil { return err } diff --git a/hugolib/alias_test.go b/hugolib/alias_test.go index 1797411fd..10ea41048 100644 --- a/hugolib/alias_test.go +++ b/hugolib/alias_test.go @@ -40,7 +40,6 @@ For some moments the old man did not reply. He stood with bowed head, buried in const ( basicTemplate = "{{.Content}}" - aliasTemplate = "ALIASTEMPLATE" ) func TestAlias(t *testing.T) { diff --git a/hugolib/cascade_test.go b/hugolib/cascade_test.go index 1f9309764..e7a4f7917 100644 --- a/hugolib/cascade_test.go +++ b/hugolib/cascade_test.go @@ -17,10 +17,10 @@ import ( "bytes" "fmt" "path" - "strings" "testing" "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/parser" @@ -244,7 +244,7 @@ cascade: return b } - t.Run("Edit descendant", func(t *testing.T) { + /*t.Run("Edit descendant", func(t *testing.T) { t.Parallel() b := newSite(t, true) @@ -318,6 +318,7 @@ cascade: b.AssertFileContent("public/post/index.html", `Banner: |Layout: |Type: post|`) b.AssertFileContent("public/post/dir/p1/index.html", `Banner: |Layout: |`) }) + */ t.Run("Edit ancestor, content only", func(t *testing.T) { t.Parallel() @@ -721,101 +722,6 @@ subsect-content| b.AssertFileContent("public/mysect/subsect/index.html", "Title: |") } -// Issue 11977. -func TestCascadeExtensionInPath(t *testing.T) { - t.Parallel() - - files := ` --- hugo.toml -- -baseURL = "https://example.org" -[languages] -[languages.en] -weight = 1 -[languages.de] --- content/_index.de.md -- -+++ -[[cascade]] -[cascade.params] -foo = 'bar' -[cascade._target] -path = '/posts/post-1.de.md' -+++ --- content/posts/post-1.de.md -- ---- -title: "Post 1" ---- --- layouts/_default/single.html -- -{{ .Title }}|{{ .Params.foo }}$ -` - b, err := TestE(t, files) - b.Assert(err, qt.IsNotNil) - b.AssertLogContains(`cascade target path "/posts/post-1.de.md" looks like a path with an extension; since Hugo v0.123.0 this will not match anything, see https://gohugo.io/methods/page/path/`) -} - -func TestCascadeExtensionInPathIgnore(t *testing.T) { - t.Parallel() - - files := ` --- hugo.toml -- -baseURL = "https://example.org" -ignoreLogs = ['cascade-pattern-with-extension'] -[languages] -[languages.en] -weight = 1 -[languages.de] --- content/_index.de.md -- -+++ -[[cascade]] -[cascade.params] -foo = 'bar' -[cascade._target] -path = '/posts/post-1.de.md' -+++ --- content/posts/post-1.de.md -- ---- -title: "Post 1" ---- --- layouts/_default/single.html -- -{{ .Title }}|{{ .Params.foo }}$ -` - b := Test(t, files) - b.AssertLogContains(`! looks like a path with an extension`) -} - -func TestCascadConfigExtensionInPath(t *testing.T) { - t.Parallel() - - files := ` --- hugo.toml -- -baseURL = "https://example.org" -[[cascade]] -[cascade.params] -foo = 'bar' -[cascade._target] -path = '/p1.md' -` - b, err := TestE(t, files) - b.Assert(err, qt.IsNotNil) - b.AssertLogContains(`looks like a path with an extension`) -} - -func TestCascadConfigExtensionInPathIgnore(t *testing.T) { - t.Parallel() - - files := ` --- hugo.toml -- -baseURL = "https://example.org" -ignoreLogs = ['cascade-pattern-with-extension'] -[[cascade]] -[cascade.params] -foo = 'bar' -[cascade._target] -path = '/p1.md' -` - b := Test(t, files) - b.AssertLogContains(`! looks like a path with an extension`) -} - func TestCascadeIssue12172(t *testing.T) { t.Parallel() @@ -939,53 +845,99 @@ path = '/p1' b.AssertFileContent("public/p1/index.html", "p1|bar") // actual content is "p1|" } -func TestCascadeWarnOverrideIssue13806(t *testing.T) { - t.Parallel() - +func TestSitesMatrixCascadeConfig(t *testing.T) { files := ` -- hugo.toml -- -disableKinds = ['home','rss','section','sitemap','taxonomy','term'] -[[cascade]] -[cascade.params] -searchable = true -[cascade.target] -kind = 'page' --- content/something.md -- +disableKinds = ["taxonomy", "term", "rss", "sitemap"] +[languages] +[languages.en] +weight = 1 +[languages.nn] +weight = 2 +[languages.sv] +weight = 3 + +[versions] +[versions."v1.0.0"] +[versions."v2.0.0"] +[versions."v2.1.0"] + +[roles] +[roles.guest] +[roles.member] +[cascade] +[cascade.sites.matrix] +languages = ["en"] +versions = ["v2**"] +roles = ["member"] +[cascade.sites.complements] +languages = ["nn"] +versions = ["v1.0.*"] +roles = ["guest"] +-- content/_index.md -- --- -title: Something -params: - searchable: false +title: "Home" +sites: + matrix: + roles: ["guest"] --- -- layouts/all.html -- All. ` - b := Test(t, files, TestOptWarn()) + b := Test(t, files) - b.AssertLogContains("! WARN") + s0 := b.H.sitesVersionsRolesMap[sitesmatrix.Vector{0, 0, 0}] // en, v2.1.0, guest + b.Assert(s0.home, qt.IsNotNil) + b.Assert(s0.home.File(), qt.IsNotNil) + b.Assert(s0.language.Name(), qt.Equals, "en") + b.Assert(s0.version.Name(), qt.Equals, "v2.1.0") + b.Assert(s0.role.Name(), qt.Equals, "guest") + s0Pconfig := s0.Home().(*pageState).m.pageConfigSource + b.Assert(s0Pconfig.SitesMatrix.Vectors(), qt.DeepEquals, []sitesmatrix.Vector{{0, 0, 0}, {0, 1, 0}}) // en, v2.1.0, guest + en, v2.0.0, guest + + s1 := b.H.sitesVersionsRolesMap[sitesmatrix.Vector{1, 2, 0}] + b.Assert(s1.home, qt.IsNotNil) + b.Assert(s1.home.File(), qt.IsNil) + b.Assert(s1.language.Name(), qt.Equals, "nn") + b.Assert(s1.version.Name(), qt.Equals, "v1.0.0") + b.Assert(s1.role.Name(), qt.Equals, "guest") + s1Pconfig := s1.Home().(*pageState).m.pageConfigSource + b.Assert(s1Pconfig.SitesMatrix.HasVector(sitesmatrix.Vector{1, 2, 0}), qt.IsTrue) // nn, v1.0.0, guest + // Every site needs a home page. This matrix adds the missing ones, (3 * 3 * 2) - 2 = 16 + b.Assert(s1Pconfig.SitesMatrix.LenVectors(), qt.Equals, 16) } -func TestCascadeNilMapIssue13853(t *testing.T) { +func TestCascadeBundledPage(t *testing.T) { t.Parallel() files := ` -- hugo.toml -- --- content/test/_index.md -- +baseURL = "https://example.org" +-- content/_index.md -- --- -title: Test +title: Home cascade: -- build: - list: local - target: - path: '{/test/**}' -- params: - title: 'Test page' - target: - path: '{/test/**}' + params: + p1: v1 +--- +-- content/b1/index.md -- --- +title: b1 +--- +-- content/b1/p2.md -- +--- +title: p2 +--- +-- layouts/all.html -- +Title: {{ .Title }}|p1: {{ .Params.p1 }}| +{{ range .Resources }} +Resource: {{ .Name }}|p1: {{ .Params.p1 }}| +{{ end }} ` - // Just verify that it does not panic. - _ = Test(t, files) + b := Test(t, files) + + b.AssertFileContent("public/b1/index.html", "Title: b1|p1: v1|", "Resource: p2.md|p1: v1|") } diff --git a/hugolib/collections.go b/hugolib/collections.go index 898d2ba12..a7d209488 100644 --- a/hugolib/collections.go +++ b/hugolib/collections.go @@ -28,7 +28,7 @@ var ( // implementations have no value on their own. // Slice is for internal use. -func (p *pageState) Slice(items any) (any, error) { +func (ps *pageState) Slice(items any) (any, error) { return page.ToPages(items) } @@ -37,7 +37,7 @@ func (p *pageState) Slice(items any) (any, error) { // Group creates a PageGroup from a key and a Pages object // This method is not meant for external use. It got its non-typed arguments to satisfy // a very generic interface in the tpl package. -func (p *pageState) Group(key any, in any) (any, error) { +func (ps *pageState) Group(key any, in any) (any, error) { pages, err := page.ToPages(in) if err != nil { return nil, err diff --git a/hugolib/config_test.go b/hugolib/config_test.go index be5086f3e..5ee13ca30 100644 --- a/hugolib/config_test.go +++ b/hugolib/config_test.go @@ -93,20 +93,21 @@ myparam = "svParamValue" ` b := Test(t, files) - enSite := b.H.Sites[0] - svSite := b.H.Sites[1] - b.Assert(enSite.Title(), qt.Equals, "English Title") - b.Assert(enSite.Home().Title(), qt.Equals, "English Title") - b.Assert(enSite.Params()["myparam"], qt.Equals, "enParamValue") - b.Assert(enSite.Params()["p1"], qt.Equals, "p1en") - b.Assert(enSite.Params()["p2"], qt.Equals, "p2base") - b.Assert(svSite.Params()["p1"], qt.Equals, "p1base") - b.Assert(enSite.conf.StaticDir[0], qt.Equals, "mystatic") - - b.Assert(svSite.Title(), qt.Equals, "Svensk Title") - b.Assert(svSite.Home().Title(), qt.Equals, "Svensk Title") - b.Assert(svSite.Params()["myparam"], qt.Equals, "svParamValue") - b.Assert(svSite.conf.StaticDir[0], qt.Equals, "mysvstatic") + b.Assert(len(b.H.Sites), qt.Equals, 2) + enSiteH := b.SiteHelper("en", "", "") + svSiteH := b.SiteHelper("sv", "", "") + b.Assert(enSiteH.S.Title(), qt.Equals, "English Title") + b.Assert(enSiteH.S.Home().Title(), qt.Equals, "English Title") + b.Assert(enSiteH.S.Params()["myparam"], qt.Equals, "enParamValue") + b.Assert(enSiteH.S.Params()["p1"], qt.Equals, "p1en") + b.Assert(enSiteH.S.Params()["p2"], qt.Equals, "p2base") + b.Assert(svSiteH.S.Params()["p1"], qt.Equals, "p1base") + b.Assert(enSiteH.S.conf.StaticDir[0], qt.Equals, "mystatic") + + b.Assert(svSiteH.S.Title(), qt.Equals, "Svensk Title") + b.Assert(svSiteH.S.Home().Title(), qt.Equals, "Svensk Title") + b.Assert(svSiteH.S.Params()["myparam"], qt.Equals, "svParamValue") + b.Assert(svSiteH.S.conf.StaticDir[0], qt.Equals, "mysvstatic") }) t.Run("disable default language", func(t *testing.T) { @@ -132,7 +133,7 @@ weight = 2 ).BuildE() b.Assert(err, qt.IsNotNil) - b.Assert(err.Error(), qt.Contains, "cannot disable default content language") + b.Assert(err.Error(), qt.Contains, `default language "sv" is disabled`) }) t.Run("no internal config from outside", func(t *testing.T) { @@ -263,7 +264,7 @@ Home. ).Build() conf := b.H.Configs.Base - b.Assert(conf.DisableLanguages, qt.DeepEquals, []string{"sv", "no"}) + b.Assert(conf.DisableLanguages, qt.DeepEquals, []string{"no", "sv"}) b.Assert(conf.DisableKinds, qt.DeepEquals, []string{"taxonomy", "term"}) } } @@ -923,9 +924,8 @@ LanguageCode: {{ eq site.LanguageCode site.Language.LanguageCode }}|{{ site.Lang }, ).Build() - { - b.Assert(b.H.Log.LoggCount(logg.LevelWarn), qt.Equals, 1) - } + b.Assert(b.H.Log.LoggCount(logg.LevelWarn), qt.Equals, 1) + b.AssertFileContent("public/index.html", ` AllPages: 4| Sections: true| @@ -983,7 +983,7 @@ WorkingDir: myworkingdir| `) } -func TestConfigMergeLanguageDeepEmptyLefSide(t *testing.T) { +func TestConfigMergeLanguageDeepEmptyLeftSide(t *testing.T) { t.Parallel() files := ` @@ -1188,7 +1188,7 @@ Foo: {{ site.Params.foo }}| ).BuildE() b.Assert(err, qt.IsNotNil) - b.Assert(err.Error(), qt.Contains, "no languages") + b.Assert(err.Error(), qt.Contains, "invalid language configuration ") }) // Issue 11044 @@ -1215,7 +1215,7 @@ weight = 1 ).BuildE() b.Assert(err, qt.IsNotNil) - b.Assert(err.Error(), qt.Contains, "defaultContentLanguage does not match any language definition") + b.Assert(err.Error(), qt.Contains, `defaultContentLanguage "sv" not found in languages configuration`) }) } @@ -1235,7 +1235,9 @@ Home. b := Test(t, files) b.Assert(b.H.Configs.Base.Module.Mounts, qt.HasLen, 7) - b.Assert(b.H.Configs.LanguageConfigSlice[0].Module.Mounts, qt.HasLen, 7) + b.Assert(b.H.Configs.Base.Languages.Config.Sorted[0].Name, qt.Equals, "en") + firstLang := b.H.Configs.Base.Languages.Config.Sorted[0].Name + b.Assert(b.H.Configs.LanguageConfigMap[firstLang].Module.Mounts, qt.HasLen, 7) } func TestDefaultContentLanguageInSubdirOnlyOneLanguage(t *testing.T) { @@ -1556,14 +1558,21 @@ func TestDisableKindsIssue12144(t *testing.T) { files := ` -- hugo.toml -- disableKinds = ["page"] -defaultContentLanguage = "pt-br" +defaultContentLanguage = "pt" +[languages] +[languages.en] +weight = 1 +title = "English" +[languages.pt] +weight = 2 +title = "Portuguese" -- layouts/index.html -- Home. --- content/custom/index.pt-br.md -- +-- content/custom/index.br.md -- --- title: "P1 pt" --- --- content/custom/index.en-us.md -- +-- content/custom/index.en.md -- --- title: "P1 us" --- diff --git a/hugolib/content_map.go b/hugolib/content_map.go index 596a8f7f9..d97f3e3a1 100644 --- a/hugolib/content_map.go +++ b/hugolib/content_map.go @@ -16,6 +16,7 @@ package hugolib import ( "context" "fmt" + "iter" "path" "path/filepath" "strings" @@ -26,8 +27,8 @@ import ( "github.com/gohugoio/hugo/common/paths" "github.com/gohugoio/hugo/hugofs/files" "github.com/gohugoio/hugo/hugolib/pagesfromdata" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "github.com/gohugoio/hugo/identity" - "github.com/gohugoio/hugo/source" "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/page/pagemeta" @@ -52,25 +53,64 @@ type contentMapConfig struct { isRebuild bool } -var _ contentNodeI = (*resourceSource)(nil) +type resourceSourceState int + +const ( + resourceStateNew resourceSourceState = iota + resourceStateAssigned +) type resourceSource struct { - langIndex int - path *paths.Path - opener hugio.OpenReadSeekCloser - fi hugofs.FileMetaInfo - rc *pagemeta.ResourceConfig + state resourceSourceState + sv sitesmatrix.Vector + path *paths.Path + opener hugio.OpenReadSeekCloser + fi hugofs.FileMetaInfo + rc *pagemeta.ResourceConfig r resource.Resource } +func (r *resourceSource) assignSiteVector(vec sitesmatrix.Vector) *resourceSource { + if r.state == resourceStateAssigned { + panic("cannot assign site vector to a resourceSource that is already assigned") + } + r.sv = vec + r.state = resourceStateAssigned + return r +} + func (r resourceSource) clone() *resourceSource { + r.state = resourceStateNew r.r = nil return &r } -func (r *resourceSource) LangIndex() int { - return r.langIndex +func (r *resourceSource) forEeachContentNode(f func(v sitesmatrix.Vector, n contentNode) bool) bool { + return f(r.sv, r) +} + +func (r *resourceSource) nodeCategorySingle() { + // Marker method. +} + +func (r *resourceSource) String() string { + var sb strings.Builder + if r.fi != nil { + sb.WriteString("filename: " + r.fi.Meta().Filename) + sb.WriteString(fmt.Sprintf(" matrix: %v", r.fi.Meta().SitesMatrix)) + sb.WriteString(fmt.Sprintf(" complements: %v", r.fi.Meta().SitesComplements)) + } + if r.rc != nil { + sb.WriteString(fmt.Sprintf("rc matrix: %v", r.rc.SitesMatrix)) + sb.WriteString(fmt.Sprintf("rc complements: %v", r.rc.SitesComplements)) + } + sb.WriteString(fmt.Sprintf(" sv: %v", r.sv)) + return sb.String() +} + +func (r *resourceSource) siteVector() sitesmatrix.Vector { + return r.sv } func (r *resourceSource) MarkStale() { @@ -78,16 +118,11 @@ func (r *resourceSource) MarkStale() { } func (r *resourceSource) resetBuildState() { - if rr, ok := r.r.(buildStateReseter); ok { + if rr, ok := r.r.(contentNodeBuildStateResetter); ok { rr.resetBuildState() } } -func (r *resourceSource) isPage() bool { - _, ok := r.r.(page.Page) - return ok -} - func (r *resourceSource) GetIdentity() identity.Identity { if r.r != nil { return r.r.(identity.IdentityProvider).GetIdentity() @@ -95,131 +130,113 @@ func (r *resourceSource) GetIdentity() identity.Identity { return r.path } -func (r *resourceSource) ForEeachIdentity(f func(identity.Identity) bool) bool { - return f(r.GetIdentity()) -} - -func (r *resourceSource) Path() string { - return r.path.Path() -} - -func (r *resourceSource) isContentNodeBranch() bool { - return false +func (p *resourceSource) nodeSourceEntryID() any { + if p.rc != nil { + return p.rc.ContentAdapterSourceEntryHash + } + if p.fi != nil { + return p.fi.Meta().Filename + } + return p.path } -var _ contentNodeI = (*resourceSources)(nil) - -type resourceSources []*resourceSource - -func (n resourceSources) MarkStale() { - for _, r := range n { - if r != nil { - r.MarkStale() +func (p *resourceSource) lookupContentNode(v sitesmatrix.Vector) contentNode { + if p.state >= resourceStateAssigned { + if p.sv == v { + return p } + return nil } -} -func (n resourceSources) Path() string { - panic("not supported") -} + // A site has not been assigned yet. -func (n resourceSources) isContentNodeBranch() bool { - return false -} + if p.rc != nil && p.rc.MatchSiteVector(v) { + return p + } -func (n resourceSources) resetBuildState() { - for _, r := range n { - if r != nil { - r.resetBuildState() - } + if p.rc != nil && p.rc.SitesMatrix.LenVectors() > 0 { + // Do not consider file mount matrix if the resource config has its own. + return nil } -} -func (n resourceSources) GetIdentity() identity.Identity { - for _, r := range n { - if r != nil { - return r.GetIdentity() - } + if p.fi != nil && p.fi.Meta().SitesMatrix.HasVector(v) { + return p } + return nil } -func (n resourceSources) ForEeachIdentity(f func(identity.Identity) bool) bool { - for _, r := range n { - if r != nil { - if f(r.GetIdentity()) { - return true - } +func (p *resourceSource) lookupContentNodes(siteVector sitesmatrix.Vector, fallback bool) iter.Seq[contentNodeForSite] { + if siteVector == p.sv { + return func(yield func(n contentNodeForSite) bool) { + yield(p) } } - return false -} -func (cfg contentMapConfig) getTaxonomyConfig(s string) (v viewName) { - for _, n := range cfg.taxonomyConfig.views { - if strings.HasPrefix(s, n.pluralTreeKey) { - return n + pc := p.rc + + var found bool + if !fallback { + if pc != nil && pc.MatchSiteVector(siteVector) { + found = true + } else { + return nil } } - return -} -func (m *pageMap) insertPageWithLock(s string, p *pageState) (contentNodeI, contentNodeI, bool) { - u, n, replaced := m.treePages.InsertIntoValuesDimensionWithLock(s, p) - - if replaced && !m.s.h.isRebuild() && m.s.conf.PrintPathWarnings { - var messageDetail string - if p1, ok := n.(*pageState); ok && p1.File() != nil { - messageDetail = fmt.Sprintf(" file: %q", p1.File().Filename()) + if !found && pc != nil { + if !pc.MatchLanguageCoarse(siteVector) { + return nil + } + if !pc.MatchVersionCoarse(siteVector) { + return nil } - if p2, ok := u.(*pageState); ok && p2.File() != nil { - messageDetail += fmt.Sprintf(" file: %q", p2.File().Filename()) + if !pc.MatchRoleCoarse(siteVector) { + return nil } + } - m.s.Log.Warnf("Duplicate content path: %q%s", s, messageDetail) + if !found && !fallback { + return nil } - return u, n, replaced + return func(yield func(n contentNodeForSite) bool) { + if !yield(p) { + return + } + } } -func (m *pageMap) insertResourceWithLock(s string, r contentNodeI) (contentNodeI, contentNodeI, bool) { - u, n, replaced := m.treeResources.InsertIntoValuesDimensionWithLock(s, r) - if replaced { - m.handleDuplicateResourcePath(s, r, n) - } - return u, n, replaced +func (r *resourceSource) Path() string { + return r.path.Base() } -func (m *pageMap) insertResource(s string, r contentNodeI) (contentNodeI, contentNodeI, bool) { - u, n, replaced := m.treeResources.InsertIntoValuesDimension(s, r) - if replaced { - m.handleDuplicateResourcePath(s, r, n) - } - return u, n, replaced +func (r *resourceSource) PathInfo() *paths.Path { + return r.path } -func (m *pageMap) handleDuplicateResourcePath(s string, updated, existing contentNodeI) { - if m.s.h.isRebuild() || !m.s.conf.PrintPathWarnings { - return - } - var messageDetail string - if r1, ok := existing.(*resourceSource); ok && r1.fi != nil { - messageDetail = fmt.Sprintf(" file: %q", r1.fi.Meta().Filename) - } - if r2, ok := updated.(*resourceSource); ok && r2.fi != nil { - messageDetail += fmt.Sprintf(" file: %q", r2.fi.Meta().Filename) +func (cfg contentMapConfig) getTaxonomyConfig(s string) (v viewName) { + for _, n := range cfg.taxonomyConfig.views { + if strings.HasPrefix(s, n.pluralTreeKey) { + return n + } } - - m.s.Log.Warnf("Duplicate resource path: %q%s", s, messageDetail) + return } -func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, buildConfig *BuildCfg) (pageCount uint64, resourceCount uint64, addErr error) { +func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, buildConfig *BuildCfg) (pageSourceCount uint64, resourceSourceCount uint64, addErr error) { if fi.IsDir() { return } + if m == nil { + panic("nil pageMap") + } + + h := m.s.h + insertResource := func(fim hugofs.FileMetaInfo) error { - resourceCount++ + resourceSourceCount++ pi := fi.Meta().PathInfo key := pi.Base() tree := m.treeResources @@ -227,37 +244,22 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, buildConfig *BuildCfg) (pageCoun commit := tree.Lock(true) defer commit() - r := func() (hugio.ReadSeekCloser, error) { - return fim.Meta().Open() - } - - var rs *resourceSource if pi.IsContent() { - // Create the page now as we need it at assembly time. - // The other resources are created if needed. - pageResource, pi, err := m.s.h.newPage( - &pageMeta{ - f: source.NewFileInfo(fim), - pathInfo: pi, - bundled: true, - }, - ) + pm, err := h.newPageMetaSourceFromFile(fi) if err != nil { - return err - } - if pageResource == nil { - // Disabled page. - return nil + return fmt.Errorf("failed to create page from file %q: %w", fi.Meta().Filename, err) } - key = pi.Base() - - rs = &resourceSource{r: pageResource, langIndex: pageResource.s.languagei} + pm.bundled = true + m.treeResources.Insert(key, pm) } else { - rs = &resourceSource{path: pi, opener: r, fi: fim, langIndex: fim.Meta().LangIndex} + r := func() (hugio.ReadSeekCloser, error) { + return fim.Meta().Open() + } + // Create one dimension now, the rest later on demand. + rs := &resourceSource{path: pi, opener: r, fi: fim, sv: fim.Meta().SitesMatrix.VectorSample()} + m.treeResources.Insert(key, rs) } - _, _, _ = m.insertResource(key, rs) - return nil } @@ -277,8 +279,8 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, buildConfig *BuildCfg) (pageCoun } case paths.TypeContentData: pc, rc, err := m.addPagesFromGoTmplFi(fi, buildConfig) - pageCount += pc - resourceCount += rc + pageSourceCount += pc + resourceSourceCount += rc if err != nil { addErr = err return @@ -291,27 +293,15 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, buildConfig *BuildCfg) (pageCoun }, )) - pageCount++ + pageSourceCount++ - // A content file. - p, pi, err := m.s.h.newPage( - &pageMeta{ - f: source.NewFileInfo(fi), - pathInfo: pi, - bundled: false, - }, - ) + pm, err := h.newPageMetaSourceFromFile(fi) if err != nil { - addErr = err - return - } - if p == nil { - // Disabled page. + addErr = fmt.Errorf("failed to create page meta from file %q: %w", fi.Meta().Filename, err) return } - m.insertPageWithLock(pi.Base(), p) - + m.treePages.InsertWithLock(pm.pathInfo.Base(), pm) } return } @@ -331,8 +321,9 @@ func (m *pageMap) addPagesFromGoTmplFi(fi hugofs.FileMetaInfo, buildConfig *Buil return } - s := m.s.h.resolveSite(fi.Meta().Lang) - f := source.NewFileInfo(fi) + sitesMatrix := fi.Meta().SitesMatrix + + s := m.s.h.resolveFirstSite(sitesMatrix) h := s.h contentAdapter := s.pageMap.treePagesFromTemplateAdapters.Get(pi.Base()) @@ -352,23 +343,16 @@ func (m *pageMap) addPagesFromGoTmplFi(fi hugofs.FileMetaInfo, buildConfig *Buil TemplateStore: ss.GetTemplateStore(), } }, - DependencyManager: s.Conf.NewIdentityManager("pagesfromdata"), + DependencyManager: s.Conf.NewIdentityManager(), Watching: s.Conf.Watching(), - HandlePage: func(pt *pagesfromdata.PagesFromTemplate, pc *pagemeta.PageConfig) error { + HandlePage: func(pt *pagesfromdata.PagesFromTemplate, pe *pagemeta.PageConfigEarly) error { s := pt.Site.(*Site) - if err := pc.CompileForPagesFromDataPre(pt.GoTmplFi.Meta().PathInfo.Base(), m.s.Log, s.conf.MediaTypes.Config); err != nil { + + if err := pe.CompileForPagesFromDataPre(pt.GoTmplFi.Meta().PathInfo.Base(), m.s.Log, s.conf.MediaTypes.Config); err != nil { return err } - ps, pi, err := h.newPage( - &pageMeta{ - f: f, - s: s, - pageMetaParams: &pageMetaParams{ - pageConfig: pc, - }, - }, - ) + ps, err := s.h.newPageMetaSourceForContentAdapter(fi, s.siteVector, pe) if err != nil { return err } @@ -378,13 +362,13 @@ func (m *pageMap) addPagesFromGoTmplFi(fi hugofs.FileMetaInfo, buildConfig *Buil return nil } - u, n, replaced := s.pageMap.insertPageWithLock(pi.Base(), ps) + u, n, replaced := s.pageMap.treePages.InsertWithLock(ps.pathInfo.Base(), ps) if h.isRebuild() { if replaced { - pt.AddChange(n.GetIdentity()) + pt.AddChange(cnh.GetIdentity(n)) } else { - pt.AddChange(u.GetIdentity()) + pt.AddChange(cnh.GetIdentity(u)) // New content not in use anywhere. // To make sure that these gets listed in any site.RegularPages ranges or similar // we could invalidate everything, but first try to collect a sample set @@ -409,18 +393,20 @@ func (m *pageMap) addPagesFromGoTmplFi(fi hugofs.FileMetaInfo, buildConfig *Buil s := pt.Site.(*Site) if err := rc.Compile( pt.GoTmplFi.Meta().PathInfo.Base(), - s.Conf.PathParser(), + pt.GoTmplFi, + s.Conf, s.conf.MediaTypes.Config, ); err != nil { return err } - rs := &resourceSource{path: rc.PathInfo, rc: rc, opener: nil, fi: pt.GoTmplFi, langIndex: s.languagei} + // Create one dimension now, the rest later on demand. + rs := &resourceSource{path: rc.PathInfo, rc: rc, opener: nil, fi: pt.GoTmplFi, sv: s.siteVector} - _, n, replaced := s.pageMap.insertResourceWithLock(rc.PathInfo.Base(), rs) + _, n, updated := s.pageMap.treeResources.InsertWithLock(rs.path.Base(), rs) - if h.isRebuild() && replaced { - pt.AddChange(n.GetIdentity()) + if h.isRebuild() && updated { + pt.AddChange(cnh.GetIdentity(n)) } return nil }, @@ -444,18 +430,28 @@ func (m *pageMap) addPagesFromGoTmplFi(fi hugofs.FileMetaInfo, buildConfig *Buil } handleBuildInfo(s, bi) - if !rebuild && bi.EnableAllLanguages { + if !rebuild && (bi.EnableAllLanguages || bi.EnableAllDimensions) { // Clone and insert the adapter for the other sites. - for _, ss := range s.h.Sites { - if s == ss { - continue + var iter iter.Seq[*Site] + if bi.EnableAllLanguages { + include := func(ss *Site) bool { + return s.siteVector.Language() != ss.siteVector.Language() + } + iter = h.allSiteLanguages(include) + } else { + include := func(ss *Site) bool { + return s.siteVector != ss.siteVector } + iter = h.allSites(include) + } + for ss := range iter { clone := contentAdapter.CloneForSite(ss) // Make sure it gets executed for the first time. bi, err := clone.Execute(context.Background()) if err != nil { + addErr = err return } diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index aafd905b4..7162325c3 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Hugo Authors. All rights reserved. +// Copyright 2025 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,7 +22,6 @@ import ( "strconv" "strings" "sync/atomic" - "time" "github.com/bep/logg" "github.com/gohugoio/hugo/cache/dynacache" @@ -30,58 +29,55 @@ import ( "github.com/gohugoio/hugo/common/paths" "github.com/gohugoio/hugo/common/predicate" "github.com/gohugoio/hugo/common/rungroup" - "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/hugofs/files" - "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/hugofs/hglob" "github.com/gohugoio/hugo/hugolib/doctree" "github.com/gohugoio/hugo/hugolib/pagesfromdata" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "github.com/gohugoio/hugo/identity" - "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/resources" - "github.com/spf13/cast" "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/resources/kinds" "github.com/gohugoio/hugo/resources/page" - "github.com/gohugoio/hugo/resources/page/pagemeta" "github.com/gohugoio/hugo/resources/resource" ) var pagePredicates = struct { - KindPage predicate.P[*pageState] - KindSection predicate.P[*pageState] - KindHome predicate.P[*pageState] - KindTerm predicate.P[*pageState] - ShouldListLocal predicate.P[*pageState] - ShouldListGlobal predicate.P[*pageState] - ShouldListAny predicate.P[*pageState] - ShouldLink predicate.P[page.Page] + KindPage predicate.PR[*pageState] + KindSection predicate.PR[*pageState] + KindHome predicate.PR[*pageState] + KindTerm predicate.PR[*pageState] + ShouldListLocal predicate.PR[*pageState] + ShouldListGlobal predicate.PR[*pageState] + ShouldListAny predicate.PR[*pageState] + ShouldLink predicate.PR[page.Page] }{ - KindPage: func(p *pageState) bool { - return p.Kind() == kinds.KindPage + KindPage: func(p *pageState) predicate.Match { + return predicate.BoolMatch(p.Kind() == kinds.KindPage) }, - KindSection: func(p *pageState) bool { - return p.Kind() == kinds.KindSection + KindSection: func(p *pageState) predicate.Match { + return predicate.BoolMatch(p.Kind() == kinds.KindSection) }, - KindHome: func(p *pageState) bool { - return p.Kind() == kinds.KindHome + KindHome: func(p *pageState) predicate.Match { + return predicate.BoolMatch(p.Kind() == kinds.KindHome) }, - KindTerm: func(p *pageState) bool { - return p.Kind() == kinds.KindTerm + KindTerm: func(p *pageState) predicate.Match { + return predicate.BoolMatch(p.Kind() == kinds.KindTerm) }, - ShouldListLocal: func(p *pageState) bool { - return p.m.shouldList(false) + ShouldListLocal: func(p *pageState) predicate.Match { + return predicate.BoolMatch(p.m.shouldList(false)) }, - ShouldListGlobal: func(p *pageState) bool { - return p.m.shouldList(true) + ShouldListGlobal: func(p *pageState) predicate.Match { + return predicate.BoolMatch(p.m.shouldList(true)) }, - ShouldListAny: func(p *pageState) bool { - return p.m.shouldListAny() + ShouldListAny: func(p *pageState) predicate.Match { + return predicate.BoolMatch(p.m.shouldListAny()) }, - ShouldLink: func(p page.Page) bool { - return !p.(*pageState).m.noLink() + ShouldLink: func(p page.Page) predicate.Match { + return predicate.BoolMatch(!p.(*pageState).m.noLink()) }, } @@ -121,20 +117,20 @@ type pageTrees struct { // Note that all of these trees share the same key structure, // so you can take a leaf Page key and do a prefix search // with key + "/" to get all of its resources. - treePages *doctree.NodeShiftTree[contentNodeI] + treePages *doctree.NodeShiftTree[contentNode] // This tree contains Resources bundled in pages. - treeResources *doctree.NodeShiftTree[contentNodeI] + treeResources *doctree.NodeShiftTree[contentNode] // All pages and resources. - treePagesResources doctree.WalkableTrees[contentNodeI] + treePagesResources doctree.WalkableTrees[contentNode] // This tree contains all taxonomy entries, e.g "/tags/blue/page1" - treeTaxonomyEntries *doctree.TreeShiftTree[*weightedContentNode] + treeTaxonomyEntries *doctree.TreeShiftTreeSlice[*weightedContentNode] // Stores the state for _content.gotmpl files. // Mostly releveant for rebuilds. - treePagesFromTemplateAdapters *doctree.TreeShiftTree[*pagesfromdata.PagesFromTemplate] + treePagesFromTemplateAdapters *doctree.TreeShiftTreeSlice[*pagesfromdata.PagesFromTemplate] // A slice of the resource trees. resourceTrees doctree.MutableTrees @@ -149,16 +145,17 @@ func (t *pageTrees) collectAndMarkStaleIdentities(p *paths.Path) []identity.Iden var ids []identity.Identity // We need only one identity sample per dimension. nCount := 0 - cb := func(n contentNodeI) bool { + cb := func(n contentNode) bool { if n == nil { return false } - n.MarkStale() + cnh.markStale(n) if nCount > 0 { return true } nCount++ - n.ForEeachIdentity(func(id identity.Identity) bool { + + cnh.toForEachIdentityProvider(n).ForEeachIdentity(func(id identity.Identity) bool { ids = append(ids, id) return false }) @@ -167,24 +164,18 @@ func (t *pageTrees) collectAndMarkStaleIdentities(p *paths.Path) []identity.Iden } tree := t.treePages nCount = 0 - tree.ForEeachInDimension(key, doctree.DimensionLanguage.Index(), - cb, - ) + tree.ForEeachInAllDimensions(key, cb) tree = t.treeResources nCount = 0 - tree.ForEeachInDimension(key, doctree.DimensionLanguage.Index(), - cb, - ) + tree.ForEeachInAllDimensions(key, cb) if p.Component() == files.ComponentFolderContent { // It may also be a bundled content resource. key := p.ForType(paths.TypeContentResource).Base() tree = t.treeResources nCount = 0 - tree.ForEeachInDimension(key, doctree.DimensionLanguage.Index(), - cb, - ) + tree.ForEeachInAllDimensions(key, cb) } return ids @@ -197,18 +188,18 @@ func (t *pageTrees) collectIdentitiesSurrounding(key string, maxSamplesPerTree i return ids } -func (t *pageTrees) collectIdentitiesSurroundingIn(key string, maxSamples int, tree *doctree.NodeShiftTree[contentNodeI]) []identity.Identity { +func (t *pageTrees) collectIdentitiesSurroundingIn(key string, maxSamples int, tree *doctree.NodeShiftTree[contentNode]) []identity.Identity { var ids []identity.Identity - section, ok := tree.LongestPrefixAll(path.Dir(key)) + section, ok := tree.LongestPrefixRaw(path.Dir(key)) if ok { count := 0 prefix := section + "/" level := strings.Count(prefix, "/") - tree.WalkPrefixRaw(prefix, func(s string, n contentNodeI) bool { + tree.WalkPrefixRaw(prefix, func(s string, n contentNode) bool { if level != strings.Count(s, "/") { return false } - n.ForEeachIdentity(func(id identity.Identity) bool { + cnh.toForEachIdentityProvider(n).ForEeachIdentity(func(id identity.Identity) bool { ids = append(ids, id) return false }) @@ -231,19 +222,18 @@ func (t *pageTrees) DeletePageAndResourcesBelow(ss ...string) { } } -// Shape shapes all trees in t to the given dimension. -func (t pageTrees) Shape(d, v int) *pageTrees { - t.treePages = t.treePages.Shape(d, v) - t.treeResources = t.treeResources.Shape(d, v) - t.treeTaxonomyEntries = t.treeTaxonomyEntries.Shape(d, v) - t.treePagesFromTemplateAdapters = t.treePagesFromTemplateAdapters.Shape(d, v) +func (t pageTrees) Shape(v sitesmatrix.Vector) *pageTrees { + t.treePages = t.treePages.Shape(v) + t.treeResources = t.treeResources.Shape(v) + t.treeTaxonomyEntries = t.treeTaxonomyEntries.Shape(v) + t.treePagesFromTemplateAdapters = t.treePagesFromTemplateAdapters.Shape(v) t.createMutableTrees() return &t } func (t *pageTrees) createMutableTrees() { - t.treePagesResources = doctree.WalkableTrees[contentNodeI]{ + t.treePagesResources = doctree.WalkableTrees[contentNode]{ t.treePages, t.treeResources, } @@ -288,17 +278,17 @@ func (q pageMapQueryPagesBelowPath) Key() string { // Apply fn to all pages in m matching the given predicate. // fn may return true to stop the walk. -func (m *pageMap) forEachPage(include predicate.P[*pageState], fn func(p *pageState) (bool, error)) error { +func (m *pageMap) forEachPage(include predicate.PR[*pageState], fn func(p *pageState) (bool, error)) error { if include == nil { - include = func(p *pageState) bool { - return true + include = func(p *pageState) predicate.Match { + return predicate.True } } - w := &doctree.NodeShiftTreeWalker[contentNodeI]{ + w := &doctree.NodeShiftTreeWalker[contentNode]{ Tree: m.treePages, LockType: doctree.LockTypeRead, - Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { - if p, ok := n.(*pageState); ok && include(p) { + Handle: func(key string, n contentNode) (bool, error) { + if p, ok := n.(*pageState); ok && include(p).OK() { if terminate, err := fn(p); terminate || err != nil { return terminate, err } @@ -310,10 +300,10 @@ func (m *pageMap) forEachPage(include predicate.P[*pageState], fn func(p *pageSt return w.Walk(context.Background()) } -func (m *pageMap) forEeachPageIncludingBundledPages(include predicate.P[*pageState], fn func(p *pageState) (bool, error)) error { +func (m *pageMap) forEeachPageIncludingBundledPages(include predicate.PR[*pageState], fn func(p *pageState) (bool, error)) error { if include == nil { - include = func(p *pageState) bool { - return true + include = func(p *pageState) predicate.Match { + return predicate.True } } @@ -321,15 +311,13 @@ func (m *pageMap) forEeachPageIncludingBundledPages(include predicate.P[*pageSta return err } - w := &doctree.NodeShiftTreeWalker[contentNodeI]{ + w := &doctree.NodeShiftTreeWalker[contentNode]{ Tree: m.treeResources, LockType: doctree.LockTypeRead, - Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { - if rs, ok := n.(*resourceSource); ok { - if p, ok := rs.r.(*pageState); ok && include(p) { - if terminate, err := fn(p); terminate || err != nil { - return terminate, err - } + Handle: func(key string, n contentNode) (bool, error) { + if p, ok := n.(*pageState); ok && include(p).OK() { + if terminate, err := fn(p); terminate || err != nil { + return terminate, err } } return false, nil @@ -362,15 +350,16 @@ func (m *pageMap) getPagesInSection(q pageMapQueryPagesInSection) page.Pages { include := q.Include if include == nil { - include = pagePredicates.ShouldListLocal + include = pagePredicates.ShouldListLocal.BoolFunc() } - w := &doctree.NodeShiftTreeWalker[contentNodeI]{ - Tree: m.treePages, - Prefix: prefix, + w := &doctree.NodeShiftTreeWalker[contentNode]{ + Tree: m.treePages, + Prefix: prefix, + Fallback: true, } - w.Handle = func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { + w.Handle = func(key string, n contentNode) (bool, error) { if q.Recursive { if p, ok := n.(*pageState); ok && include(p) { pas = append(pas, p) @@ -382,7 +371,7 @@ func (m *pageMap) getPagesInSection(q pageMapQueryPagesInSection) page.Pages { pas = append(pas, p) } - if n.isContentNodeBranch() { + if cnh.isBranchNode(n) { currentBranch := key + "/" if otherBranch == "" || otherBranch != currentBranch { w.SkipPrefix(currentBranch) @@ -421,7 +410,7 @@ func (m *pageMap) getPagesWithTerm(q pageMapQueryPagesBelowPath) page.Pages { var pas page.Pages include := q.Include if include == nil { - include = pagePredicates.ShouldListLocal + include = pagePredicates.ShouldListLocal.BoolFunc() } err := m.treeTaxonomyEntries.WalkPrefix( @@ -485,65 +474,98 @@ func (m *pageMap) getTermsForPageInTaxonomy(path, taxonomy string) page.Pages { func (m *pageMap) forEachResourceInPage( ps *pageState, lockType doctree.LockType, - exact bool, - handle func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error), + fallback bool, + transform func(resourceKey string, n contentNode) (n2 contentNode, ns doctree.NodeTransformState, err error), + handle func(resourceKey string, n contentNode) (bool, error), ) error { keyPage := ps.Path() if keyPage == "/" { keyPage = "" } + prefix := paths.AddTrailingSlash(ps.Path()) + isBranch := ps.IsNode() - rw := &doctree.NodeShiftTreeWalker[contentNodeI]{ + rwr := &doctree.NodeShiftTreeWalker[contentNode]{ Tree: m.treeResources, Prefix: prefix, LockType: lockType, - Exact: exact, + Fallback: fallback, } - rw.Handle = func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { - if isBranch { - // A resourceKey always represents a filename with extension. - // A page key points to the logical path of a page, which when sourced from the filesystem - // may represent a directory (bundles) or a single content file (e.g. p1.md). - // So, to avoid any overlapping ambiguity, we start looking from the owning directory. - s := resourceKey - - for { - s = path.Dir(s) - ownerKey, found := m.treePages.LongestPrefixAll(s) - if !found { - return true, nil - } - if ownerKey == keyPage { - break - } + shouldSkipOrTerminate := func(resourceKey string) (ns doctree.NodeTransformState) { + if !isBranch { + return + } - if s != ownerKey && strings.HasPrefix(s, ownerKey) { - // Keep looking - continue - } + // A resourceKey always represents a filename with extension. + // A page key points to the logical path of a page, which when sourced from the filesystem + // may represent a directory (bundles) or a single content file (e.g. p1.md). + // So, to avoid any overlapping ambiguity, we start looking from the owning directory. + s := resourceKey - // Stop walking downwards, someone else owns this resource. - rw.SkipPrefix(ownerKey + "/") - return false, nil + for { + s = path.Dir(s) + ownerKey, found := m.treePages.LongestPrefixRaw(s) + + if !found { + return doctree.NodeTransformStateTerminate + } + if ownerKey == keyPage { + break + } + + if s != ownerKey && strings.HasPrefix(s, ownerKey) { + // Keep looking + continue + } + + // Stop walking downwards, someone else owns this resource. + rwr.SkipPrefix(ownerKey + "/") + return doctree.NodeTransformStateSkip + } + return + } + + if transform != nil { + rwr.Transform = func(resourceKey string, n contentNode) (n2 contentNode, ns doctree.NodeTransformState, err error) { + if ns = shouldSkipOrTerminate(resourceKey); ns >= doctree.NodeTransformStateSkip { + return } + return transform(resourceKey, n) + } + } + rwr.Handle = func(resourceKey string, n contentNode) (terminate bool, err error) { + if transform == nil { + if ns := shouldSkipOrTerminate(resourceKey); ns >= doctree.NodeTransformStateSkip { + return ns == doctree.NodeTransformStateTerminate, nil + } } - return handle(resourceKey, n, match) + return handle(resourceKey, n) } - return rw.Walk(context.Background()) + return rwr.Walk(context.Background()) } func (m *pageMap) getResourcesForPage(ps *pageState) (resource.Resources, error) { var res resource.Resources - m.forEachResourceInPage(ps, doctree.LockTypeNone, false, func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { - rs := n.(*resourceSource) - if rs.r != nil { - res = append(res, rs.r) + m.forEachResourceInPage(ps, doctree.LockTypeNone, true, nil, func(resourceKey string, n contentNode) (bool, error) { + switch n := n.(type) { + case *resourceSource: + r := n.r + if r == nil { + panic(fmt.Sprintf("getResourcesForPage: resource %q for page %q has no resource, sites matrix %v/%v", resourceKey, ps.Path(), ps.siteVector(), n.sv)) + } + + res = append(res, r) + case *pageState: + res = append(res, n) + default: + panic(fmt.Sprintf("getResourcesForPage: unknown type %T", n)) } + return false, nil }) return res, nil @@ -555,6 +577,7 @@ func (m *pageMap) getOrCreateResourcesForPage(ps *pageState) resource.Resources keyPage = "" } key := keyPage + "/get-resources-for-page" + v, err := m.cacheResources.GetOrCreate(key, func(string) (resource.Resources, error) { res, err := m.getResourcesForPage(ps) if err != nil { @@ -633,297 +656,48 @@ func (m *pageMap) getOrCreateResourcesForPage(ps *pageState) resource.Resources return v } -type weightedContentNode struct { - n contentNodeI - weight int - term *pageWithOrdinal -} - -type buildStateReseter interface { - resetBuildState() -} - -type contentNodeI interface { - identity.IdentityProvider - identity.ForEeachIdentityProvider - Path() string - isContentNodeBranch() bool - buildStateReseter - resource.StaleMarker -} - -var _ contentNodeI = (*contentNodeIs)(nil) - -type contentNodeIs []contentNodeI - -func (n contentNodeIs) Path() string { - return n[0].Path() -} - -func (n contentNodeIs) isContentNodeBranch() bool { - return n[0].isContentNodeBranch() -} +var _ doctree.Transformer[contentNode] = (*contentNodeTransformerRaw)(nil) -func (n contentNodeIs) GetIdentity() identity.Identity { - return n[0].GetIdentity() -} +type contentNodeTransformerRaw struct{} -func (n contentNodeIs) ForEeachIdentity(f func(identity.Identity) bool) bool { - for _, nn := range n { - if nn != nil { - if nn.ForEeachIdentity(f) { - return true - } +func (t *contentNodeTransformerRaw) Append(n contentNode, ns ...contentNode) (contentNode, bool) { + if n == nil { + if len(ns) == 1 { + return ns[0], true } + var ss contentNodes = ns + return ss, true } - return false -} -func (n contentNodeIs) resetBuildState() { - for _, nn := range n { - if nn != nil { - nn.resetBuildState() - } - } -} - -func (n contentNodeIs) MarkStale() { - for _, nn := range n { - resource.MarkStale(nn) - } -} - -type contentNodeShifter struct { - numLanguages int -} - -func (s *contentNodeShifter) Delete(n contentNodeI, dimension doctree.Dimension) (contentNodeI, bool, bool) { - lidx := dimension[0] - switch v := n.(type) { - case contentNodeIs: - deleted := v[lidx] - resource.MarkStale(deleted) - wasDeleted := deleted != nil - v[lidx] = nil - isEmpty := true - for _, vv := range v { - if vv != nil { - isEmpty = false - break - } - } - return deleted, wasDeleted, isEmpty - case resourceSources: - deleted := v[lidx] - resource.MarkStale(deleted) - wasDeleted := deleted != nil - v[lidx] = nil - isEmpty := true - for _, vv := range v { - if vv != nil { - isEmpty = false - break - } - } - return deleted, wasDeleted, isEmpty - case *resourceSource: - if lidx != v.LangIndex() { - return nil, false, false - } - resource.MarkStale(v) - return v, true, true - case *pageState: - if lidx != v.s.languagei { - return nil, false, false - } - resource.MarkStale(v) - return v, true, true - default: - panic(fmt.Sprintf("unknown type %T", n)) - } -} - -func (s *contentNodeShifter) Shift(n contentNodeI, dimension doctree.Dimension, exact bool) (contentNodeI, bool, doctree.DimensionFlag) { - lidx := dimension[0] - // How accurate is the match. - accuracy := doctree.DimensionLanguage switch v := n.(type) { - case contentNodeIs: - if len(v) == 0 { - panic("empty contentNodeIs") - } - vv := v[lidx] - if vv != nil { - return vv, true, accuracy - } - return nil, false, 0 - case resourceSources: - vv := v[lidx] - if vv != nil { - return vv, true, doctree.DimensionLanguage - } - if exact { - return nil, false, 0 - } - // For non content resources, pick the first match. - for _, vv := range v { - if vv != nil { - if vv.isPage() { - return nil, false, 0 - } - return vv, true, 0 - } - } - case *resourceSource: - if v.LangIndex() == lidx { - return v, true, doctree.DimensionLanguage - } - if !v.isPage() && !exact { - return v, true, 0 - } - case *pageState: - if v.s.languagei == lidx { - return n, true, doctree.DimensionLanguage - } - default: - panic(fmt.Sprintf("unknown type %T", n)) - } - return nil, false, 0 -} - -func (s *contentNodeShifter) ForEeachInDimension(n contentNodeI, d int, f func(contentNodeI) bool) { - if d != doctree.DimensionLanguage.Index() { - panic("only language dimension supported") - } - - switch vv := n.(type) { - case contentNodeIs: - for _, v := range vv { - if v != nil { - if f(v) { - return - } - } - } - default: - f(vv) - } -} - -func (s *contentNodeShifter) InsertInto(old, new contentNodeI, dimension doctree.Dimension) (contentNodeI, contentNodeI, bool) { - langi := dimension[doctree.DimensionLanguage.Index()] - switch vv := old.(type) { - case *pageState: - newp, ok := new.(*pageState) - if !ok { - panic(fmt.Sprintf("unknown type %T", new)) - } - if vv.s.languagei == newp.s.languagei && newp.s.languagei == langi { - return new, vv, true - } - is := make(contentNodeIs, s.numLanguages) - is[vv.s.languagei] = old - is[langi] = new - return is, old, false - case contentNodeIs: - oldv := vv[langi] - vv[langi] = new - return vv, oldv, oldv != nil - case resourceSources: - oldv := vv[langi] - vv[langi] = new.(*resourceSource) - return vv, oldv, oldv != nil - case *resourceSource: - newp, ok := new.(*resourceSource) - if !ok { - panic(fmt.Sprintf("unknown type %T", new)) - } - if vv.LangIndex() == newp.LangIndex() && newp.LangIndex() == langi { - return new, vv, true - } - rs := make(resourceSources, s.numLanguages) - rs[vv.LangIndex()] = vv - rs[langi] = newp - return rs, vv, false - - default: - panic(fmt.Sprintf("unknown type %T", old)) - } -} - -func (s *contentNodeShifter) Insert(old, new contentNodeI) (contentNodeI, contentNodeI, bool) { - switch vv := old.(type) { - case *pageState: - newp, ok := new.(*pageState) - if !ok { - panic(fmt.Sprintf("unknown type %T", new)) - } - if vv.s.languagei == newp.s.languagei { - if newp != old { - resource.MarkStale(old) - } - return new, vv, true - } - is := make(contentNodeIs, s.numLanguages) - is[newp.s.languagei] = new - is[vv.s.languagei] = old - return is, old, false - case contentNodeIs: - newp, ok := new.(*pageState) - if !ok { - panic(fmt.Sprintf("unknown type %T", new)) - } - oldp := vv[newp.s.languagei] - if oldp != newp { - resource.MarkStale(oldp) - } - vv[newp.s.languagei] = new - return vv, oldp, oldp != nil - case *resourceSource: - newp, ok := new.(*resourceSource) - if !ok { - panic(fmt.Sprintf("unknown type %T", new)) - } - if vv.LangIndex() == newp.LangIndex() { - if vv != newp { - resource.MarkStale(vv) - } - return new, vv, true - } - rs := make(resourceSources, s.numLanguages) - rs[newp.LangIndex()] = newp - rs[vv.LangIndex()] = vv - return rs, vv, false - case resourceSources: - newp, ok := new.(*resourceSource) - if !ok { - panic(fmt.Sprintf("unknown type %T", new)) - } - oldp := vv[newp.LangIndex()] - if oldp != newp { - resource.MarkStale(oldp) - } - vv[newp.LangIndex()] = newp - return vv, oldp, oldp != nil + case contentNodes: + v = append(v, ns...) + return v, true default: - panic(fmt.Sprintf("unknown type %T", old)) + ss := make(contentNodes, 0, 1+len(ns)) + ss = append(ss, v) + ss = append(ss, ns...) + return ss, true } } -func newPageMap(i int, s *Site, mcache *dynacache.Cache, pageTrees *pageTrees) *pageMap { +func newPageMap(s *Site, mcache *dynacache.Cache, pageTrees *pageTrees) *pageMap { var m *pageMap + vec := s.siteVector + languageVersionRole := fmt.Sprintf("s%d/%d&%d", vec.Language(), vec.Version(), vec.Role()) + var taxonomiesConfig taxonomiesConfig = s.conf.Taxonomies m = &pageMap{ - pageTrees: pageTrees.Shape(0, i), - cachePages1: dynacache.GetOrCreatePartition[string, page.Pages](mcache, fmt.Sprintf("/pag1/%d", i), dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}), - cachePages2: dynacache.GetOrCreatePartition[string, page.Pages](mcache, fmt.Sprintf("/pag2/%d", i), dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}), - cacheGetTerms: dynacache.GetOrCreatePartition[string, map[string]page.Pages](mcache, fmt.Sprintf("/gett/%d", i), dynacache.OptionsPartition{Weight: 5, ClearWhen: dynacache.ClearOnRebuild}), - cacheResources: dynacache.GetOrCreatePartition[string, resource.Resources](mcache, fmt.Sprintf("/ress/%d", i), dynacache.OptionsPartition{Weight: 60, ClearWhen: dynacache.ClearOnRebuild}), - cacheContentRendered: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentSummary]](mcache, fmt.Sprintf("/cont/ren/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}), - cacheContentPlain: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentPlainPlainWords]](mcache, fmt.Sprintf("/cont/pla/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}), - contentTableOfContents: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentTableOfContents]](mcache, fmt.Sprintf("/cont/toc/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}), + pageTrees: pageTrees.Shape(vec), + cachePages1: dynacache.GetOrCreatePartition[string, page.Pages](mcache, fmt.Sprintf("/pag1/%s", languageVersionRole), dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}), + cachePages2: dynacache.GetOrCreatePartition[string, page.Pages](mcache, fmt.Sprintf("/pag2/%s", languageVersionRole), dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}), + cacheGetTerms: dynacache.GetOrCreatePartition[string, map[string]page.Pages](mcache, fmt.Sprintf("/gett/%s", languageVersionRole), dynacache.OptionsPartition{Weight: 5, ClearWhen: dynacache.ClearOnRebuild}), + cacheResources: dynacache.GetOrCreatePartition[string, resource.Resources](mcache, fmt.Sprintf("/ress/%s", languageVersionRole), dynacache.OptionsPartition{Weight: 60, ClearWhen: dynacache.ClearOnRebuild}), + cacheContentRendered: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentSummary]](mcache, fmt.Sprintf("/cont/ren/%s", languageVersionRole), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}), + cacheContentPlain: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentPlainPlainWords]](mcache, fmt.Sprintf("/cont/pla/%s", languageVersionRole), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}), + contentTableOfContents: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentTableOfContents]](mcache, fmt.Sprintf("/cont/toc/%s", languageVersionRole), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}), contentDataFileSeenItems: maps.NewCache[string, map[uint64]bool](), @@ -934,12 +708,12 @@ func newPageMap(i int, s *Site, mcache *dynacache.Cache, pageTrees *pageTrees) * taxonomyTermDisabled: !s.conf.IsKindEnabled(kinds.KindTerm), pageDisabled: !s.conf.IsKindEnabled(kinds.KindPage), }, - i: i, + i: s.siteVector.Language(), s: s, } - m.pageReverseIndex = newContentTreeTreverseIndex(func(get func(key any) (contentNodeI, bool), set func(key any, val contentNodeI)) { - add := func(k string, n contentNodeI) { + m.pageReverseIndex = newContentTreeTreverseIndex(func(get func(key any) (contentNode, bool), set func(key any, val contentNode)) { + add := func(k string, n contentNode) { existing, found := get(k) if found && existing != ambiguousContentNode { set(k, ambiguousContentNode) @@ -948,10 +722,10 @@ func newPageMap(i int, s *Site, mcache *dynacache.Cache, pageTrees *pageTrees) * } } - w := &doctree.NodeShiftTreeWalker[contentNodeI]{ + w := &doctree.NodeShiftTreeWalker[contentNode]{ Tree: m.treePages, LockType: doctree.LockTypeRead, - Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { + Handle: func(s string, n contentNode) (bool, error) { p := n.(*pageState) if p.PathInfo() != nil { add(p.PathInfo().BaseNameNoIdentifier(), p) @@ -968,61 +742,52 @@ func newPageMap(i int, s *Site, mcache *dynacache.Cache, pageTrees *pageTrees) * return m } -func newContentTreeTreverseIndex(init func(get func(key any) (contentNodeI, bool), set func(key any, val contentNodeI))) *contentTreeReverseIndex { +func newContentTreeTreverseIndex(init func(get func(key any) (contentNode, bool), set func(key any, val contentNode))) *contentTreeReverseIndex { return &contentTreeReverseIndex{ initFn: init, - mm: maps.NewCache[any, contentNodeI](), + mm: maps.NewCache[any, contentNode](), } } type contentTreeReverseIndex struct { - initFn func(get func(key any) (contentNodeI, bool), set func(key any, val contentNodeI)) - mm *maps.Cache[any, contentNodeI] + initFn func(get func(key any) (contentNode, bool), set func(key any, val contentNode)) + mm *maps.Cache[any, contentNode] } func (c *contentTreeReverseIndex) Reset() { c.mm.Reset() } -func (c *contentTreeReverseIndex) Get(key any) contentNodeI { - v, _ := c.mm.InitAndGet(key, func(get func(key any) (contentNodeI, bool), set func(key any, val contentNodeI)) error { +func (c *contentTreeReverseIndex) Get(key any) contentNode { + v, _ := c.mm.InitAndGet(key, func(get func(key any) (contentNode, bool), set func(key any, val contentNode)) error { c.initFn(get, set) return nil }) return v } -type sitePagesAssembler struct { - *Site - assembleChanges *WhatChanged - ctx context.Context -} - func (m *pageMap) debugPrint(prefix string, maxLevel int, w io.Writer) { noshift := false - var prevKey string - pageWalker := &doctree.NodeShiftTreeWalker[contentNodeI]{ + pageWalker := &doctree.NodeShiftTreeWalker[contentNode]{ NoShift: noshift, Tree: m.treePages, Prefix: prefix, - WalkContext: &doctree.WalkContext[contentNodeI]{}, + WalkContext: &doctree.WalkContext[contentNode]{}, } resourceWalker := pageWalker.Extend() resourceWalker.Tree = m.treeResources - pageWalker.Handle = func(keyPage string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { + pageWalker.Handle = func(keyPage string, n contentNode) (bool, error) { level := strings.Count(keyPage, "/") if level > maxLevel { return false, nil } const indentStr = " " p := n.(*pageState) - s := strings.TrimPrefix(keyPage, paths.CommonDirPath(prevKey, keyPage)) - lenIndent := len(keyPage) - len(s) - fmt.Fprint(w, strings.Repeat(indentStr, lenIndent)) - info := fmt.Sprintf("%s lm: %s (%s)", s, p.Lastmod().Format("2006-01-02"), p.Kind()) + lenIndent := 0 + info := fmt.Sprintf("%s lm: %s (%s)", keyPage, p.Lastmod().Format("2006-01-02"), p.Kind()) fmt.Fprintln(w, info) switch p.Kind() { case kinds.KindTerm: @@ -1037,13 +802,12 @@ func (m *pageMap) debugPrint(prefix string, maxLevel int, w io.Writer) { ) } - isBranch := n.isContentNodeBranch() - prevKey = keyPage + isBranch := cnh.isBranchNode(n) resourceWalker.Prefix = keyPage + "/" - resourceWalker.Handle = func(ss string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { + resourceWalker.Handle = func(ss string, n contentNode) (bool, error) { if isBranch { - ownerKey, _ := pageWalker.Tree.LongestPrefix(ss, true, nil) + ownerKey, _ := pageWalker.Tree.LongestPrefix(ss, false, nil) if ownerKey != keyPage { // Stop walking downwards, someone else owns this resource. pageWalker.SkipPrefix(ownerKey + "/") @@ -1075,7 +839,7 @@ func (h *HugoSites) dynacacheGCFilenameIfNotWatchedAndDrainMatching(filename str if cps.Watch { continue } - np := glob.NormalizePath(path.Join(cps.Component, cps.Path)) + np := hglob.NormalizePath(path.Join(cps.Component, cps.Path)) g, err := h.ResourceSpec.BuildConfig().MatchCacheBuster(h.Log, np) if err == nil && g != nil { cacheBusters = append(cacheBusters, g) @@ -1316,7 +1080,7 @@ func (h *HugoSites) resolveAndResetDependententPageOutputs(ctx context.Context, resetCounter.Add(1) h.Log.Trace(logg.StringFunc(func() string { p := po.p - return fmt.Sprintf("Resetting page output %s for %s for output %s\n", p.Kind(), p.Path(), po.f.Name) + return fmt.Sprintf("%s Resetting page output %q for %q for output %q\n", p.s.resolveDimensionNames(), p.Kind(), p.Path(), po.f.Name) })) } @@ -1382,780 +1146,21 @@ func (h *HugoSites) resolveAndResetDependententPageOutputs(ctx context.Context, return checkedCount, resetCount, err } -// Calculate and apply aggregate values to the page tree (e.g. dates, cascades). -func (sa *sitePagesAssembler) applyAggregates() error { - sectionPageCount := map[string]int{} - - pw := &doctree.NodeShiftTreeWalker[contentNodeI]{ - Tree: sa.pageMap.treePages, - LockType: doctree.LockTypeRead, - WalkContext: &doctree.WalkContext[contentNodeI]{}, - } - rw := pw.Extend() - rw.Tree = sa.pageMap.treeResources - sa.lastmod = time.Time{} - rebuild := sa.s.h.isRebuild() - - pw.Handle = func(keyPage string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { - pageBundle := n.(*pageState) - - if pageBundle.Kind() == kinds.KindTerm { - // Delay this until they're created. - return false, nil - } - - if pageBundle.IsPage() { - rootSection := pageBundle.Section() - sectionPageCount[rootSection]++ - } - - // Handle cascades first to get any default dates set. - var cascade *maps.Ordered[page.PageMatcher, page.PageMatcherParamsConfig] - if keyPage == "" { - // Home page gets it's cascade from the site config. - cascade = sa.conf.Cascade.Config - - if pageBundle.m.pageConfig.CascadeCompiled == nil { - // Pass the site cascade downwards. - pw.WalkContext.Data().Insert(keyPage, cascade) - } - } else { - _, data := pw.WalkContext.Data().LongestPrefix(paths.Dir(keyPage)) - if data != nil { - cascade = data.(*maps.Ordered[page.PageMatcher, page.PageMatcherParamsConfig]) - } - } - - if rebuild { - if (pageBundle.IsHome() || pageBundle.IsSection()) && pageBundle.m.setMetaPostCount > 0 { - oldDates := pageBundle.m.pageConfig.Dates - - // We need to wait until after the walk to determine if any of the dates have changed. - pw.WalkContext.AddPostHook( - func() error { - if oldDates != pageBundle.m.pageConfig.Dates { - sa.assembleChanges.Add(pageBundle) - } - return nil - }, - ) - } - } - - // Combine the cascade map with front matter. - if err := pageBundle.setMetaPost(cascade); err != nil { - return false, err - } - - // We receive cascade values from above. If this leads to a change compared - // to the previous value, we need to mark the page and its dependencies as changed. - if rebuild && pageBundle.m.setMetaPostCascadeChanged { - sa.assembleChanges.Add(pageBundle) - } - - const eventName = "dates" - if n.isContentNodeBranch() { - if pageBundle.m.pageConfig.CascadeCompiled != nil { - // Pass it down. - pw.WalkContext.Data().Insert(keyPage, pageBundle.m.pageConfig.CascadeCompiled) - } - - wasZeroDates := pageBundle.m.pageConfig.Dates.IsAllDatesZero() - if wasZeroDates || pageBundle.IsHome() { - pw.WalkContext.AddEventListener(eventName, keyPage, func(e *doctree.Event[contentNodeI]) { - sp, ok := e.Source.(*pageState) - if !ok { - return - } - - if wasZeroDates { - pageBundle.m.pageConfig.Dates.UpdateDateAndLastmodAndPublishDateIfAfter(sp.m.pageConfig.Dates) - } - - if pageBundle.IsHome() { - if pageBundle.m.pageConfig.Dates.Lastmod.After(pageBundle.s.lastmod) { - pageBundle.s.lastmod = pageBundle.m.pageConfig.Dates.Lastmod - } - if sp.m.pageConfig.Dates.Lastmod.After(pageBundle.s.lastmod) { - pageBundle.s.lastmod = sp.m.pageConfig.Dates.Lastmod - } - } - }) - } - } - - // Send the date info up the tree. - pw.WalkContext.SendEvent(&doctree.Event[contentNodeI]{Source: n, Path: keyPage, Name: eventName}) - - isBranch := n.isContentNodeBranch() - rw.Prefix = keyPage + "/" - - rw.Handle = func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { - if isBranch { - ownerKey, _ := pw.Tree.LongestPrefix(resourceKey, true, nil) - if ownerKey != keyPage { - // Stop walking downwards, someone else owns this resource. - rw.SkipPrefix(ownerKey + "/") - return false, nil - } - } - rs := n.(*resourceSource) - if rs.isPage() { - pageResource := rs.r.(*pageState) - relPath := pageResource.m.pathInfo.BaseRel(pageBundle.m.pathInfo) - pageResource.m.resourcePath = relPath - var cascade *maps.Ordered[page.PageMatcher, page.PageMatcherParamsConfig] - // Apply cascade (if set) to the page. - _, data := pw.WalkContext.Data().LongestPrefix(resourceKey) - if data != nil { - cascade = data.(*maps.Ordered[page.PageMatcher, page.PageMatcherParamsConfig]) - } - if err := pageResource.setMetaPost(cascade); err != nil { - return false, err - } - } - - return false, nil - } - return false, rw.Walk(sa.ctx) - } - - if err := pw.Walk(sa.ctx); err != nil { - return err - } - - if err := pw.WalkContext.HandleEventsAndHooks(); err != nil { - return err - } - - if !sa.s.conf.C.IsMainSectionsSet() { - var mainSection string - var maxcount int - for section, counter := range sectionPageCount { - if section != "" && counter > maxcount { - mainSection = section - maxcount = counter - } - } - sa.s.conf.C.SetMainSections([]string{mainSection}) - - } - - return nil -} - -func (sa *sitePagesAssembler) applyAggregatesToTaxonomiesAndTerms() error { - walkContext := &doctree.WalkContext[contentNodeI]{} - - handlePlural := func(key string) error { - var pw *doctree.NodeShiftTreeWalker[contentNodeI] - pw = &doctree.NodeShiftTreeWalker[contentNodeI]{ - Tree: sa.pageMap.treePages, - Prefix: key, // We also want to include the root taxonomy nodes, so no trailing slash. - LockType: doctree.LockTypeRead, - WalkContext: walkContext, - Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { - p := n.(*pageState) - if p.Kind() != kinds.KindTerm { - // The other kinds were handled in applyAggregates. - if p.m.pageConfig.CascadeCompiled != nil { - // Pass it down. - pw.WalkContext.Data().Insert(s, p.m.pageConfig.CascadeCompiled) - } - } - - if p.Kind() != kinds.KindTerm && p.Kind() != kinds.KindTaxonomy { - // Already handled. - return false, nil - } - - const eventName = "dates" - - if p.Kind() == kinds.KindTerm { - var cascade *maps.Ordered[page.PageMatcher, page.PageMatcherParamsConfig] - _, data := pw.WalkContext.Data().LongestPrefix(s) - if data != nil { - cascade = data.(*maps.Ordered[page.PageMatcher, page.PageMatcherParamsConfig]) - } - if err := p.setMetaPost(cascade); err != nil { - return false, err - } - if !p.s.shouldBuild(p) { - sa.pageMap.treePages.Delete(s) - sa.pageMap.treeTaxonomyEntries.DeletePrefix(paths.AddTrailingSlash(s)) - } else if err := sa.pageMap.treeTaxonomyEntries.WalkPrefix( - doctree.LockTypeRead, - paths.AddTrailingSlash(s), - func(ss string, wn *weightedContentNode) (bool, error) { - // Send the date info up the tree. - pw.WalkContext.SendEvent(&doctree.Event[contentNodeI]{Source: wn.n, Path: ss, Name: eventName}) - return false, nil - }, - ); err != nil { - return false, err - } - } - - // Send the date info up the tree. - pw.WalkContext.SendEvent(&doctree.Event[contentNodeI]{Source: n, Path: s, Name: eventName}) - - if p.m.pageConfig.Dates.IsAllDatesZero() { - pw.WalkContext.AddEventListener(eventName, s, func(e *doctree.Event[contentNodeI]) { - sp, ok := e.Source.(*pageState) - if !ok { - return - } - - p.m.pageConfig.Dates.UpdateDateAndLastmodAndPublishDateIfAfter(sp.m.pageConfig.Dates) - }) - } - - return false, nil - }, - } - - if err := pw.Walk(sa.ctx); err != nil { - return err - } - return nil - } - - for _, viewName := range sa.pageMap.cfg.taxonomyConfig.views { - if err := handlePlural(viewName.pluralTreeKey); err != nil { - return err - } - } - - if err := walkContext.HandleEventsAndHooks(); err != nil { - return err - } - - return nil -} - -func (sa *sitePagesAssembler) assembleTermsAndTranslations() error { - if sa.pageMap.cfg.taxonomyTermDisabled { - return nil - } - - var ( - pages = sa.pageMap.treePages - entries = sa.pageMap.treeTaxonomyEntries - views = sa.pageMap.cfg.taxonomyConfig.views - ) - - rebuild := sa.s.h.isRebuild() - - lockType := doctree.LockTypeWrite - w := &doctree.NodeShiftTreeWalker[contentNodeI]{ - Tree: pages, - LockType: lockType, - Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { - ps := n.(*pageState) - - if ps.m.noLink() { - return false, nil - } - - for _, viewName := range views { - vals := types.ToStringSlicePreserveString(getParam(ps, viewName.plural, false)) - if vals == nil { - continue - } - - w := getParamToLower(ps, viewName.plural+"_weight") - weight, err := cast.ToIntE(w) - if err != nil { - sa.Log.Warnf("Unable to convert taxonomy weight %#v to int for %q", w, n.Path()) - // weight will equal zero, so let the flow continue - } - - for i, v := range vals { - if v == "" { - continue - } - viewTermKey := "/" + viewName.plural + "/" + v - pi := sa.Site.Conf.PathParser().Parse(files.ComponentFolderContent, viewTermKey+"/_index.md") - term := pages.Get(pi.Base()) - if term == nil { - if rebuild { - // A new tag was added in server mode. - taxonomy := pages.Get(viewName.pluralTreeKey) - if taxonomy != nil { - sa.assembleChanges.Add(taxonomy.GetIdentity()) - } - } - - m := &pageMeta{ - term: v, - singular: viewName.singular, - s: sa.Site, - pathInfo: pi, - pageMetaParams: &pageMetaParams{ - pageConfig: &pagemeta.PageConfig{ - PageConfigEarly: pagemeta.PageConfigEarly{ - Kind: kinds.KindTerm, - }, - }, - }, - } - n, pi, err := sa.h.newPage(m) - if err != nil { - return false, err - } - pages.InsertIntoValuesDimension(pi.Base(), n) - term = pages.Get(pi.Base()) - } else { - m := term.(*pageState).m - m.term = v - m.singular = viewName.singular - } - - if s == "" { - // Consider making this the real value. - s = "/" - } - - key := pi.Base() + s - - entries.Insert(key, &weightedContentNode{ - weight: weight, - n: n, - term: &pageWithOrdinal{pageState: term.(*pageState), ordinal: i}, - }) - } - } - - return false, nil - }, - } - - return w.Walk(sa.ctx) -} - -func (sa *sitePagesAssembler) assembleResources() error { - pagesTree := sa.pageMap.treePages - resourcesTree := sa.pageMap.treeResources - - lockType := doctree.LockTypeWrite - w := &doctree.NodeShiftTreeWalker[contentNodeI]{ - Tree: pagesTree, - LockType: lockType, - Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { - ps := n.(*pageState) - - // This is a little out of place, but is conveniently put here. - // Check if translationKey is set by user. - // This is to support the manual way of setting the translationKey in front matter. - if ps.m.pageConfig.TranslationKey != "" { - sa.s.h.translationKeyPages.Append(ps.m.pageConfig.TranslationKey, ps) - } - - // Prepare resources for this page. - ps.shiftToOutputFormat(true, 0) - targetPaths := ps.targetPaths() - baseTarget := targetPaths.SubResourceBaseTarget - duplicateResourceFiles := true - if ps.m.pageConfig.ContentMediaType.IsMarkdown() { - duplicateResourceFiles = ps.s.ContentSpec.Converters.GetMarkupConfig().Goldmark.DuplicateResourceFiles - } - - if !sa.h.isRebuild() { - if ps.hasRenderableOutput() { - // For multi output pages this will not be complete, but will have to do for now. - sa.h.progressReporter.numPagesToRender.Add(1) - } - } - - duplicateResourceFiles = duplicateResourceFiles || ps.s.Conf.IsMultihost() - - err := sa.pageMap.forEachResourceInPage( - ps, lockType, - !duplicateResourceFiles, - func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { - rs := n.(*resourceSource) - if !match.Has(doctree.DimensionLanguage) { - // We got an alternative language version. - // Clone this and insert it into the tree. - rs = rs.clone() - resourcesTree.InsertIntoCurrentDimension(resourceKey, rs) - } - if rs.r != nil { - return false, nil - } - - relPathOriginal := rs.path.Unnormalized().PathRel(ps.m.pathInfo.Unnormalized()) - relPath := rs.path.BaseRel(ps.m.pathInfo) - - var targetBasePaths []string - if ps.s.Conf.IsMultihost() { - baseTarget = targetPaths.SubResourceBaseLink - // In multihost we need to publish to the lang sub folder. - targetBasePaths = []string{ps.s.GetTargetLanguageBasePath()} // TODO(bep) we don't need this as a slice anymore. - - } - - if rs.rc != nil && rs.rc.Content.IsResourceValue() { - if rs.rc.Name == "" { - rs.rc.Name = relPathOriginal - } - r, err := ps.m.s.ResourceSpec.NewResourceWrapperFromResourceConfig(rs.rc) - if err != nil { - return false, err - } - rs.r = r - return false, nil - } - - var mt media.Type - if rs.rc != nil { - mt = rs.rc.ContentMediaType - } - - var filename string - if rs.fi != nil { - filename = rs.fi.Meta().Filename - } - - rd := resources.ResourceSourceDescriptor{ - OpenReadSeekCloser: rs.opener, - Path: rs.path, - GroupIdentity: rs.path, - TargetPath: relPathOriginal, // Use the original path for the target path, so the links can be guessed. - TargetBasePaths: targetBasePaths, - BasePathRelPermalink: targetPaths.SubResourceBaseLink, - BasePathTargetPath: baseTarget, - SourceFilenameOrPath: filename, - NameNormalized: relPath, - NameOriginal: relPathOriginal, - MediaType: mt, - LazyPublish: !ps.m.pageConfig.Build.PublishResources, - } - - if rs.rc != nil { - rc := rs.rc - rd.OpenReadSeekCloser = rc.Content.ValueAsOpenReadSeekCloser() - if rc.Name != "" { - rd.NameNormalized = rc.Name - rd.NameOriginal = rc.Name - } - if rc.Title != "" { - rd.Title = rc.Title - } - rd.Params = rc.Params - } - - r, err := ps.m.s.ResourceSpec.NewResource(rd) - if err != nil { - return false, err - } - rs.r = r - return false, nil - }, - ) - - return false, err - }, - } - - return w.Walk(sa.ctx) -} - -func (sa *sitePagesAssembler) assemblePagesStep1(ctx context.Context) error { - if err := sa.addMissingTaxonomies(); err != nil { - return err - } - if err := sa.addMissingRootSections(); err != nil { - return err - } - if err := sa.addStandalonePages(); err != nil { - return err - } - if err := sa.applyAggregates(); err != nil { - return err - } - return nil -} - -func (sa *sitePagesAssembler) assemblePagesStep2() error { - if err := sa.removeShouldNotBuild(); err != nil { - return err - } - if err := sa.assembleTermsAndTranslations(); err != nil { - return err - } - if err := sa.applyAggregatesToTaxonomiesAndTerms(); err != nil { - return err - } - - return nil -} - -func (sa *sitePagesAssembler) assemblePagesStepFinal() error { - if err := sa.assembleResources(); err != nil { - return err - } - return nil -} - -// Remove any leftover node that we should not build for some reason (draft, expired, scheduled in the future). -// Note that for the home and section kinds we just disable the nodes to preserve the structure. -func (sa *sitePagesAssembler) removeShouldNotBuild() error { - s := sa.Site - var keys []string - w := &doctree.NodeShiftTreeWalker[contentNodeI]{ - LockType: doctree.LockTypeRead, - Tree: sa.pageMap.treePages, - Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { - p := n.(*pageState) - if !s.shouldBuild(p) { - switch p.Kind() { - case kinds.KindHome, kinds.KindSection, kinds.KindTaxonomy: - // We need to keep these for the structure, but disable - // them so they don't get listed/rendered. - (&p.m.pageConfig.Build).Disable() - default: - keys = append(keys, key) - } - } - return false, nil - }, - } - if err := w.Walk(sa.ctx); err != nil { - return err - } - - if len(keys) == 0 { - return nil - } - - sa.pageMap.DeletePageAndResourcesBelow(keys...) - - return nil -} - -// // Create the fixed output pages, e.g. sitemap.xml, if not already there. -func (sa *sitePagesAssembler) addStandalonePages() error { - s := sa.Site - m := s.pageMap - tree := m.treePages - - commit := tree.Lock(true) - defer commit() - - addStandalone := func(key, kind string, f output.Format) { - if !s.Conf.IsMultihost() { - switch kind { - case kinds.KindSitemapIndex, kinds.KindRobotsTXT: - // Only one for all languages. - if s.languagei != 0 { - return - } - } - } - - if !sa.Site.conf.IsKindEnabled(kind) || tree.Has(key) { - return - } - - m := &pageMeta{ - s: s, - pathInfo: s.Conf.PathParser().Parse(files.ComponentFolderContent, key+f.MediaType.FirstSuffix.FullSuffix), - pageMetaParams: &pageMetaParams{ - pageConfig: &pagemeta.PageConfig{ - PageConfigEarly: pagemeta.PageConfigEarly{ - Kind: kind, - }, - }, - }, - standaloneOutputFormat: f, - } - - p, _, _ := s.h.newPage(m) - - tree.InsertIntoValuesDimension(key, p) - } - - addStandalone("/404", kinds.KindStatus404, output.HTTPStatus404HTMLFormat) - - if s.conf.EnableRobotsTXT { - if m.i == 0 || s.Conf.IsMultihost() { - addStandalone("/_robots", kinds.KindRobotsTXT, output.RobotsTxtFormat) - } - } - - sitemapEnabled := false - for _, s := range s.h.Sites { - if s.conf.IsKindEnabled(kinds.KindSitemap) { - sitemapEnabled = true - break - } - } - - if sitemapEnabled { - of := output.SitemapFormat - if s.conf.Sitemap.Filename != "" { - of.BaseName = paths.Filename(s.conf.Sitemap.Filename) - } - addStandalone("/_sitemap", kinds.KindSitemap, of) - - skipSitemapIndex := s.Conf.IsMultihost() || !(s.Conf.DefaultContentLanguageInSubdir() || s.Conf.IsMultilingual()) - if !skipSitemapIndex { - of = output.SitemapIndexFormat - if s.conf.Sitemap.Filename != "" { - of.BaseName = paths.Filename(s.conf.Sitemap.Filename) - } - addStandalone("/_sitemapindex", kinds.KindSitemapIndex, of) - } - } - - return nil -} - -func (sa *sitePagesAssembler) addMissingRootSections() error { - var hasHome bool - - // Add missing root sections. - seen := map[string]bool{} - var w *doctree.NodeShiftTreeWalker[contentNodeI] - w = &doctree.NodeShiftTreeWalker[contentNodeI]{ - LockType: doctree.LockTypeWrite, - Tree: sa.pageMap.treePages, - Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { - if n == nil { - panic("n is nil") - } - - ps := n.(*pageState) - - if ps.Lang() != sa.Lang() { - panic(fmt.Sprintf("lang mismatch: %q: %s != %s", s, ps.Lang(), sa.Lang())) - } - - if s == "" { - hasHome = true - sa.home = ps - return false, nil - } - - switch ps.Kind() { - case kinds.KindPage, kinds.KindSection: - // OK - default: - // Skip taxonomy nodes etc. - return false, nil - } - - p := ps.m.pathInfo - section := p.Section() - if section == "" || seen[section] { - return false, nil - } - seen[section] = true - - // Try to preserve the original casing if possible. - sectionUnnormalized := p.Unnormalized().Section() - pth := sa.s.Conf.PathParser().Parse(files.ComponentFolderContent, "/"+sectionUnnormalized+"/_index.md") - nn := w.Tree.Get(pth.Base()) - - if nn == nil { - m := &pageMeta{ - s: sa.Site, - pathInfo: pth, - } - - ps, pth, err := sa.h.newPage(m) - if err != nil { - return false, err - } - w.Tree.InsertIntoValuesDimension(pth.Base(), ps) - } - - // /a/b, we don't need to walk deeper. - if strings.Count(s, "/") > 1 { - w.SkipPrefix(s + "/") - } - - return false, nil - }, - } - - if err := w.Walk(sa.ctx); err != nil { - return err - } - - if !hasHome { - p := sa.Site.Conf.PathParser().Parse(files.ComponentFolderContent, "/_index.md") - m := &pageMeta{ - s: sa.Site, - pathInfo: p, - pageMetaParams: &pageMetaParams{ - pageConfig: &pagemeta.PageConfig{ - PageConfigEarly: pagemeta.PageConfigEarly{ - Kind: kinds.KindHome, - }, - }, - }, - } - n, p, err := sa.h.newPage(m) - if err != nil { - return err - } - w.Tree.InsertIntoValuesDimensionWithLock(p.Base(), n) - sa.home = n - } - - return nil -} - -func (sa *sitePagesAssembler) addMissingTaxonomies() error { - if sa.pageMap.cfg.taxonomyDisabled && sa.pageMap.cfg.taxonomyTermDisabled { - return nil - } - - tree := sa.pageMap.treePages - - commit := tree.Lock(true) - defer commit() - - for _, viewName := range sa.pageMap.cfg.taxonomyConfig.views { - key := viewName.pluralTreeKey - if v := tree.Get(key); v == nil { - m := &pageMeta{ - s: sa.Site, - pathInfo: sa.Conf.PathParser().Parse(files.ComponentFolderContent, key+"/_index.md"), - pageMetaParams: &pageMetaParams{ - pageConfig: &pagemeta.PageConfig{ - PageConfigEarly: pagemeta.PageConfigEarly{ - Kind: kinds.KindTaxonomy, - }, - }, - }, - singular: viewName.singular, - } - p, _, _ := sa.h.newPage(m) - tree.InsertIntoValuesDimension(key, p) - } - } - - return nil -} - -func (m *pageMap) CreateSiteTaxonomies(ctx context.Context) error { - m.s.taxonomies = make(page.TaxonomyList) +func (m *pageMap) CreateSiteTaxonomies(ctx context.Context) (page.TaxonomyList, error) { + taxonomies := make(page.TaxonomyList) if m.cfg.taxonomyDisabled && m.cfg.taxonomyTermDisabled { - return nil + return taxonomies, nil } for _, viewName := range m.cfg.taxonomyConfig.views { key := viewName.pluralTreeKey - m.s.taxonomies[viewName.plural] = make(page.Taxonomy) - w := &doctree.NodeShiftTreeWalker[contentNodeI]{ + taxonomies[viewName.plural] = make(page.Taxonomy) + w := &doctree.NodeShiftTreeWalker[contentNode]{ Tree: m.treePages, Prefix: paths.AddTrailingSlash(key), LockType: doctree.LockTypeRead, - Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { + Handle: func(s string, n contentNode) (bool, error) { p := n.(*pageState) switch p.Kind() { @@ -2163,7 +1168,7 @@ func (m *pageMap) CreateSiteTaxonomies(ctx context.Context) error { if !p.m.shouldList(true) { return false, nil } - taxonomy := m.s.taxonomies[viewName.plural] + taxonomy := taxonomies[viewName.plural] if taxonomy == nil { return true, fmt.Errorf("missing taxonomy: %s", viewName.plural) } @@ -2193,17 +1198,22 @@ func (m *pageMap) CreateSiteTaxonomies(ctx context.Context) error { } if err := w.Walk(ctx); err != nil { - return err + return nil, err } } - for _, taxonomy := range m.s.taxonomies { + for _, taxonomy := range taxonomies { for _, v := range taxonomy { v.Sort() } } - return nil + return taxonomies, nil +} + +type term struct { + view viewName + term string } type viewName struct { diff --git a/hugolib/content_map_page_assembler.go b/hugolib/content_map_page_assembler.go new file mode 100644 index 000000000..a659f1b24 --- /dev/null +++ b/hugolib/content_map_page_assembler.go @@ -0,0 +1,1361 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "context" + "fmt" + "path" + "strings" + "time" + + "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/hugofs/files" + "github.com/gohugoio/hugo/hugolib/doctree" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/output" + "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/kinds" + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/page/pagemeta" + "github.com/spf13/cast" +) + +// Handles the flow of creating all the pages and resources for all sites. +// This includes applying any cascade configuration passed down the tree. +type allPagesAssembler struct { + // Dependencies. + ctx context.Context + h *HugoSites + m *pageMap + + assembleChanges *WhatChanged + seenTerms map[term]sitesmatrix.Vectors + droppedPages map[*Site][]string // e.g. drafts, expired, future. + + // Internal state. + pw *doctree.NodeShiftTreeWalker[contentNode] // walks pages. + rw *doctree.NodeShiftTreeWalker[contentNode] // walks resources. +} + +func newAllPagesAssembler( + ctx context.Context, + h *HugoSites, + m *pageMap, + assembleChanges *WhatChanged, +) *allPagesAssembler { + rw := &doctree.NodeShiftTreeWalker[contentNode]{ + Tree: m.treeResources, + LockType: doctree.LockTypeNone, + NoShift: true, + WalkContext: &doctree.WalkContext[contentNode]{}, + } + pw := rw.Extend() + pw.Tree = m.treePages + + return &allPagesAssembler{ + ctx: ctx, + h: h, + m: m, + assembleChanges: assembleChanges, + seenTerms: map[term]sitesmatrix.Vectors{}, + droppedPages: map[*Site][]string{}, + + pw: pw, + rw: rw, + } +} + +type sitePagesAssembler struct { + s *Site + assembleChanges *WhatChanged + a *allPagesAssembler + ctx context.Context +} + +// No locking. +func (a *allPagesAssembler) createAllPages() error { + var ( + sites = a.h.sitesVersionsRolesMap + h = a.h + treePages = a.m.treePages + + getViews = func(vec sitesmatrix.Vector) []viewName { + return h.languageSiteForSiteVector(vec).pageMap.cfg.taxonomyConfig.views + } + ) + + resourceOwnerInfo := struct { + n contentNode + s string + }{} + + defer func() { + for site, dropped := range a.droppedPages { + for _, s := range dropped { + site.pageMap.treePages.Delete(s) + site.pageMap.resourceTrees.DeletePrefix(paths.AddTrailingSlash(s)) + } + } + }() + + if a.h.Conf.Watching() { + defer func() { + if a.h.isRebuild() && a.h.previousSeenTerms != nil { + // Find removed terms. + for t := range a.h.previousSeenTerms { + if _, found := a.seenTerms[t]; !found { + // This term has been removed. + a.pw.Tree.Delete(t.view.pluralTreeKey) + a.rw.Tree.DeletePrefix(t.view.pluralTreeKey + "/") + } + } + // Find new terms. + for t := range a.seenTerms { + if _, found := a.h.previousSeenTerms[t]; !found { + // This term is new. + if n, ok := a.pw.Tree.GetRaw(t.view.pluralTreeKey); ok { + a.assembleChanges.Add(cnh.GetIdentity(n)) + } + } + } + } + a.h.previousSeenTerms = a.seenTerms + }() + } + + newHomePageMetaSource := func() *pageMetaSource { + pi := a.h.Conf.PathParser().Parse(files.ComponentFolderContent, "/_index.md") + return &pageMetaSource{ + pathInfo: pi, + sitesMatrixBase: a.h.Conf.AllSitesMatrix(), + sitesMatrixBaseOnly: true, + pageConfigSource: &pagemeta.PageConfigEarly{ + Kind: kinds.KindHome, + }, + } + } + + if err := a.createMissingPages(); err != nil { + return err + } + + if treePages.Len() == 0 { + // No pages, insert a home page to get something to walk on. + p := newHomePageMetaSource() + treePages.InsertRaw(p.pathInfo.Base(), p) + } + + getCascades := func(wctx *doctree.WalkContext[contentNode], s string) *page.PageMatcherParamsConfigs { + if wctx == nil { + panic("nil walk context") + } + var cascades *page.PageMatcherParamsConfigs + data := wctx.Data() + if s != "" { + _, data := data.LongestPrefix(s) + + if data != nil { + cascades = data.(*page.PageMatcherParamsConfigs) + } + } + + if cascades == nil { + for s := range a.h.allSiteLanguages(nil) { + cascades = cascades.Append(s.conf.Cascade) + } + } + return cascades + } + + doTransformPages := func(s string, n contentNode, cascades *page.PageMatcherParamsConfigs) (n2 contentNode, ns doctree.NodeTransformState, err error) { + defer func() { + cascadesLen := cascades.Len() + if n2 == nil { + if ns < doctree.NodeTransformStateSkip { + ns = doctree.NodeTransformStateSkip + } + } else { + n2.forEeachContentNode( + func(vec sitesmatrix.Vector, nn contentNode) bool { + if pms, ok := nn.(contentNodeCascadeProvider); ok { + cascades = cascades.Prepend(pms.getCascade()) + } + return true + }, + ) + } + + if s == "" || cascades.Len() > cascadesLen { + // New cascade values added, pass them downwards. + a.rw.WalkContext.Data().Insert(paths.AddTrailingSlash(s), cascades) + } + }() + + var handlePageMetaSource func(v contentNode, n contentNodesMap, replaceVector bool) (n2 contentNode, err error) + handlePageMetaSource = func(v contentNode, n contentNodesMap, replaceVector bool) (n2 contentNode, err error) { + if n != nil { + n2 = n + } + + switch ms := v.(type) { + case *pageMetaSource: + if err := ms.initEarly(a.h, cascades); err != nil { + return nil, err + } + + sitesMatrix := ms.pageConfigSource.SitesMatrix + + sitesMatrix.ForEachVector(func(vec sitesmatrix.Vector) bool { + site, found := sites[vec] + if !found { + panic(fmt.Sprintf("site not found for %v", vec)) + } + + var p *pageState + p, err = site.newPageFromPageMetasource(ms, cascades) + if err != nil { + return false + } + + var drop bool + if !site.shouldBuild(p) { + switch p.Kind() { + case kinds.KindHome, kinds.KindSection, kinds.KindTaxonomy: + // We need to keep these for the structure, but disable + // them so they don't get listed/rendered. + (&p.m.pageConfig.Build).Disable() + default: + // Skip this page. + a.droppedPages[site] = append(a.droppedPages[site], v.Path()) + drop = true + } + } + + if !drop && n == nil { + if n2 == nil { + // Avoid creating a map for one node. + n2 = p + } else { + // Upgrade to a map. + n = make(contentNodesMap) + ps := n2.(*pageState) + n[ps.s.siteVector] = ps + n2 = n + } + } + + if n == nil { + return true + } + + pp, found := n[vec] + + var w1, w2 int + if wp, ok := pp.(contentNodeContentWeightProvider); ok { + w1 = wp.contentWeight() + } + w2 = p.contentWeight() + + if found && !replaceVector && w1 > w2 { + return true + } + + n[vec] = p + return true + }) + case *pageState: + if n == nil { + n2 = ms + return + } + n[ms.s.siteVector] = ms + case contentNodesMap: + for _, vv := range ms { + var err error + n2, err = handlePageMetaSource(vv, n, replaceVector) + if err != nil { + return nil, err + } + if m, ok := n2.(contentNodesMap); ok { + n = m + } + + } + default: + panic(fmt.Sprintf("unexpected type %T", v)) + } + + return + } + + // The common case. + ns = doctree.NodeTransformStateReplaced + + handleContentNodeSeq := func(v contentNodeSeq) (contentNode, doctree.NodeTransformState, error) { + is := make(contentNodesMap) + for ms := range v { + _, err := handlePageMetaSource(ms, is, false) + if err != nil { + return nil, 0, fmt.Errorf("failed to create page from pageMetaSource %s: %w", s, err) + } + } + return is, ns, nil + } + + switch v := n.(type) { + case contentNodeSeq, contentNodes: + return handleContentNodeSeq(contentNodeToSeq(v)) + case *pageMetaSource: + n2, err = handlePageMetaSource(v, nil, false) + if err != nil { + return nil, 0, fmt.Errorf("failed to create page from pageMetaSource %s: %w", s, err) + } + return + case *pageState: + // Nothing to do. + ns = doctree.NodeTransformStateNone + return v, ns, nil + case contentNodesMap: + ns = doctree.NodeTransformStateNone + for _, vv := range v { + switch m := vv.(type) { + case *pageMetaSource: + ns = doctree.NodeTransformStateUpdated + _, err := handlePageMetaSource(m, v, true) + if err != nil { + return nil, 0, fmt.Errorf("failed to create page from pageMetaSource %s: %w", s, err) + } + default: + // Nothing to do. + } + } + + return v, ns, nil + + default: + panic(fmt.Sprintf("unexpected type %T", n)) + } + } + + var ( + seenRootSections = map[string]bool{ + "": true, // The home section. + } + seenHome bool + ) + + var unpackPageMetaSources func(n contentNode) contentNode + unpackPageMetaSources = func(n contentNode) contentNode { + switch nn := n.(type) { + case *pageState: + return nn.m.pageMetaSource + case contentNodesMap: + if len(nn) == 0 { + return nil + } + var iter contentNodeSeq = func(yield func(contentNode) bool) { + seen := map[*pageMetaSource]struct{}{} + for _, v := range nn { + vv := unpackPageMetaSources(v) + pms := vv.(*pageMetaSource) + if _, found := seen[pms]; !found { + if !yield(pms) { + return + } + seen[pms] = struct{}{} + } + } + } + return iter + case contentNodes: + if len(nn) == 0 { + return nil + } + var iter contentNodeSeq = func(yield func(contentNode) bool) { + seen := map[*pageMetaSource]struct{}{} + for _, v := range nn { + vv := unpackPageMetaSources(v) + pms := vv.(*pageMetaSource) + if _, found := seen[pms]; !found { + if !yield(pms) { + return + } + seen[pms] = struct{}{} + } + } + } + return iter + case *pageMetaSource: + return nn + default: + panic(fmt.Sprintf("unexpected type %T", n)) + } + } + + transformPages := func(s string, n contentNode, cascades *page.PageMatcherParamsConfigs) (n2 contentNode, ns doctree.NodeTransformState, err error) { + if a.h.isRebuild() { + cascadesPrevious := getCascades(h.previousPageTreesWalkContext, s) + h1, h2 := cascadesPrevious.SourceHash(), cascades.SourceHash() + if h1 != h2 { + // Force rebuild from the source. + n = unpackPageMetaSources(n) + } + } + if n == nil { + panic("nil node") + } + n2, ns, err = doTransformPages(s, n, cascades) + + return + } + + transformPagesAndCreateMissingHome := func(s string, n contentNode, isResource bool, cascades *page.PageMatcherParamsConfigs) (n2 contentNode, ns doctree.NodeTransformState, err error) { + if n == nil { + panic("nil node " + s) + } + level := strings.Count(s, "/") + + if s == "" { + seenHome = true + } + + if !isResource && s != "" && !seenHome { + + var homePages contentNode + homePages, ns, err = transformPages("", newHomePageMetaSource(), cascades) + if err != nil { + return + } + treePages.InsertRaw("", homePages) + seenHome = true + } + + n2, ns, err = transformPages(s, n, cascades) + if err != nil || ns >= doctree.NodeTransformStateSkip { + return + } + + if n2 == nil { + ns = doctree.NodeTransformStateSkip + return + } + + if isResource { + // Done. + return + } + + isTaxonomy := !a.h.getFirstTaxonomyConfig(s).IsZero() + isRootSetion := !isTaxonomy && level == 1 && cnh.isBranchNode(n) + + if isRootSetion { + // This is a root section. + seenRootSections[cnh.PathInfo(n).Section()] = true + } else if !isTaxonomy { + p := cnh.PathInfo(n) + rootSection := p.Section() + if !seenRootSections[rootSection] { + seenRootSections[rootSection] = true + // Try to preserve the original casing if possible. + sectionUnnormalized := p.Unnormalized().Section() + rootSectionPath := a.h.Conf.PathParser().Parse(files.ComponentFolderContent, "/"+sectionUnnormalized+"/_index.md") + var rootSectionPages contentNode + rootSectionPages, _, err = transformPages(rootSectionPath.Base(), &pageMetaSource{ + pathInfo: rootSectionPath, + sitesMatrixBase: n2.(contentNodeForSites).siteVectors(), + pageConfigSource: &pagemeta.PageConfigEarly{ + Kind: kinds.KindSection, + }, + }, cascades) + if err != nil { + return + } + treePages.InsertRaw(rootSectionPath.Base(), rootSectionPages) + } + } + + const eventNameSitesMatrix = "sitesmatrix" + + if s == "" || isRootSetion { + + // Every page needs a home and a root section (.FirstSection). + // We don't know yet what language, version, role combination that will + // be created below, so collect that information and create the missing pages + // on demand. + nm, replaced := contentNodeToContentNodesPage(n2) + + missingVectorsForHomeOrRootSection := sitesmatrix.Vectors{} + + if s == "" { + // We need a complete set of home pages. + a.h.Conf.AllSitesMatrix().ForEachVector(func(vec sitesmatrix.Vector) bool { + if _, found := nm[vec]; !found { + missingVectorsForHomeOrRootSection[vec] = struct{}{} + } + return true + }) + } else { + a.pw.WalkContext.AddEventListener(eventNameSitesMatrix, s, + func(e *doctree.Event[contentNode]) { + n := e.Source + e.StopPropagation() + n.forEeachContentNode( + func(vec sitesmatrix.Vector, nn contentNode) bool { + if _, found := nm[vec]; !found { + missingVectorsForHomeOrRootSection[vec] = struct{}{} + } + return true + }) + }, + ) + } + + // We need to wait until after the walk to have a complete set. + a.pw.WalkContext.AddPostHook( + func() error { + if i := len(missingVectorsForHomeOrRootSection); i > 0 { + // Pick one, the rest will be created later. + vec := missingVectorsForHomeOrRootSection.Sample() + + kind := kinds.KindSection + if s == "" { + kind = kinds.KindHome + } + + pms := &pageMetaSource{ + pathInfo: cnh.PathInfo(n), + sitesMatrixBase: missingVectorsForHomeOrRootSection, + sitesMatrixBaseOnly: true, + pageConfigSource: &pagemeta.PageConfigEarly{ + Kind: kind, + }, + } + nm[vec] = pms + + _, ns, err := transformPages(s, nm, cascades) + if err != nil { + return err + } + if ns == doctree.NodeTransformStateReplaced { + // Should not happen. + panic(fmt.Sprintf("expected no replacement for %q", s)) + } + + if replaced { + a.pw.Tree.InsertRaw(s, nm) + } + } + return nil + }, + ) + } + + if s != "" { + a.rw.WalkContext.SendEvent(&doctree.Event[contentNode]{Source: n2, Path: s, Name: eventNameSitesMatrix}) + } + + return + } + + transformPagesAndCreateMissingStructuralNodes := func(s string, n contentNode, isResource bool) (n2 contentNode, ns doctree.NodeTransformState, err error) { + cascades := getCascades(a.pw.WalkContext, s) + n2, ns, err = transformPagesAndCreateMissingHome(s, n, isResource, cascades) + if err != nil || ns >= doctree.NodeTransformStateSkip { + return + } + n2.forEeachContentNode( + func(vec sitesmatrix.Vector, nn contentNode) bool { + if ps, ok := nn.(*pageState); ok { + if ps.m.noLink() { + return true + } + for _, viewName := range getViews(vec) { + vals := types.ToStringSlicePreserveString(getParam(ps, viewName.plural, false)) + if vals == nil { + continue + } + for _, v := range vals { + if v == "" { + continue + } + t := term{view: viewName, term: v} + if vectors, found := a.seenTerms[t]; found { + if _, found := vectors[vec]; found { + continue + } + vectors[vec] = struct{}{} + } else { + a.seenTerms[t] = sitesmatrix.Vectors{ + vec: struct{}{}, + } + } + + } + } + } + return true + }, + ) + + return + } + + shouldSkipOrTerminate := func(s string) (ns doctree.NodeTransformState) { + owner := resourceOwnerInfo.n + if owner == nil { + return doctree.NodeTransformStateTerminate + } + if !cnh.isBranchNode(owner) { + return + } + + // A resourceKey always represents a filename with extension. + // A page key points to the logical path of a page, which when sourced from the filesystem + // may represent a directory (bundles) or a single content file (e.g. p1.md). + // So, to avoid any overlapping ambiguity, we start looking from the owning directory. + for { + s = path.Dir(s) + ownerKey, found := treePages.LongestPrefixRaw(s) + if !found { + return doctree.NodeTransformStateTerminate + } + if ownerKey == resourceOwnerInfo.s { + break + } + + if s != ownerKey && strings.HasPrefix(s, ownerKey) { + // Keep looking + continue + } + + // Stop walking downwards, someone else owns this resource. + a.rw.SkipPrefix(ownerKey + "/") + return doctree.NodeTransformStateSkip + } + return + } + + forEeachResourceOwnerPage := func(fn func(p *pageState) bool) bool { + switch nn := resourceOwnerInfo.n.(type) { + case *pageState: + return fn(nn) + case contentNodesMap: + for _, p := range nn { + if !fn(p.(*pageState)) { + return false + } + } + default: + panic(fmt.Sprintf("unknown type %T", nn)) + } + return true + } + + a.rw.Transform = func(s string, n contentNode) (n2 contentNode, ns doctree.NodeTransformState, err error) { + if ns = shouldSkipOrTerminate(s); ns >= doctree.NodeTransformStateSkip { + return + } + + if cnh.isPageNode(n) { + return transformPagesAndCreateMissingStructuralNodes(s, n, true) + } + + nodes := make(contentNodesMap) + ns = doctree.NodeTransformStateReplaced + n2 = nodes + + forEeachResourceOwnerPage( + func(p *pageState) bool { + duplicateResourceFiles := a.h.Cfg.IsMultihost() + if !duplicateResourceFiles && p.m.pageConfigSource.ContentMediaType.IsMarkdown() { + duplicateResourceFiles = p.s.ContentSpec.Converters.GetMarkupConfig().Goldmark.DuplicateResourceFiles + } + + if _, found := nodes[p.s.siteVector]; !found { + var rs *resourceSource + match := cnh.findContentNodeForSiteVector(p.s.siteVector, duplicateResourceFiles, contentNodeToSeq(n)) + if match == nil { + return true + } + + rs = match.(*resourceSource) + + if rs != nil { + if rs.state == resourceStateNew { + nodes[p.s.siteVector] = rs.assignSiteVector(p.s.siteVector) + } else { + nodes[p.s.siteVector] = rs.clone().assignSiteVector(p.s.siteVector) + } + } + } + return true + }, + ) + + return + } + + // Create missing term pages. + a.pw.WalkContext.AddPostHook( + func() error { + for k, v := range a.seenTerms { + viewTermKey := "/" + k.view.plural + "/" + k.term + + pi := a.h.Conf.PathParser().Parse(files.ComponentFolderContent, viewTermKey+"/_index.md") + termKey := pi.Base() + + n, found := a.pw.Tree.GetRaw(termKey) + + if found { + // Merge. + n.forEeachContentNode( + func(vec sitesmatrix.Vector, nn contentNode) bool { + delete(v, vec) + return true + }, + ) + } + + if len(v) > 0 { + p := &pageMetaSource{ + pathInfo: pi, + sitesMatrixBase: v, + pageConfigSource: &pagemeta.PageConfigEarly{ + Kind: kinds.KindTerm, + }, + } + var n2 contentNode = p + if found { + n2 = contentNodes{n, p} + } + n2, ns, err := transformPages(termKey, n2, getCascades(a.pw.WalkContext, termKey)) + if err != nil { + return fmt.Errorf("failed to create term page %q: %w", termKey, err) + } + + switch ns { + case doctree.NodeTransformStateReplaced: + a.pw.Tree.InsertRaw(termKey, n2) + } + + } + } + return nil + }, + ) + + a.pw.Transform = func(s string, n contentNode) (n2 contentNode, ns doctree.NodeTransformState, err error) { + n2, ns, err = transformPagesAndCreateMissingStructuralNodes(s, n, false) + + if err != nil || ns >= doctree.NodeTransformStateSkip { + return + } + + if iep, ok := n2.(contentNodeIsEmptyProvider); ok && iep.isEmpty() { + ns = doctree.NodeTransformStateDeleted + } + + if ns == doctree.NodeTransformStateDeleted { + return + } + + // Walk nested resources. + resourceOwnerInfo.s = s + resourceOwnerInfo.n = n2 + a.rw.Prefix = s + "/" + if err := a.rw.Walk(a.ctx); err != nil { + return nil, 0, err + } + + return + } + + a.pw.Handle = nil + + if err := a.pw.Walk(a.ctx); err != nil { + return err + } + + if err := a.pw.WalkContext.HandleEventsAndHooks(); err != nil { + return err + } + + return nil +} + +// Calculate and apply aggregate values to the page tree (e.g. dates). +func (sa *sitePagesAssembler) applyAggregates() error { + sectionPageCount := map[string]int{} + + pw := &doctree.NodeShiftTreeWalker[contentNode]{ + Tree: sa.s.pageMap.treePages, + LockType: doctree.LockTypeRead, + WalkContext: &doctree.WalkContext[contentNode]{}, + } + rw := pw.Extend() + rw.Tree = sa.s.pageMap.treeResources + sa.s.lastmod = time.Time{} + rebuild := sa.s.h.isRebuild() + + pw.Handle = func(keyPage string, n contentNode) (bool, error) { + pageBundle := n.(*pageState) + + if pageBundle.Kind() == kinds.KindTerm { + // Delay this until they're created. + return false, nil + } + + if pageBundle.IsPage() { + rootSection := pageBundle.Section() + sectionPageCount[rootSection]++ + } + + if rebuild { + if pageBundle.IsHome() || pageBundle.IsSection() { + oldDates := pageBundle.m.pageConfig.Dates + + // We need to wait until after the walk to determine if any of the dates have changed. + pw.WalkContext.AddPostHook( + func() error { + if oldDates != pageBundle.m.pageConfig.Dates { + sa.assembleChanges.Add(pageBundle) + } + return nil + }, + ) + } + } + + const eventName = "dates" + if cnh.isBranchNode(n) { + wasZeroDates := pageBundle.m.pageConfig.Dates.IsAllDatesZero() + if wasZeroDates || pageBundle.IsHome() { + pw.WalkContext.AddEventListener(eventName, keyPage, func(e *doctree.Event[contentNode]) { + sp, ok := e.Source.(*pageState) + if !ok { + return + } + + if wasZeroDates { + pageBundle.m.pageConfig.Dates.UpdateDateAndLastmodAndPublishDateIfAfter(sp.m.pageConfig.Dates) + } + + if pageBundle.IsHome() { + if pageBundle.m.pageConfig.Dates.Lastmod.After(pageBundle.s.lastmod) { + pageBundle.s.lastmod = pageBundle.m.pageConfig.Dates.Lastmod + } + if sp.m.pageConfig.Dates.Lastmod.After(pageBundle.s.lastmod) { + pageBundle.s.lastmod = sp.m.pageConfig.Dates.Lastmod + } + } + }) + } + } + + // Send the date info up the tree. + pw.WalkContext.SendEvent(&doctree.Event[contentNode]{Source: n, Path: keyPage, Name: eventName}) + + isBranch := cnh.isBranchNode(n) + rw.Prefix = keyPage + "/" + rw.IncludeRawFilter = func(s string, n contentNode) bool { + // Only page nodes. + return cnh.isPageNode(n) + } + + rw.IncludeFilter = func(s string, n contentNode) bool { + switch n.(type) { + case *pageState: + return true + default: + // We only want to handle page nodes here. + return false + } + } + + rw.Handle = func(resourceKey string, n contentNode) (bool, error) { + if isBranch { + ownerKey, _ := pw.Tree.LongestPrefix(resourceKey, false, nil) + if ownerKey != keyPage { + // Stop walking downwards, someone else owns this resource. + rw.SkipPrefix(ownerKey + "/") + return false, nil + } + } + switch rs := n.(type) { + case *pageState: + relPath := rs.m.pathInfo.BaseRel(pageBundle.m.pathInfo) + rs.m.resourcePath = relPath + } + + return false, nil + } + return false, rw.Walk(sa.ctx) + } + + if err := pw.Walk(sa.ctx); err != nil { + return err + } + + if err := pw.WalkContext.HandleEventsAndHooks(); err != nil { + return err + } + + if !sa.s.conf.C.IsMainSectionsSet() { + var mainSection string + var maxcount int + for section, counter := range sectionPageCount { + if section != "" && counter > maxcount { + mainSection = section + maxcount = counter + } + } + sa.s.conf.C.SetMainSections([]string{mainSection}) + + } + + return nil +} + +func (sa *sitePagesAssembler) applyAggregatesToTaxonomiesAndTerms() error { + walkContext := &doctree.WalkContext[contentNode]{} + + handlePlural := func(key string) error { + var pw *doctree.NodeShiftTreeWalker[contentNode] + pw = &doctree.NodeShiftTreeWalker[contentNode]{ + Tree: sa.s.pageMap.treePages, + Prefix: key, // We also want to include the root taxonomy nodes, so no trailing slash. + LockType: doctree.LockTypeRead, + WalkContext: walkContext, + Handle: func(s string, n contentNode) (bool, error) { + p := n.(*pageState) + + if p.Kind() != kinds.KindTerm && p.Kind() != kinds.KindTaxonomy { + // Already handled. + return false, nil + } + + const eventName = "dates" + + if p.Kind() == kinds.KindTerm { + if !p.s.shouldBuild(p) { + sa.s.pageMap.treePages.Delete(s) + sa.s.pageMap.treeTaxonomyEntries.DeletePrefix(paths.AddTrailingSlash(s)) + } else if err := sa.s.pageMap.treeTaxonomyEntries.WalkPrefix( + doctree.LockTypeRead, + paths.AddTrailingSlash(s), + func(ss string, wn *weightedContentNode) (bool, error) { + // Send the date info up the tree. + pw.WalkContext.SendEvent(&doctree.Event[contentNode]{Source: wn.n, Path: ss, Name: eventName}) + return false, nil + }, + ); err != nil { + return false, err + } + } + + // Send the date info up the tree. + pw.WalkContext.SendEvent(&doctree.Event[contentNode]{Source: n, Path: s, Name: eventName}) + + if p.m.pageConfig.Dates.IsAllDatesZero() { + pw.WalkContext.AddEventListener(eventName, s, func(e *doctree.Event[contentNode]) { + sp, ok := e.Source.(*pageState) + if !ok { + return + } + + p.m.pageConfig.Dates.UpdateDateAndLastmodAndPublishDateIfAfter(sp.m.pageConfig.Dates) + }) + } + + return false, nil + }, + } + + if err := pw.Walk(sa.ctx); err != nil { + return err + } + return nil + } + + for _, viewName := range sa.s.pageMap.cfg.taxonomyConfig.views { + if err := handlePlural(viewName.pluralTreeKey); err != nil { + return err + } + } + + if err := walkContext.HandleEventsAndHooks(); err != nil { + return err + } + + return nil +} + +func (sa *sitePagesAssembler) assembleTerms() error { + if sa.s.pageMap.cfg.taxonomyTermDisabled { + return nil + } + + var ( + pages = sa.s.pageMap.treePages + entries = sa.s.pageMap.treeTaxonomyEntries + views = sa.s.pageMap.cfg.taxonomyConfig.views + ) + + lockType := doctree.LockTypeWrite + w := &doctree.NodeShiftTreeWalker[contentNode]{ + Tree: pages, + LockType: lockType, + Handle: func(s string, n contentNode) (bool, error) { + ps := n.(*pageState) + + if ps.m.noLink() { + return false, nil + } + + for _, viewName := range views { + vals := types.ToStringSlicePreserveString(getParam(ps, viewName.plural, false)) + if vals == nil { + continue + } + + w := getParamToLower(ps, viewName.plural+"_weight") + weight, err := cast.ToIntE(w) + if err != nil { + sa.s.Log.Warnf("Unable to convert taxonomy weight %#v to int for %q", w, n.Path()) + // weight will equal zero, so let the flow continue + } + + for i, v := range vals { + if v == "" { + continue + } + viewTermKey := "/" + viewName.plural + "/" + v + pi := sa.s.Conf.PathParser().Parse(files.ComponentFolderContent, viewTermKey+"/_index.md") + termNode := pages.Get(pi.Base()) + if termNode == nil { + // This means that the term page has been disabled (e.g. a draft). + continue + } + + m := termNode.(*pageState).m + m.term = v + m.singular = viewName.singular + + if s == "" { + s = "/" + } + + key := pi.Base() + s + + entries.Insert(key, &weightedContentNode{ + weight: weight, + n: n, + term: &pageWithOrdinal{pageState: termNode.(*pageState), ordinal: i}, + }) + } + } + + return false, nil + }, + } + + if err := w.Walk(sa.ctx); err != nil { + return err + } + + return nil +} + +func (sa *sitePagesAssembler) assemblePagesStepFinal() error { + if err := sa.assembleResourcesAndSetHome(); err != nil { + return err + } + return nil +} + +func (sa *sitePagesAssembler) assembleResourcesAndSetHome() error { + pagesTree := sa.s.pageMap.treePages + + lockType := doctree.LockTypeWrite + w := &doctree.NodeShiftTreeWalker[contentNode]{ + Tree: pagesTree, + LockType: lockType, + Handle: func(s string, n contentNode) (bool, error) { + ps := n.(*pageState) + + if s == "" { + sa.s.home = ps + } else if ps.s.home == nil { + panic(fmt.Sprintf("[%v] expected home page to be set for %q", sa.s.siteVector, s)) + } + + // This is a little out of place, but is conveniently put here. + // Check if translationKey is set by user. + // This is to support the manual way of setting the translationKey in front matter. + if ps.m.pageConfig.TranslationKey != "" { + sa.s.h.translationKeyPages.Append(ps.m.pageConfig.TranslationKey, ps) + } + + if !sa.s.h.isRebuild() { + if ps.hasRenderableOutput() { + // For multi output pages this will not be complete, but will have to do for now. + sa.s.h.progressReporter.numPagesToRender.Add(1) + } + } + + // Prepare resources for this page. + ps.shiftToOutputFormat(true, 0) + targetPaths := ps.targetPaths() + baseTarget := targetPaths.SubResourceBaseTarget + + err := sa.s.pageMap.forEachResourceInPage( + ps, lockType, + false, + nil, + func(resourceKey string, n contentNode) (bool, error) { + if _, ok := n.(*pageState); ok { + return false, nil + } + rs := n.(*resourceSource) + + relPathOriginal := rs.path.Unnormalized().PathRel(ps.m.pathInfo.Unnormalized()) + relPath := rs.path.BaseRel(ps.m.pathInfo) + + var targetBasePaths []string + if ps.s.Conf.IsMultihost() { + baseTarget = targetPaths.SubResourceBaseLink + // In multihost we need to publish to the lang sub folder. + targetBasePaths = []string{ps.s.GetTargetLanguageBasePath()} // TODO(bep) we don't need this as a slice anymore. + + } + + if rs.rc != nil && rs.rc.Content.IsResourceValue() { + if rs.rc.Name == "" { + rs.rc.Name = relPathOriginal + } + r, err := ps.s.ResourceSpec.NewResourceWrapperFromResourceConfig(rs.rc) + if err != nil { + return false, err + } + rs.r = r + return false, nil + } + + var mt media.Type + if rs.rc != nil { + mt = rs.rc.ContentMediaType + } + + var filename string + if rs.fi != nil { + filename = rs.fi.Meta().Filename + } + + rd := resources.ResourceSourceDescriptor{ + OpenReadSeekCloser: rs.opener, + Path: rs.path, + GroupIdentity: rs.path, + TargetPath: relPathOriginal, // Use the original path for the target path, so the links can be guessed. + TargetBasePaths: targetBasePaths, + BasePathRelPermalink: targetPaths.SubResourceBaseLink, + BasePathTargetPath: baseTarget, + SourceFilenameOrPath: filename, + NameNormalized: relPath, + NameOriginal: relPathOriginal, + MediaType: mt, + LazyPublish: !ps.m.pageConfig.Build.PublishResources, + } + + if rs.rc != nil { + rc := rs.rc + rd.OpenReadSeekCloser = rc.Content.ValueAsOpenReadSeekCloser() + if rc.Name != "" { + rd.NameNormalized = rc.Name + rd.NameOriginal = rc.Name + } + if rc.Title != "" { + rd.Title = rc.Title + } + rd.Params = rc.Params + } + + r, err := ps.s.ResourceSpec.NewResource(rd) + if err != nil { + return false, err + } + + rs.r = r + + return false, nil + }, + ) + + return false, err + }, + } + + return w.Walk(sa.ctx) +} + +func (sa *sitePagesAssembler) assemblePagesStep1() error { + if err := sa.applyAggregates(); err != nil { + return err + } + return nil +} + +func (sa *sitePagesAssembler) assemblePagesStep2() error { + if err := sa.assembleTerms(); err != nil { + return err + } + + if err := sa.applyAggregatesToTaxonomiesAndTerms(); err != nil { + return err + } + + return nil +} + +// No locking. +func (a *allPagesAssembler) createMissingTaxonomies() error { + if a.m.cfg.taxonomyDisabled && a.m.cfg.taxonomyTermDisabled { + return nil + } + + tree := a.m.treePages + + viewLanguages := map[viewName][]int{} + for _, s := range a.h.sitesLanguages { + if s.pageMap.cfg.taxonomyDisabled && s.pageMap.cfg.taxonomyTermDisabled { + continue + } + for _, viewName := range s.pageMap.cfg.taxonomyConfig.views { + viewLanguages[viewName] = append(viewLanguages[viewName], s.siteVector.Language()) + } + } + + for viewName, languages := range viewLanguages { + key := viewName.pluralTreeKey + if a.h.isRebuild() { + if v := tree.Get(key); v != nil { + // Already there. + continue + } + } + + matrixAllForLanguages := sitesmatrix.NewIntSetsBuilder(a.h.Conf.ConfiguredDimensions()).WithLanguageIndices(languages...).WithAllIfNotSet().Build() + + pi := a.h.Conf.PathParser().Parse(files.ComponentFolderContent, key+"/_index.md") + p := &pageMetaSource{ + pathInfo: pi, + sitesMatrixBase: matrixAllForLanguages, + pageConfigSource: &pagemeta.PageConfigEarly{ + Kind: kinds.KindTaxonomy, + }, + } + tree.AppendRaw(key, p) + } + + return nil +} + +// Create the fixed output pages, e.g. sitemap.xml, if not already there. +// TODO2 revise, especially around disabled and config per language/site. +// No locking. +func (a *allPagesAssembler) createMissingStandalonePages() error { + m := a.m + tree := m.treePages + oneSiteStore := (sitesmatrix.Vectors{sitesmatrix.Vector{0, 0, 0}: struct{}{}}).ToVectorStore() + + addStandalone := func(key, kind string, f output.Format) { + if !a.h.Conf.IsKindEnabled(kind) || tree.Has(key) { + return + } + + var sitesMatrixBase sitesmatrix.VectorIterator + + sitesMatrixBase = a.h.Conf.AllSitesMatrix() + if !a.h.Conf.IsMultihost() { + switch kind { + case kinds.KindSitemapIndex, kinds.KindRobotsTXT: + // First site only. + sitesMatrixBase = oneSiteStore + } + } + + p := &pageMetaSource{ + pathInfo: a.h.Conf.PathParser().Parse(files.ComponentFolderContent, key+f.MediaType.FirstSuffix.FullSuffix), + standaloneOutputFormat: f, + sitesMatrixBase: sitesMatrixBase, + sitesMatrixBaseOnly: true, + pageConfigSource: &pagemeta.PageConfigEarly{ + Kind: kind, + }, + } + + tree.InsertRaw(key, p) + } + + addStandalone("/404", kinds.KindStatus404, output.HTTPStatus404HTMLFormat) + + if a.h.Configs.Base.EnableRobotsTXT { + if m.i == 0 || a.h.Conf.IsMultihost() { + addStandalone("/_robots", kinds.KindRobotsTXT, output.RobotsTxtFormat) + } + } + + sitemapEnabled := false + for _, s := range a.h.Sites { + if s.conf.IsKindEnabled(kinds.KindSitemap) { + sitemapEnabled = true + break + } + } + + if sitemapEnabled { + of := output.SitemapFormat + if a.h.Configs.Base.Sitemap.Filename != "" { + of.BaseName = paths.Filename(a.h.Configs.Base.Sitemap.Filename) + } + addStandalone("/_sitemap", kinds.KindSitemap, of) + + skipSitemapIndex := a.h.Conf.IsMultihost() || !(a.h.Conf.DefaultContentLanguageInSubdir() || a.h.Conf.IsMultilingual()) + if !skipSitemapIndex { + of = output.SitemapIndexFormat + if a.h.Configs.Base.Sitemap.Filename != "" { + of.BaseName = paths.Filename(a.h.Configs.Base.Sitemap.Filename) + } + addStandalone("/_sitemapindex", kinds.KindSitemapIndex, of) + } + } + + return nil +} + +// No locking. +func (a *allPagesAssembler) createMissingPages() error { + if err := a.createMissingTaxonomies(); err != nil { + return err + } + + if !a.h.isRebuild() { + if err := a.createMissingStandalonePages(); err != nil { + return err + } + } + return nil +} diff --git a/hugolib/content_map_page_contentnode.go b/hugolib/content_map_page_contentnode.go new file mode 100644 index 000000000..46f59a43f --- /dev/null +++ b/hugolib/content_map_page_contentnode.go @@ -0,0 +1,437 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "iter" + + "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/resource" +) + +var _ contentNode = (*contentNodeSeq)(nil) + +var ( + _ contentNode = (*resourceSource)(nil) + _ contentNodeContentWeightProvider = (*pageState)(nil) + _ contentNodeForSites = (*pageState)(nil) + _ contentNodePage = (*contentNodes)(nil) + _ contentNodePage = (*pageMetaSource)(nil) + _ contentNodePage = (*pageState)(nil) + _ contentNodeSourceEntryIDProvider = (*pageState)(nil) + _ contentNodeSourceEntryIDProvider = (*pageMetaSource)(nil) + _ contentNodeSourceEntryIDProvider = (*resourceSource)(nil) + _ contentNodeLookupContentNode = (*contentNodesMap)(nil) + _ contentNodeLookupContentNode = (contentNodes)(nil) + _ contentNodeSingle = (*pageMetaSource)(nil) + _ contentNodeSingle = (*pageState)(nil) + _ contentNodeSingle = (*resourceSource)(nil) +) + +var ( + _ contentNode = (*contentNodesMap)(nil) + _ contentNodeMap = (*contentNodesMap)(nil) +) + +func (n contentNodesMap) ForEeachInAllDimensions(f func(contentNode) bool) { + for _, nn := range n { + if !f(nn) { + return + } + } +} + +func (n contentNodesMap) Path() string { + return n.sample().Path() +} + +type ( + contentNode interface { + Path() string + contentNodeForEach + } + + contentNodeBuildStateResetter interface { + resetBuildState() + } + + contentNodeSeq2 iter.Seq2[sitesmatrix.Vector, contentNode] + + contentNodeForSite interface { + contentNode + siteVector() sitesmatrix.Vector + } + + contentNodeForEach interface { + // forEeachContentNode iterates over all content nodes. + // calling f with the site vector and the content node. + // If f returns false, the iteration stops. + // It returns false if the iteration was stopped early. + forEeachContentNode(f func(sitesmatrix.Vector, contentNode) bool) bool + } + + contentNodeForSites interface { + contentNode + siteVectors() sitesmatrix.VectorIterator + } + + contentNodePage interface { + contentNode + nodeCategoryPage() // Marker interface. + } + + contentNodeSingle interface { + contentNode + nodeCategorySingle() // Marker interface. + } + + contentNodeLookupContentNode interface { + contentNode + lookupContentNode(v sitesmatrix.Vector) contentNode + } + + contentNodeLookupContentNodes interface { + lookupContentNodes(v sitesmatrix.Vector, fallback bool) iter.Seq[contentNodeForSite] + } + + contentNodeSampleProvider interface { + // sample is used to get a sample contentNode from a collection. + sample() contentNode + } + + contentNodeCascadeProvider interface { + getCascade() *page.PageMatcherParamsConfigs + } + + contentNodeContentWeightProvider interface { + contentWeight() int + } + + contentNodeIsEmptyProvider interface { + isEmpty() bool + } + + contentNodeSourceEntryIDProvider interface { + nodeSourceEntryID() any + } + + contentNodeMap interface { + contentNode + contentNodeLookupContentNode + } +) + +type contentNodeSeq iter.Seq[contentNode] + +func (n contentNodeSeq) Path() string { + return n.sample().Path() +} + +func (n contentNodeSeq) isEmpty() bool { + for range n { + return false + } + return true +} + +func (n contentNodeSeq) forEeachContentNode(f func(sitesmatrix.Vector, contentNode) bool) bool { + for nn := range n { + if !nn.forEeachContentNode(f) { + return false + } + } + return true +} + +func (n contentNodeSeq) sample() contentNode { + for nn := range n { + return nn + } + return nil +} + +func (n contentNodeSeq2) forEeachContentNode(f func(sitesmatrix.Vector, contentNode) bool) bool { + for _, nn := range n { + if !nn.forEeachContentNode(f) { + return false + } + } + return true +} + +type contentNodes []contentNode + +func (n contentNodes) Path() string { + return n.sample().Path() +} + +func (m contentNodes) isEmpty() bool { + return len(m) == 0 +} + +func (n contentNodes) forEeachContentNode(f func(v sitesmatrix.Vector, n contentNode) bool) bool { + for _, nn := range n { + if !nn.forEeachContentNode(f) { + return false + } + } + return true +} + +func (n contentNodes) lookupContentNode(v sitesmatrix.Vector) contentNode { + for _, nn := range n { + if vv := nn.(contentNodeLookupContentNode).lookupContentNode(v); vv != nil { + return vv + } + } + return nil +} + +func (m *contentNodes) nodeCategoryPage() { + // Marker method. +} + +func (n contentNodes) sample() contentNode { + if len(n) == 0 { + panic("pageMetaSourcesSlice is empty") + } + return n[0] +} + +type contentNodesMap map[sitesmatrix.Vector]contentNode + +var cnh helperContentNode + +type helperContentNode struct{} + +func (h helperContentNode) findContentNodeForSiteVector(q sitesmatrix.Vector, fallback bool, candidates contentNodeSeq) contentNodeForSite { + var ( + best contentNodeForSite = nil + bestDistance int + ) + + for n := range candidates { + // The order of candidates is unstable, so we need to compare the matches to + // get stable output. This compare will also make sure that we pick + // language, version and role according to their individual sort order: + // Closer is better, and matches above are better than matches below. + if m := n.(contentNodeLookupContentNodes).lookupContentNodes(q, fallback); m != nil { + for nn := range m { + vec := nn.siteVector() + if q == vec { + // Exact match. + return nn + } + + distance := q.Distance(vec) + + if best == nil { + best = nn + bestDistance = distance + } else { + distanceAbs := absint(distance) + bestDistanceAbs := absint(bestDistance) + if distanceAbs < bestDistanceAbs { + // Closer is better. + best = nn + bestDistance = distance + } else if distanceAbs == bestDistanceAbs && distance > 0 { + // Positive distance is better than negative. + best = nn + bestDistance = distance + } + } + } + } + } + + return best +} + +func (h helperContentNode) markStale(n contentNode) { + n.forEeachContentNode( + func(_ sitesmatrix.Vector, nn contentNode) bool { + resource.MarkStale(nn) + return true + }, + ) +} + +func (h helperContentNode) resetBuildState(n contentNode) { + n.forEeachContentNode( + func(_ sitesmatrix.Vector, nn contentNode) bool { + if nnn, ok := nn.(contentNodeBuildStateResetter); ok { + nnn.resetBuildState() + } + return true + }, + ) +} + +func (h helperContentNode) isBranchNode(n contentNode) bool { + switch nn := n.(type) { + case *pageMetaSource: + return nn.pathInfo.IsBranchBundle() + case *pageState: + return nn.IsNode() + case contentNodeSampleProvider: + return h.isBranchNode(nn.sample()) + default: + return false + } +} + +func (h helperContentNode) isPageNode(n contentNode) bool { + n, _ = h.one(n) + switch n.(type) { + case contentNodePage: + return true + default: + return false + } +} + +func (helperContentNode) one(n contentNode) (nn contentNode, hasMore bool) { + n.forEeachContentNode(func(_ sitesmatrix.Vector, n contentNode) bool { + if nn == nil { + nn = n + return true + } else { + hasMore = true + return false + } + }) + return +} + +func (h helperContentNode) GetIdentity(n contentNode) identity.Identity { + switch nn := n.(type) { + case identity.IdentityProvider: + return nn.GetIdentity() + case contentNodeSampleProvider: + return h.GetIdentity(nn.sample()) + default: + return identity.Anonymous + } +} + +func (h helperContentNode) PathInfo(n contentNode) *paths.Path { + switch nn := n.(type) { + case interface{ PathInfo() *paths.Path }: + return nn.PathInfo() + case contentNodeSampleProvider: + return h.PathInfo(nn.sample()) + default: + return nil + } +} + +func (h helperContentNode) toForEachIdentityProvider(n contentNode) identity.ForEeachIdentityProvider { + return identity.ForEeachIdentityProviderFunc( + func(cb func(id identity.Identity) bool) bool { + return n.forEeachContentNode( + func(vec sitesmatrix.Vector, nn contentNode) bool { + return cb(h.GetIdentity(n)) + }, + ) + }) +} + +type weightedContentNode struct { + n contentNode + weight int + term *pageWithOrdinal +} + +func (n contentNodesMap) isEmpty() bool { + return len(n) == 0 +} + +func absint(i int) int { + if i < 0 { + return -i + } + return i +} + +func contentNodeToContentNodesPage(n contentNode) (contentNodesMap, bool) { + switch v := n.(type) { + case contentNodesMap: + return v, false + case *pageState: + return contentNodesMap{v.s.siteVector: v}, true + default: + panic(fmt.Sprintf("contentNodeToContentNodesPage: unexpected type %T", n)) + } +} + +func contentNodeToSeq(n contentNodeForEach) contentNodeSeq { + if nn, ok := n.(contentNodeSeq); ok { + return nn + } + return func(yield func(contentNode) bool) { + n.forEeachContentNode(func(_ sitesmatrix.Vector, nn contentNode) bool { + return yield(nn) + }) + } +} + +func contentNodeToSeq2(n contentNodeForEach) contentNodeSeq2 { + if nn, ok := n.(contentNodeSeq2); ok { + return nn + } + return func(yield func(sitesmatrix.Vector, contentNode) bool) { + n.forEeachContentNode(func(vec sitesmatrix.Vector, nn contentNode) bool { + return yield(vec, nn) + }) + } +} + +func (ps contentNodesMap) forEeachContentNode(f func(v sitesmatrix.Vector, n contentNode) bool) bool { + for vec, nn := range ps { + if !f(vec, nn) { + return false + } + } + return true +} + +func (n contentNodesMap) lookupContentNode(v sitesmatrix.Vector) contentNode { + if vv, ok := n[v]; ok { + return vv + } + return nil +} + +func (n contentNodesMap) sample() contentNode { + for _, nn := range n { + return nn + } + return nil +} + +func (n contentNodesMap) siteVectors() sitesmatrix.VectorIterator { + return sitesmatrix.VectorIteratorFunc(func(yield func(v sitesmatrix.Vector) bool) bool { + for k := range n { + if !yield(k) { + return false + } + } + return true + }) +} diff --git a/hugolib/content_map_page_contentnodeshifter.go b/hugolib/content_map_page_contentnodeshifter.go new file mode 100644 index 000000000..b9d183ddd --- /dev/null +++ b/hugolib/content_map_page_contentnodeshifter.go @@ -0,0 +1,162 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" + "github.com/gohugoio/hugo/resources/resource" +) + +type contentNodeShifter struct { + conf config.AllProvider // Used for logging/debugging. +} + +func (s *contentNodeShifter) Delete(n contentNode, vec sitesmatrix.Vector) (contentNode, bool, bool) { + switch v := n.(type) { + case contentNodesMap: + deleted, wasDeleted := v[vec] + if wasDeleted { + delete(v, vec) + resource.MarkStale(deleted) + } + return deleted, wasDeleted, len(v) == 0 + case contentNodeForSite: + if v.siteVector() != vec { + return nil, false, false + } + resource.MarkStale(v) + return v, true, true + default: + v = v.(contentNodeSingle) // Ensure single node. + resource.MarkStale(v) + return v, true, true + + } +} + +func (s *contentNodeShifter) DeleteFunc(v contentNode, f func(n contentNode) bool) bool { + switch ss := v.(type) { + case contentNodeSingle: + if f(ss) { + resource.MarkStale(ss) + return true + } + return false + case contentNodes: + for i, n := range ss { + if f(n) { + resource.MarkStale(n) + ss = append(ss[:i], ss[i+1:]...) + } + } + return len(ss) == 0 + case contentNodesMap: + for k, n := range ss { + if f(n) { + resource.MarkStale(n) + delete(ss, k) + } + } + return len(ss) == 0 + default: + panic(fmt.Sprintf("DeleteFunc: unknown type %T", v)) + } +} + +func (s *contentNodeShifter) ForEeachInAllDimensions(n contentNode, f func(contentNode) bool) { + if n == nil { + return + } + if v, ok := n.(interface { + // Implemented by all the list nodes. + ForEeachInAllDimensions(f func(contentNode) bool) + }); ok { + v.ForEeachInAllDimensions(f) + return + } + f(n) +} + +func (s *contentNodeShifter) ForEeachInDimension(n contentNode, vec sitesmatrix.Vector, d int, f func(contentNode) bool) { +LOOP1: + for vec2, v := range contentNodeToSeq2(n) { + for i, v := range vec2 { + if i != d && v != vec[i] { + continue LOOP1 + } + } + if !f(v) { + return + } + } +} + +func (s *contentNodeShifter) Insert(old, new contentNode) (contentNode, contentNode, bool) { + new = new.(contentNodeSingle) // Ensure single node. + + switch vv := old.(type) { + case contentNodeSingle: + return contentNodes{vv, new}, old, false + case contentNodes: + s := make(contentNodes, 0, len(vv)+1) + s = append(s, vv...) + s = append(s, new) + return s, old, false + case contentNodesMap: + switch new := new.(type) { + case contentNodeForSite: + oldp := vv[new.siteVector()] + updated := oldp != new + if updated { + resource.MarkStale(oldp) + } + vv[new.siteVector()] = new + return vv, oldp, updated + default: + s := make(contentNodes, 0, len(vv)+1) + for _, v := range vv { + s = append(s, v) + } + s = append(s, new) + return s, vv, false + } + default: + panic(fmt.Sprintf("Insert: unknown type %T", old)) + } +} + +func (s *contentNodeShifter) Shift(n contentNode, siteVector sitesmatrix.Vector, fallback bool) (contentNode, bool) { + switch v := n.(type) { + case contentNodeLookupContentNode: + if vv := v.lookupContentNode(siteVector); vv != nil { + return vv, true + } + default: + panic(fmt.Sprintf("Shift: unknown type %T", n)) + } + + if !fallback { + // Done + return nil, false + } + + if vvv := cnh.findContentNodeForSiteVector(siteVector, fallback, contentNodeToSeq(n)); vvv != nil { + return vvv, true + } + + return nil, false +} diff --git a/hugolib/content_map_test.go b/hugolib/content_map_test.go index 9a22c3c3d..971be7b55 100644 --- a/hugolib/content_map_test.go +++ b/hugolib/content_map_test.go @@ -15,13 +15,12 @@ package hugolib import ( "fmt" - "path/filepath" "strings" "sync" "testing" qt "github.com/frankban/quicktest" - "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" ) func TestContentMapSite(t *testing.T) { @@ -200,7 +199,10 @@ Home: {{ .Title }}| } // Issue #11840 +// Resource language fallback should be the closest language going up. func TestBundleResourceLanguageBestMatch(t *testing.T) { + t.Parallel() + files := ` -- hugo.toml -- defaultContentLanguage = "fr" @@ -215,7 +217,7 @@ weight = 3 -- layouts/index.html -- {{ $bundle := site.GetPage "bundle" }} {{ $r := $bundle.Resources.GetMatch "*.txt" }} -{{ .Language.Lang }}: {{ $r.RelPermalink }}|{{ $r.Content }} +{{ .Language.Lang }}: {{ with $r }}{{ .RelPermalink }}|{{ .Content }}{{ end}} -- content/bundle/index.fr.md -- --- title: "Bundle Fr" @@ -234,61 +236,13 @@ Data fr Data en ` - b := Test(t, files) - - b.AssertFileContent("public/fr/index.html", "fr: /fr/bundle/data.fr.txt|Data fr") - b.AssertFileContent("public/en/index.html", "en: /en/bundle/data.en.txt|Data en") - b.AssertFileContent("public/de/index.html", "de: /fr/bundle/data.fr.txt|Data fr") -} - -func TestBundleMultipleContentPageWithSamePath(t *testing.T) { - t.Parallel() - - files := ` --- hugo.toml -- -printPathWarnings = true --- layouts/all.html -- -All. --- content/bundle/index.md -- ---- -title: "Bundle md" -foo: md ---- --- content/bundle/index.html -- ---- -title: "Bundle html" -foo: html ---- --- content/bundle/data.txt -- -Data. --- content/p1.md -- ---- -title: "P1 md" -foo: md ---- --- content/p1.html -- ---- -title: "P1 html" -foo: html ---- --- layouts/index.html -- -{{ $bundle := site.GetPage "bundle" }} -Bundle: {{ $bundle.Title }}|{{ $bundle.Params.foo }}|{{ $bundle.File.Filename }}| -{{ $p1 := site.GetPage "p1" }} -P1: {{ $p1.Title }}|{{ $p1.Params.foo }}|{{ $p1.File.Filename }}| -` - - for range 3 { - b := Test(t, files, TestOptInfo()) + for range 5 { + b := Test(t, files) - b.AssertLogContains("INFO Duplicate content path: \"/p1\"") + b.AssertFileContent("public/fr/index.html", "fr: /fr/bundle/data.fr.txt|Data fr") + b.AssertFileContent("public/en/index.html", "en: /en/bundle/data.en.txt|Data en") - // There's multiple content files sharing the same logical path and language. - // This is a little arbitrary, but we have to pick one and prefer the Markdown version. - b.AssertFileContent("public/index.html", - filepath.FromSlash("Bundle: Bundle md|md|/content/bundle/index.md|"), - filepath.FromSlash("P1: P1 md|md|/content/p1.md|"), - ) + b.AssertFileContent("public/de/index.html", "de: /fr/bundle/data.fr.txt|Data fr") } } @@ -361,7 +315,6 @@ p1-foo.txt ` b := Test(t, files) - b.Build() b.AssertFileExists("public/s1/index.html", true) b.AssertFileExists("public/s1/foo.txt", true) @@ -443,7 +396,7 @@ func TestContentTreeReverseIndex(t *testing.T) { c := qt.New(t) pageReverseIndex := newContentTreeTreverseIndex( - func(get func(key any) (contentNodeI, bool), set func(key any, val contentNodeI)) { + func(get func(key any) (contentNode, bool), set func(key any, val contentNode)) { for i := range 10 { key := fmt.Sprint(i) set(key, &testContentNode{key: key}) @@ -467,7 +420,7 @@ func TestContentTreeReverseIndexPara(t *testing.T) { for range 10 { pageReverseIndex := newContentTreeTreverseIndex( - func(get func(key any) (contentNodeI, bool), set func(key any, val contentNodeI)) { + func(get func(key any) (contentNode, bool), set func(key any, val contentNode)) { for i := range 10 { key := fmt.Sprint(i) set(key, &testContentNode{key: key}) @@ -489,26 +442,12 @@ type testContentNode struct { key string } -func (n *testContentNode) GetIdentity() identity.Identity { - return identity.StringIdentity(n.key) -} - -func (n *testContentNode) ForEeachIdentity(cb func(id identity.Identity) bool) bool { - panic("not supported") -} - func (n *testContentNode) Path() string { return n.key } -func (n *testContentNode) isContentNodeBranch() bool { - return false -} - -func (n *testContentNode) resetBuildState() { -} - -func (n *testContentNode) MarkStale() { +func (n *testContentNode) forEeachContentNode(f func(v sitesmatrix.Vector, n contentNode) bool) bool { + panic("not supported") } // Issue 12274. diff --git a/hugolib/dates_test.go b/hugolib/dates_test.go index 1a88cfd7d..2bdc47892 100644 --- a/hugolib/dates_test.go +++ b/hugolib/dates_test.go @@ -309,9 +309,9 @@ tags: [t1] b := Test(t, files) b.AssertFileContent("public/s1/index.html", ` - Date: 2099-05-06 + Date: 2024-04-03 PublishDate: 2024-04-03 - Lastmod: 2099-05-06 + Lastmod: 2024-04-04 `) b.AssertFileContent("public/tags/index.html", ` @@ -332,9 +332,9 @@ tags: [t1] b = Test(t, files) b.AssertFileContent("public/s1/index.html", ` - Date: 2099-05-06 + Date: 2024-04-03 PublishDate: 2024-04-04 - Lastmod: 2099-05-06 + Lastmod: 2024-04-03 `) b.AssertFileContent("public/tags/index.html", ` diff --git a/hugolib/doctree/dimensions.go b/hugolib/doctree/dimensions.go deleted file mode 100644 index bcc3cae00..000000000 --- a/hugolib/doctree/dimensions.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2024 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package doctree - -const ( - // Language is currently the only dimension in the Hugo build matrix. - DimensionLanguage DimensionFlag = 1 << iota -) - -// Dimension is a row in the Hugo build matrix which currently has one value: language. -type Dimension [1]int - -// DimensionFlag is a flag in the Hugo build matrix. -type DimensionFlag byte - -// Has returns whether the given flag is set. -func (d DimensionFlag) Has(o DimensionFlag) bool { - return d&o == o -} - -// Set sets the given flag. -func (d DimensionFlag) Set(o DimensionFlag) DimensionFlag { - return d | o -} - -// Index returns this flag's index in the Dimensions array. -func (d DimensionFlag) Index() int { - if d == 0 { - panic("dimension flag not set") - } - return int(d - 1) -} diff --git a/hugolib/doctree/dimensions_test.go b/hugolib/doctree/dimensions_test.go deleted file mode 100644 index 598f22a2d..000000000 --- a/hugolib/doctree/dimensions_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2024 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package doctree - -import ( - "testing" - - qt "github.com/frankban/quicktest" -) - -func TestDimensionFlag(t *testing.T) { - c := qt.New(t) - - var zero DimensionFlag - var d DimensionFlag - var o DimensionFlag = 1 - var p DimensionFlag = 12 - - c.Assert(d.Has(o), qt.Equals, false) - d = d.Set(o) - c.Assert(d.Has(o), qt.Equals, true) - c.Assert(d.Has(d), qt.Equals, true) - c.Assert(func() { zero.Index() }, qt.PanicMatches, "dimension flag not set") - c.Assert(DimensionLanguage.Index(), qt.Equals, 0) - c.Assert(p.Index(), qt.Equals, 11) -} diff --git a/hugolib/doctree/nodeshiftree_test.go b/hugolib/doctree/nodeshiftree_test.go index 0f4fb6f68..946e6d187 100644 --- a/hugolib/doctree/nodeshiftree_test.go +++ b/hugolib/doctree/nodeshiftree_test.go @@ -24,6 +24,7 @@ import ( qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/common/para" "github.com/gohugoio/hugo/hugolib/doctree" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "github.com/google/go-cmp/cmp" ) @@ -47,17 +48,17 @@ func TestTree(t *testing.T) { ) a := &testValue{ID: "/a"} - zeroZero.InsertIntoValuesDimension("/a", a) + zeroZero.Insert("/a", a) ab := &testValue{ID: "/a/b"} - zeroZero.InsertIntoValuesDimension("/a/b", ab) + zeroZero.Insert("/a/b", ab) c.Assert(zeroZero.Get("/a"), eq, &testValue{ID: "/a", Lang: 0}) - s, v := zeroZero.LongestPrefix("/a/b/c", true, nil) + s, v := zeroZero.LongestPrefix("/a/b/c", false, nil) c.Assert(v, eq, ab) c.Assert(s, eq, "/a/b") // Change language. - oneZero := zeroZero.Increment(0) + oneZero := zeroZero.Shape(sitesmatrix.Vector{1, 0, 0}) c.Assert(zeroZero.Get("/a"), eq, &testValue{ID: "/a", Lang: 0}) c.Assert(oneZero.Get("/a"), eq, &testValue{ID: "/a", Lang: 1}) } @@ -71,12 +72,12 @@ func TestTreeData(t *testing.T) { }, ) - tree.InsertIntoValuesDimension("", &testValue{ID: "HOME"}) - tree.InsertIntoValuesDimension("/a", &testValue{ID: "/a"}) - tree.InsertIntoValuesDimension("/a/b", &testValue{ID: "/a/b"}) - tree.InsertIntoValuesDimension("/b", &testValue{ID: "/b"}) - tree.InsertIntoValuesDimension("/b/c", &testValue{ID: "/b/c"}) - tree.InsertIntoValuesDimension("/b/c/d", &testValue{ID: "/b/c/d"}) + tree.Insert("", &testValue{ID: "HOME"}) + tree.Insert("/a", &testValue{ID: "/a"}) + tree.Insert("/a/b", &testValue{ID: "/a/b"}) + tree.Insert("/b", &testValue{ID: "/b"}) + tree.Insert("/b/c", &testValue{ID: "/b/c"}) + tree.Insert("/b/c/d", &testValue{ID: "/b/c/d"}) var values []string @@ -85,7 +86,7 @@ func TestTreeData(t *testing.T) { w := &doctree.NodeShiftTreeWalker[*testValue]{ Tree: tree, WalkContext: ctx, - Handle: func(s string, t *testValue, match doctree.DimensionFlag) (bool, error) { + Handle: func(s string, t *testValue) (bool, error) { ctx.Data().Insert(s, map[string]any{ "id": t.ID, }) @@ -112,22 +113,22 @@ func TestTreeEvents(t *testing.T) { }, ) - tree.InsertIntoValuesDimension("/a", &testValue{ID: "/a", Weight: 2, IsBranch: true}) - tree.InsertIntoValuesDimension("/a/p1", &testValue{ID: "/a/p1", Weight: 5}) - tree.InsertIntoValuesDimension("/a/p", &testValue{ID: "/a/p2", Weight: 6}) - tree.InsertIntoValuesDimension("/a/s1", &testValue{ID: "/a/s1", Weight: 5, IsBranch: true}) - tree.InsertIntoValuesDimension("/a/s1/p1", &testValue{ID: "/a/s1/p1", Weight: 8}) - tree.InsertIntoValuesDimension("/a/s1/p1", &testValue{ID: "/a/s1/p2", Weight: 9}) - tree.InsertIntoValuesDimension("/a/s1/s2", &testValue{ID: "/a/s1/s2", Weight: 6, IsBranch: true}) - tree.InsertIntoValuesDimension("/a/s1/s2/p1", &testValue{ID: "/a/s1/s2/p1", Weight: 8}) - tree.InsertIntoValuesDimension("/a/s1/s2/p2", &testValue{ID: "/a/s1/s2/p2", Weight: 7}) + tree.Insert("/a", &testValue{ID: "/a", Weight: 2, IsBranch: true}) + tree.Insert("/a/p1", &testValue{ID: "/a/p1", Weight: 5}) + tree.Insert("/a/p", &testValue{ID: "/a/p2", Weight: 6}) + tree.Insert("/a/s1", &testValue{ID: "/a/s1", Weight: 5, IsBranch: true}) + tree.Insert("/a/s1/p1", &testValue{ID: "/a/s1/p1", Weight: 8}) + tree.Insert("/a/s1/p1", &testValue{ID: "/a/s1/p2", Weight: 9}) + tree.Insert("/a/s1/s2", &testValue{ID: "/a/s1/s2", Weight: 6, IsBranch: true}) + tree.Insert("/a/s1/s2/p1", &testValue{ID: "/a/s1/s2/p1", Weight: 8}) + tree.Insert("/a/s1/s2/p2", &testValue{ID: "/a/s1/s2/p2", Weight: 7}) w := &doctree.NodeShiftTreeWalker[*testValue]{ Tree: tree, WalkContext: &doctree.WalkContext[*testValue]{}, } - w.Handle = func(s string, t *testValue, match doctree.DimensionFlag) (bool, error) { + w.Handle = func(s string, t *testValue) (bool, error) { if t.IsBranch { w.WalkContext.AddEventListener("weight", s, func(e *doctree.Event[*testValue]) { if e.Source.Weight > t.Weight { @@ -165,19 +166,19 @@ func TestTreeInsert(t *testing.T) { ) a := &testValue{ID: "/a"} - tree.InsertIntoValuesDimension("/a", a) + tree.Insert("/a", a) ab := &testValue{ID: "/a/b"} - tree.InsertIntoValuesDimension("/a/b", ab) + tree.Insert("/a/b", ab) c.Assert(tree.Get("/a"), eq, &testValue{ID: "/a", Lang: 0}) c.Assert(tree.Get("/notfound"), qt.IsNil) ab2 := &testValue{ID: "/a/b", Lang: 0} - v, _, ok := tree.InsertIntoValuesDimension("/a/b", ab2) + v, _, ok := tree.Insert("/a/b", ab2) c.Assert(ok, qt.IsTrue) c.Assert(v, qt.DeepEquals, ab2) - tree1 := tree.Increment(0) + tree1 := tree.Shape(sitesmatrix.Vector{1, 0, 0}) c.Assert(tree1.Get("/a/b"), qt.DeepEquals, &testValue{ID: "/a/b", Lang: 1}) } @@ -199,13 +200,13 @@ func TestTreePara(t *testing.T) { a := &testValue{ID: "/a"} lock := tree.Lock(true) defer lock() - tree.InsertIntoValuesDimension("/a", a) + tree.Insert("/a", a) ab := &testValue{ID: "/a/b"} - tree.InsertIntoValuesDimension("/a/b", ab) + tree.Insert("/a/b", ab) key := fmt.Sprintf("/a/b/c/%d", i) val := &testValue{ID: key} - tree.InsertIntoValuesDimension(key, val) + tree.Insert(key, val) c.Assert(tree.Get(key), eq, val) // s, _ := tree.LongestPrefix(key, nil) // c.Assert(s, eq, "/a/b") @@ -232,38 +233,57 @@ type testShifter struct { echo bool } -func (s *testShifter) ForEeachInDimension(n *testValue, d int, f func(n *testValue) bool) { - if d != doctree.DimensionLanguage.Index() { +func (s *testShifter) ForEeachInDimension(n *testValue, v sitesmatrix.Vector, d int, f func(n *testValue) bool) { + if d != sitesmatrix.Language { panic("not implemented") } f(n) } -func (s *testShifter) Insert(old, new *testValue) (*testValue, *testValue, bool) { - return new, old, true +func (s *testShifter) ForEeachInAllDimensions(n *testValue, f func(*testValue) bool) { + if n == nil { + return + } + if !f(n) { + return + } + for _, v := range s.All(n) { + if v != n && !f(v) { + return + } + } } -func (s *testShifter) InsertInto(old, new *testValue, dimension doctree.Dimension) (*testValue, *testValue, bool) { +func (s *testShifter) Insert(old, new *testValue) (*testValue, *testValue, bool) { return new, old, true } -func (s *testShifter) Delete(n *testValue, dimension doctree.Dimension) (*testValue, bool, bool) { +func (s *testShifter) Delete(n *testValue, dimension sitesmatrix.Vector) (*testValue, bool, bool) { return nil, true, true } -func (s *testShifter) Shift(n *testValue, dimension doctree.Dimension, exact bool) (*testValue, bool, doctree.DimensionFlag) { +func (s *testShifter) DeleteFunc(v *testValue, f func(*testValue) bool) bool { + return f(v) +} + +func (s *testShifter) Shift(n *testValue, dimension sitesmatrix.Vector, fallback bool) (v *testValue, ok bool) { + ok = true + v = n if s.echo { - return n, true, doctree.DimensionLanguage + return } if n.NoCopy { if n.Lang == dimension[0] { - return n, true, doctree.DimensionLanguage + return } - return nil, false, doctree.DimensionLanguage + v = nil + ok = false + return } c := *n c.Lang = dimension[0] - return &c, true, doctree.DimensionLanguage + v = &c + return } func (s *testShifter) All(n *testValue) []*testValue { @@ -291,7 +311,7 @@ func BenchmarkTreeInsert(b *testing.B) { for i := range numElements { lang := rand.Intn(2) - tree.InsertIntoValuesDimension(fmt.Sprintf("/%d", i), &testValue{ID: fmt.Sprintf("/%d", i), Lang: lang, Weight: i, NoCopy: true}) + tree.Insert(fmt.Sprintf("/%d", i), &testValue{ID: fmt.Sprintf("/%d", i), Lang: lang, Weight: i, NoCopy: true}) } } } @@ -325,13 +345,13 @@ func BenchmarkWalk(b *testing.B) { for i := range numElements { lang := rand.Intn(2) - tree.InsertIntoValuesDimension(fmt.Sprintf("/%d", i), &testValue{ID: fmt.Sprintf("/%d", i), Lang: lang, Weight: i, NoCopy: true}) + tree.Insert(fmt.Sprintf("/%d", i), &testValue{ID: fmt.Sprintf("/%d", i), Lang: lang, Weight: i, NoCopy: true}) } return tree } - handle := func(s string, t *testValue, match doctree.DimensionFlag) (bool, error) { + handle := func(s string, t *testValue) (bool, error) { return false, nil } @@ -357,7 +377,7 @@ func BenchmarkWalk(b *testing.B) { for i := 0; i < b.N; i++ { for d1 := range 1 { for d2 := range 2 { - tree := base.Shape(d1, d2) + tree := base.Shape(sitesmatrix.Vector{d1, d2, 0}) w := &doctree.NodeShiftTreeWalker[*testValue]{ Tree: tree, Handle: handle, diff --git a/hugolib/doctree/nodeshifttree.go b/hugolib/doctree/nodeshifttree.go index 298b24d1b..aa1b6a723 100644 --- a/hugolib/doctree/nodeshifttree.go +++ b/hugolib/doctree/nodeshifttree.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Hugo Authors. All rights reserved. +// Copyright 2025 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,20 +21,27 @@ import ( "sync" radix "github.com/armon/go-radix" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "github.com/gohugoio/hugo/resources/resource" ) type ( Config[T any] struct { // Shifter handles tree transformations. - Shifter Shifter[T] + Shifter Shifter[T] + TransformerRaw Transformer[T] } // Shifter handles tree transformations. Shifter[T any] interface { - // ForEeachInDimension will call the given function for each value in the given dimension. - // If the function returns true, the walk will stop. - ForEeachInDimension(n T, d int, f func(T) bool) + // ForEeachInDimension will call the given function for each value in the given dimension d. + // This is typicall used to e.g. walk all language translations of a given node. + // If the function returns false, the walk will stop. + ForEeachInDimension(n T, vec sitesmatrix.Vector, d int, f func(T) bool) + + // ForEeachInAllDimensions will call the given function for each value in all dimensions. + // If the function returns false, the walk will stop. + ForEeachInAllDimensions(n T, f func(T) bool) // Insert inserts new into the tree into the dimension it provides. // It may replace old. @@ -42,19 +49,23 @@ type ( // and a bool indicating if an existing record is updated. Insert(old, new T) (T, T, bool) - // Insert inserts new into the given dimension. - // It may replace old. - // It returns the updated and existing T - // and a bool indicating if an existing record is updated. - InsertInto(old, new T, dimension Dimension) (T, T, bool) - // Delete deletes T from the given dimension and returns the deleted T and whether the dimension was deleted and if it's empty after the delete. - Delete(v T, dimension Dimension) (T, bool, bool) + Delete(v T, dimension sitesmatrix.Vector) (T, bool, bool) + + // DeleteFunc deletes nodes in v from the tree where the given function returns true. + // It returns true if it's empty after the delete. + DeleteFunc(v T, f func(n T) bool) bool + + // Shift shifts v into the given dimension, + // if fallback is true, it will fall back a fallback match if found. + // It returns the shifted T and a bool indicating if the shift was successful. + Shift(v T, dimension sitesmatrix.Vector, fallback bool) (T, bool) + } - // Shift shifts T into the given dimension - // and returns the shifted T and a bool indicating if the shift was successful and - // how accurate a match T is according to its dimensions. - Shift(v T, dimension Dimension, exact bool) (T, bool, DimensionFlag) + Transformer[T any] interface { + // Append appends vs to t and returns the updated or replaced T and a bool indicating if T was replaced. + // Note that t may be the zero value and should be ignored. + Append(t T, ts ...T) (T, bool) } ) @@ -64,9 +75,10 @@ type ( type NodeShiftTree[T any] struct { tree *radix.Tree - // E.g. [language, role]. - dims Dimension - shifter Shifter[T] + // [language, version, role]. + siteVector sitesmatrix.Vector + shifter Shifter[T] + transformerRaw Transformer[T] mu *sync.RWMutex } @@ -77,30 +89,57 @@ func New[T any](cfg Config[T]) *NodeShiftTree[T] { } return &NodeShiftTree[T]{ - mu: &sync.RWMutex{}, - shifter: cfg.Shifter, - tree: radix.New(), + mu: &sync.RWMutex{}, + + shifter: cfg.Shifter, + transformerRaw: cfg.TransformerRaw, + tree: radix.New(), } } +// SiteVector returns the site vector of the current dimension. +func (r *NodeShiftTree[T]) SiteVector() sitesmatrix.Vector { + return r.siteVector +} + +// Delete deletes the node at the given key in the current dimension. func (r *NodeShiftTree[T]) Delete(key string) (T, bool) { return r.delete(key) } -func (r *NodeShiftTree[T]) DeleteRaw(key string) { - r.delete(key) -} +// DeleteFuncRaw deletes nodes in the tree at the given key where the given function returns true. +// It will delete nodes in all dimensions. +func (r *NodeShiftTree[T]) DeleteFuncRaw(key string, f func(T) bool) (T, int) { + var count int + var lastDeleted T -func (r *NodeShiftTree[T]) DeleteAll(key string) { - r.tree.WalkPrefix(key, func(key string, value any) bool { - v, ok := r.tree.Delete(key) - if ok { - resource.MarkStale(v) + v, ok := r.tree.Get(key) + if !ok { + return lastDeleted, count + } + + isEmpty := r.shifter.DeleteFunc(v.(T), func(n T) bool { + if f(n) { + count++ + lastDeleted = n + return true } return false }) + + if isEmpty { + r.tree.Delete(key) + } + + return lastDeleted, count } +// DeleteRaw deletes the node at the given key without any shifting or transformation. +func (r *NodeShiftTree[T]) DeleteRaw(key string) { + r.tree.Delete(key) +} + +// DeletePrefix deletes all nodes with the given prefix in the current dimension. func (r *NodeShiftTree[T]) DeletePrefix(prefix string) int { count := 0 var keys []string @@ -121,7 +160,7 @@ func (r *NodeShiftTree[T]) delete(key string) (T, bool) { var deleted T if v, ok := r.tree.Get(key); ok { var isEmpty bool - deleted, wasDeleted, isEmpty = r.shifter.Delete(v.(T), r.dims) + deleted, wasDeleted, isEmpty = r.shifter.Delete(v.(T), r.siteVector) if isEmpty { r.tree.Delete(key) } @@ -129,7 +168,7 @@ func (r *NodeShiftTree[T]) delete(key string) (T, bool) { return deleted, wasDeleted } -func (t *NodeShiftTree[T]) DeletePrefixAll(prefix string) int { +func (t *NodeShiftTree[T]) DeletePrefixRaw(prefix string) int { count := 0 t.tree.WalkPrefix(prefix, func(key string, value any) bool { @@ -143,28 +182,10 @@ func (t *NodeShiftTree[T]) DeletePrefixAll(prefix string) int { return count } -// Increment the value of dimension d by 1. -func (t *NodeShiftTree[T]) Increment(d int) *NodeShiftTree[T] { - return t.Shape(d, t.dims[d]+1) -} - -func (r *NodeShiftTree[T]) InsertIntoCurrentDimension(s string, v T) (T, T, bool) { - s = mustValidateKey(cleanKey(s)) - var ( - updated bool - existing T - ) - if vv, ok := r.tree.Get(s); ok { - v, existing, updated = r.shifter.InsertInto(vv.(T), v, r.dims) - } - r.tree.Insert(s, v) - return v, existing, updated -} - -// InsertIntoValuesDimension inserts v into the tree at the given key and the +// Insert inserts v into the tree at the given key and the // dimension defined by the value. // It returns the updated and existing T and a bool indicating if an existing record is updated. -func (r *NodeShiftTree[T]) InsertIntoValuesDimension(s string, v T) (T, T, bool) { +func (r *NodeShiftTree[T]) Insert(s string, v T) (T, T, bool) { s = mustValidateKey(cleanKey(s)) var ( updated bool @@ -173,21 +194,35 @@ func (r *NodeShiftTree[T]) InsertIntoValuesDimension(s string, v T) (T, T, bool) if vv, ok := r.tree.Get(s); ok { v, existing, updated = r.shifter.Insert(vv.(T), v) } - r.tree.Insert(s, v) + r.insert(s, v) return v, existing, updated } +func (r *NodeShiftTree[T]) insert(s string, v any) (any, bool) { + if v == nil { + panic("nil value") + } + n, updated := r.tree.Insert(s, v) + + return n, updated +} + +// InsertRaw inserts v into the tree at the given key without any shifting or transformation. +func (r *NodeShiftTree[T]) InsertRaw(s string, v any) (any, bool) { + return r.insert(s, v) +} + func (r *NodeShiftTree[T]) InsertRawWithLock(s string, v any) (any, bool) { r.mu.Lock() defer r.mu.Unlock() - return r.tree.Insert(s, v) + return r.insert(s, v) } // It returns the updated and existing T and a bool indicating if an existing record is updated. -func (r *NodeShiftTree[T]) InsertIntoValuesDimensionWithLock(s string, v T) (T, T, bool) { +func (r *NodeShiftTree[T]) InsertWithLock(s string, v T) (T, T, bool) { r.mu.Lock() defer r.mu.Unlock() - return r.InsertIntoValuesDimension(s, v) + return r.Insert(s, v) } func (t *NodeShiftTree[T]) Len() int { @@ -203,31 +238,29 @@ func (t *NodeShiftTree[T]) CanLock() bool { } // Lock locks the data store for read or read/write access until commit is invoked. -// Note that Root is not thread-safe outside of this transaction construct. +// Note that NodeShiftTree is not thread-safe outside of this transaction construct. func (t *NodeShiftTree[T]) Lock(writable bool) (commit func()) { if writable { t.mu.Lock() } else { t.mu.RLock() } - return func() { - if writable { - t.mu.Unlock() - } else { - t.mu.RUnlock() - } + + if writable { + return t.mu.Unlock } + return t.mu.RUnlock } // LongestPrefix finds the longest prefix of s that exists in the tree that also matches the predicate (if set). // Set exact to true to only match exact in the current dimension (e.g. language). -func (r *NodeShiftTree[T]) LongestPrefix(s string, exact bool, predicate func(v T) bool) (string, T) { +func (r *NodeShiftTree[T]) LongestPrefix(s string, fallback bool, predicate func(v T) bool) (string, T) { for { longestPrefix, v, found := r.tree.LongestPrefix(s) if found { - if t, ok, _ := r.shift(v.(T), exact); ok && (predicate == nil || predicate(t)) { - return longestPrefix, t + if v, ok := r.shift(v.(T), fallback); ok && (predicate == nil || predicate(v)) { + return longestPrefix, v } } @@ -242,12 +275,23 @@ func (r *NodeShiftTree[T]) LongestPrefix(s string, exact bool, predicate func(v } } -// LongestPrefixAll returns the longest prefix considering all tree dimensions. -func (r *NodeShiftTree[T]) LongestPrefixAll(s string) (string, bool) { +// LongestPrefiValueRaw returns the longest prefix and value considering all dimensions +func (r *NodeShiftTree[T]) LongestPrefiValueRaw(s string) (string, T) { + s, v, found := r.tree.LongestPrefix(s) + if !found { + var t T + return s, t + } + return s, v.(T) +} + +// LongestPrefixRaw returns the longest prefix considering all dimensions. +func (r *NodeShiftTree[T]) LongestPrefixRaw(s string) (string, bool) { s, _, found := r.tree.LongestPrefix(s) return s, found } +// GetRaw returns the raw value at the given key without any shifting or transformation. func (r *NodeShiftTree[T]) GetRaw(s string) (T, bool) { v, ok := r.tree.Get(s) if !ok { @@ -257,6 +301,20 @@ func (r *NodeShiftTree[T]) GetRaw(s string) (T, bool) { return v.(T), true } +// AppendRaw appends ts to the node at the given key without any shifting or transformation. +func (r *NodeShiftTree[T]) AppendRaw(s string, ts ...T) (T, bool) { + if r.transformerRaw == nil { + panic("transformerRaw is required") + } + n, found := r.GetRaw(s) + n2, replaced := r.transformerRaw.Append(n, ts...) + if replaced || !found { + r.insert(s, n2) + } + return n2, replaced || !found +} + +// WalkPrefixRaw walks all nodes with the given prefix in the tree without any shifting or transformation. func (r *NodeShiftTree[T]) WalkPrefixRaw(prefix string, walker func(key string, value T) bool) { walker2 := func(key string, value any) bool { return walker(key, value.(T)) @@ -264,15 +322,15 @@ func (r *NodeShiftTree[T]) WalkPrefixRaw(prefix string, walker func(key string, r.tree.WalkPrefix(prefix, walker2) } -// Shape the tree for dimension d to value v. -func (t *NodeShiftTree[T]) Shape(d, v int) *NodeShiftTree[T] { +// Shape returns a new NodeShiftTree shaped to the given dimension. +func (t *NodeShiftTree[T]) Shape(v sitesmatrix.Vector) *NodeShiftTree[T] { x := t.clone() - x.dims[d] = v + x.siteVector = v return x } func (t *NodeShiftTree[T]) String() string { - return fmt.Sprintf("Root{%v}", t.dims) + return fmt.Sprintf("Root{%v}", t.siteVector) } func (r *NodeShiftTree[T]) Get(s string) T { @@ -280,37 +338,76 @@ func (r *NodeShiftTree[T]) Get(s string) T { return t } -func (r *NodeShiftTree[T]) ForEeachInDimension(s string, d int, f func(T) bool) { +func (r *NodeShiftTree[T]) ForEeachInDimension(s string, dims sitesmatrix.Vector, d int, f func(T) bool) { s = cleanKey(s) v, ok := r.tree.Get(s) if !ok { return } - r.shifter.ForEeachInDimension(v.(T), d, f) + r.shifter.ForEeachInDimension(v.(T), dims, d, f) +} + +func (r *NodeShiftTree[T]) ForEeachInAllDimensions(s string, f func(T) bool) { + s = cleanKey(s) + v, ok := r.tree.Get(s) + if !ok { + return + } + r.shifter.ForEeachInAllDimensions(v.(T), f) } type WalkFunc[T any] func(string, T) (bool, error) +//go:generate stringer -type=NodeTransformState +type NodeTransformState int + +const ( + NodeTransformStateNone NodeTransformState = iota + NodeTransformStateUpdated // Node is updated in place. + NodeTransformStateReplaced // Node is replaced and needs to be re-inserted into the tree. + NodeTransformStateDeleted // Node is deleted and should be removed from the tree. + NodeTransformStateSkip // Skip this node, but continue the walk. + NodeTransformStateTerminate // Terminate the walk. +) + type NodeShiftTreeWalker[T any] struct { // The tree to walk. Tree *NodeShiftTree[T] + // Transform will be called for each node in the main tree. + // v2 will replace v1 in the tree. + // The first bool indicates if the value was replaced and needs to be re-inserted into the tree. + // the second bool indicates if the walk should skip this node. + // the third bool indicates if the walk should terminate. + Transform func(s string, v1 T) (v2 T, state NodeTransformState, err error) + // Handle will be called for each node in the main tree. // If the callback returns true, the walk will stop. // The callback can optionally return a callback for the nested tree. - Handle func(s string, v T, exact DimensionFlag) (terminate bool, err error) + Handle func(s string, v T) (terminate bool, err error) // Optional prefix filter. Prefix string + // IncludeFilter is an optional filter that can be used to filter nodes. + // If it returns false, the node will be skipped. + // Note that v is the shifted value from the tree. + IncludeFilter func(s string, v T) bool + + // IncludeRawFilter is an optional filter that can be used to filter nodes. + // If it returns false, the node will be skipped. + // Note that v is the raw value from the tree. + IncludeRawFilter func(s string, v T) bool + // Enable read or write locking if needed. LockType LockType // When set, no dimension shifting will be performed. NoShift bool - // Don't fall back to alternative dimensions (e.g. language). - Exact bool + // WHen set, will try to fall back to alternative match, + // typically a shared resoures common for all languages. + Fallback bool // Used in development only. Debug bool @@ -340,12 +437,17 @@ func (r *NodeShiftTreeWalker[T]) SkipPrefix(prefix ...string) { } // ShouldSkip returns whether the given key should be skipped in the walk. -func (r *NodeShiftTreeWalker[T]) ShouldSkip(s string) bool { +func (r *NodeShiftTreeWalker[T]) ShouldSkip(s string, v T) bool { for _, prefix := range r.skipPrefixes { if strings.HasPrefix(s, prefix) { return true } } + if r.IncludeRawFilter != nil { + if !r.IncludeRawFilter(s, v) { + return true + } + } return false } @@ -353,59 +455,108 @@ func (r *NodeShiftTreeWalker[T]) Walk(ctx context.Context) error { if r.Tree == nil { panic("Tree is required") } - r.resetLocalState() - if r.LockType > LockTypeNone { - commit1 := r.Tree.Lock(r.LockType == LockTypeWrite) - defer commit1() - } + var deletes []string - main := r.Tree + handleT := func(s string, t T) (ns NodeTransformState, err error) { + if r.IncludeFilter != nil && !r.IncludeFilter(s, t) { + return + } + if r.Transform != nil { + if !r.NoShift { + panic("Transform must be performed with NoShift=true") + } + if ns, err = func() (ns NodeTransformState, err error) { + t, ns, err = r.Transform(s, t) + if ns >= NodeTransformStateSkip || err != nil { + return + } + + switch ns { + case NodeTransformStateReplaced: + r.Tree.tree.Insert(s, t) + case NodeTransformStateDeleted: + // Delay delete until after the walk. + deletes = append(deletes, s) + ns = NodeTransformStateSkip + } + return + }(); ns >= NodeTransformStateSkip || err != nil { + return + } + } - var err error - fnMain := func(s string, v any) bool { - if r.ShouldSkip(s) { - return false + if r.Handle != nil { + var terminate bool + terminate, err = r.Handle(s, t) + if terminate || err != nil { + return + } + } + + return + } + + return func() error { + if r.LockType > LockTypeNone { + unlock := r.Tree.Lock(r.LockType == LockTypeWrite) + defer unlock() } - t, ok, exact := r.toT(r.Tree, v) - if !ok { + r.resetLocalState() + + main := r.Tree + + var err error + + handleV := func(s string, v any) (terminate bool) { + // Context cancellation check. + if ctx != nil && ctx.Err() != nil { + err = ctx.Err() + return true + } + if r.ShouldSkip(s, v.(T)) { + return false + } + var t T + if r.NoShift { + t = v.(T) + } else { + var ok bool + t, ok = r.toT(r.Tree, v) + if !ok { + return false + } + } + var ns NodeTransformState + ns, err = handleT(s, t) + if ns == NodeTransformStateTerminate || err != nil { + return true + } return false } - var terminate bool - terminate, err = r.Handle(s, t, exact) - if terminate || err != nil { - return true + if r.Prefix != "" { + main.tree.WalkPrefix(r.Prefix, handleV) + } else { + main.tree.Walk(handleV) } - return false - } - if r.Prefix != "" { - main.tree.WalkPrefix(r.Prefix, fnMain) - } else { - main.tree.Walk(fnMain) - } + // This is currently only performed with no shift. + for _, s := range deletes { + main.tree.Delete(s) + } - if err != nil { return err - } - - return nil + }() } func (r *NodeShiftTreeWalker[T]) resetLocalState() { r.skipPrefixes = nil } -func (r *NodeShiftTreeWalker[T]) toT(tree *NodeShiftTree[T], v any) (t T, ok bool, exact DimensionFlag) { - if r.NoShift { - t = v.(T) - ok = true - } else { - t, ok, exact = tree.shift(v.(T), r.Exact) - } - return +func (r *NodeShiftTreeWalker[T]) toT(tree *NodeShiftTree[T], v any) (T, bool) { + return tree.shift(v.(T), r.Fallback) } func (r *NodeShiftTree[T]) Has(s string) bool { @@ -417,36 +568,22 @@ func (t NodeShiftTree[T]) clone() *NodeShiftTree[T] { return &t } -func (r *NodeShiftTree[T]) shift(t T, exact bool) (T, bool, DimensionFlag) { - return r.shifter.Shift(t, r.dims, exact) +func (r *NodeShiftTree[T]) shift(t T, fallback bool) (T, bool) { + return r.shifter.Shift(t, r.siteVector, fallback) } func (r *NodeShiftTree[T]) get(s string) (T, bool) { s = cleanKey(s) v, ok := r.tree.Get(s) + if !ok { var t T return t, false } - t, ok, _ := r.shift(v.(T), true) - return t, ok -} - -type WalkConfig[T any] struct { - // Optional prefix filter. - Prefix string - - // Callback will be called for each node in the tree. - // If the callback returns true, the walk will stop. - Callback func(ctx *WalkContext[T], s string, t T) (bool, error) - - // Enable read or write locking if needed. - LockType LockType - - // When set, no dimension shifting will be performed. - NoShift bool + if v, ok := r.shift(v.(T), false); ok { + return v, true + } - // Exact will only match exact in the current dimension (e.g. language), - // and will not look for alternatives. - Exact bool + var t T + return t, false } diff --git a/hugolib/doctree/nodetransformstate_string.go b/hugolib/doctree/nodetransformstate_string.go new file mode 100644 index 000000000..03c018dbc --- /dev/null +++ b/hugolib/doctree/nodetransformstate_string.go @@ -0,0 +1,28 @@ +// Code generated by "stringer -type=NodeTransformState"; DO NOT EDIT. + +package doctree + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[NodeTransformStateNone-0] + _ = x[NodeTransformStateUpdated-1] + _ = x[NodeTransformStateReplaced-2] + _ = x[NodeTransformStateDeleted-3] + _ = x[NodeTransformStateSkip-4] + _ = x[NodeTransformStateTerminate-5] +} + +const _NodeTransformState_name = "NodeTransformStateNoneNodeTransformStateUpdatedNodeTransformStateReplacedNodeTransformStateDeletedNodeTransformStateSkipNodeTransformStateTerminate" + +var _NodeTransformState_index = [...]uint8{0, 22, 47, 73, 98, 120, 147} + +func (i NodeTransformState) String() string { + if i < 0 || i >= NodeTransformState(len(_NodeTransformState_index)-1) { + return "NodeTransformState(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _NodeTransformState_name[_NodeTransformState_index[i]:_NodeTransformState_index[i+1]] +} diff --git a/hugolib/doctree/support.go b/hugolib/doctree/support.go index f1b713b31..9cdc36fda 100644 --- a/hugolib/doctree/support.go +++ b/hugolib/doctree/support.go @@ -15,8 +15,12 @@ package doctree import ( "fmt" + "iter" "strings" "sync" + + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" ) var _ MutableTrees = MutableTrees{} @@ -45,7 +49,7 @@ func (ctx *WalkContext[T]) AddEventListener(event, path string, handler func(*Ev ctx.eventHandlers[event] = append( ctx.eventHandlers[event], func(e *Event[T]) { // Propagate events up the tree only. - if strings.HasPrefix(e.Path, path) { + if e.Path != path && strings.HasPrefix(e.Path, path) { handler(e) } }, @@ -65,6 +69,29 @@ func (ctx *WalkContext[T]) Data() *SimpleThreadSafeTree[any] { return ctx.data } +func (ctx *WalkContext[T]) initDataRaw() { + ctx.dataRawInit.Do(func() { + ctx.dataRaw = maps.NewCache[sitesmatrix.Vector, *SimpleThreadSafeTree[any]]() + }) +} + +func (ctx *WalkContext[T]) DataRaw(vec sitesmatrix.Vector) *SimpleThreadSafeTree[any] { + ctx.initDataRaw() + v, _ := ctx.dataRaw.GetOrCreate(vec, func() (*SimpleThreadSafeTree[any], error) { + return NewSimpleThreadSafeTree[any](), nil + }) + return v +} + +func (ctx *WalkContext[T]) DataRawForEeach() iter.Seq2[sitesmatrix.Vector, *SimpleThreadSafeTree[any]] { + ctx.initDataRaw() + return func(yield func(vec sitesmatrix.Vector, data *SimpleThreadSafeTree[any]) bool) { + ctx.dataRaw.ForEeach(func(vec sitesmatrix.Vector, data *SimpleThreadSafeTree[any]) bool { + return yield(vec, data) + }) + } +} + // SendEvent sends an event up the tree. func (ctx *WalkContext[T]) SendEvent(event *Event[T]) { ctx.events = append(ctx.events, event) @@ -110,9 +137,8 @@ type LockType int // MutableTree is a tree that can be modified. type MutableTree interface { DeleteRaw(key string) - DeleteAll(key string) DeletePrefix(prefix string) int - DeletePrefixAll(prefix string) int + DeletePrefixRaw(prefix string) int Lock(writable bool) (commit func()) CanLock() bool // Used for troubleshooting only. } @@ -142,12 +168,6 @@ func (t MutableTrees) DeleteRaw(key string) { } } -func (t MutableTrees) DeleteAll(key string) { - for _, tree := range t { - tree.DeleteAll(key) - } -} - func (t MutableTrees) DeletePrefix(prefix string) int { var count int for _, tree := range t { @@ -156,10 +176,10 @@ func (t MutableTrees) DeletePrefix(prefix string) int { return count } -func (t MutableTrees) DeletePrefixAll(prefix string) int { +func (t MutableTrees) DeletePrefixRaw(prefix string) int { var count int for _, tree := range t { - count += tree.DeletePrefixAll(prefix) + count += tree.DeletePrefixRaw(prefix) } return count } @@ -187,8 +207,12 @@ func (t MutableTrees) CanLock() bool { // WalkContext is passed to the Walk callback. type WalkContext[T any] struct { - data *SimpleThreadSafeTree[any] - dataInit sync.Once + data *SimpleThreadSafeTree[any] + dataInit sync.Once + + dataRaw *maps.Cache[sitesmatrix.Vector, *SimpleThreadSafeTree[any]] + dataRawInit sync.Once + eventHandlers eventHandlers[T] events []*Event[T] diff --git a/hugolib/doctree/treeshifttree.go b/hugolib/doctree/treeshifttree.go index 8f958d828..bcbb3d361 100644 --- a/hugolib/doctree/treeshifttree.go +++ b/hugolib/doctree/treeshifttree.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Hugo Authors. All rights reserved. +// Copyright 2025 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,52 +13,55 @@ package doctree -import "iter" +import ( + "iter" -var _ TreeThreadSafe[string] = (*TreeShiftTree[string])(nil) + "github.com/gohugoio/hugo/hugolib/sitesmatrix" +) -type TreeShiftTree[T comparable] struct { - // This tree is shiftable in one dimension. - d int +var _ TreeThreadSafe[string] = (*TreeShiftTreeSlice[string])(nil) - // The value of the current dimension. - v int +type TreeShiftTreeSlice[T comparable] struct { + // v points to a specific tree in the slice. + v sitesmatrix.Vector // The zero value of T. zero T - // Will be of length equal to the length of the dimension. - trees []*SimpleThreadSafeTree[T] + // trees is a 3D slice that holds all the trees. + // Note that we have tested a version backed by a map, which is as fast to use, but is twice as epxensive/slow to create. + trees [][][]*SimpleThreadSafeTree[T] } -func NewTreeShiftTree[T comparable](d, length int) *TreeShiftTree[T] { - if length <= 0 { - panic("length must be > 0") - } - trees := make([]*SimpleThreadSafeTree[T], length) - for i := range length { - trees[i] = NewSimpleThreadSafeTree[T]() +func NewTreeShiftTree[T comparable](v sitesmatrix.Vector) *TreeShiftTreeSlice[T] { + trees := make([][][]*SimpleThreadSafeTree[T], v[0]) + for i := 0; i < v[0]; i++ { + trees[i] = make([][]*SimpleThreadSafeTree[T], v[1]) + for j := 0; j < v[1]; j++ { + trees[i][j] = make([]*SimpleThreadSafeTree[T], v[2]) + for k := 0; k < v[2]; k++ { + trees[i][j][k] = NewSimpleThreadSafeTree[T]() + } + } } - return &TreeShiftTree[T]{d: d, trees: trees} + return &TreeShiftTreeSlice[T]{trees: trees} } -func (t TreeShiftTree[T]) Shape(d, v int) *TreeShiftTree[T] { - if d != t.d { - panic("dimension mismatch") - } - if v >= len(t.trees) { - panic("value out of range") - } +func (t TreeShiftTreeSlice[T]) Shape(v sitesmatrix.Vector) *TreeShiftTreeSlice[T] { t.v = v return &t } -func (t *TreeShiftTree[T]) Get(s string) T { - return t.trees[t.v].Get(s) +func (t *TreeShiftTreeSlice[T]) tree() *SimpleThreadSafeTree[T] { + return t.trees[t.v[0]][t.v[1]][t.v[2]] +} + +func (t *TreeShiftTreeSlice[T]) Get(s string) T { + return t.tree().Get(s) } -func (t *TreeShiftTree[T]) DeleteAllFunc(s string, f func(s string, v T) bool) { - for _, tt := range t.trees { +func (t *TreeShiftTreeSlice[T]) DeleteAllFunc(s string, f func(s string, v T) bool) { + for tt := range t.Trees() { if v := tt.Get(s); v != t.zero { if f(s, v) { // Delete. @@ -68,24 +71,38 @@ func (t *TreeShiftTree[T]) DeleteAllFunc(s string, f func(s string, v T) bool) { } } -func (t *TreeShiftTree[T]) LongestPrefix(s string) (string, T) { - return t.trees[t.v].LongestPrefix(s) +func (t *TreeShiftTreeSlice[T]) Trees() iter.Seq[*SimpleThreadSafeTree[T]] { + return func(yield func(v *SimpleThreadSafeTree[T]) bool) { + for _, l1 := range t.trees { + for _, l2 := range l1 { + for _, l3 := range l2 { + if !yield(l3) { + return + } + } + } + } + } +} + +func (t *TreeShiftTreeSlice[T]) LongestPrefix(s string) (string, T) { + return t.tree().LongestPrefix(s) } -func (t *TreeShiftTree[T]) Insert(s string, v T) T { - return t.trees[t.v].Insert(s, v) +func (t *TreeShiftTreeSlice[T]) Insert(s string, v T) T { + return t.tree().Insert(s, v) } -func (t *TreeShiftTree[T]) Lock(lockType LockType) func() { - return t.trees[t.v].Lock(lockType) +func (t *TreeShiftTreeSlice[T]) Lock(lockType LockType) func() { + return t.tree().Lock(lockType) } -func (t *TreeShiftTree[T]) WalkPrefix(lockType LockType, s string, f func(s string, v T) (bool, error)) error { - return t.trees[t.v].WalkPrefix(lockType, s, f) +func (t *TreeShiftTreeSlice[T]) WalkPrefix(lockType LockType, s string, f func(s string, v T) (bool, error)) error { + return t.tree().WalkPrefix(lockType, s, f) } -func (t *TreeShiftTree[T]) WalkPrefixRaw(lockType LockType, s string, f func(s string, v T) (bool, error)) error { - for _, tt := range t.trees { +func (t *TreeShiftTreeSlice[T]) WalkPrefixRaw(lockType LockType, s string, f func(s string, v T) (bool, error)) error { + for tt := range t.Trees() { if err := tt.WalkPrefix(lockType, s, f); err != nil { return err } @@ -93,31 +110,31 @@ func (t *TreeShiftTree[T]) WalkPrefixRaw(lockType LockType, s string, f func(s s return nil } -func (t *TreeShiftTree[T]) WalkPath(lockType LockType, s string, f func(s string, v T) (bool, error)) error { - return t.trees[t.v].WalkPath(lockType, s, f) +func (t *TreeShiftTreeSlice[T]) WalkPath(lockType LockType, s string, f func(s string, v T) (bool, error)) error { + return t.tree().WalkPath(lockType, s, f) } -func (t *TreeShiftTree[T]) All(lockType LockType) iter.Seq2[string, T] { - return t.trees[t.v].All(lockType) +func (t *TreeShiftTreeSlice[T]) All(lockType LockType) iter.Seq2[string, T] { + return t.tree().All(lockType) } -func (t *TreeShiftTree[T]) LenRaw() int { +func (t *TreeShiftTreeSlice[T]) LenRaw() int { var count int - for _, tt := range t.trees { + for tt := range t.Trees() { count += tt.tree.Len() } return count } -func (t *TreeShiftTree[T]) Delete(key string) { - for _, tt := range t.trees { +func (t *TreeShiftTreeSlice[T]) Delete(key string) { + for tt := range t.Trees() { tt.tree.Delete(key) } } -func (t *TreeShiftTree[T]) DeletePrefix(prefix string) int { +func (t *TreeShiftTreeSlice[T]) DeletePrefix(prefix string) int { var count int - for _, tt := range t.trees { + for tt := range t.Trees() { count += tt.tree.DeletePrefix(prefix) } return count diff --git a/hugolib/doctree/treeshifttree_test.go b/hugolib/doctree/treeshifttree_test.go index c39ff38aa..b0ae79633 100644 --- a/hugolib/doctree/treeshifttree_test.go +++ b/hugolib/doctree/treeshifttree_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Hugo Authors. All rights reserved. +// Copyright 2025 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,39 @@ import ( qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/hugolib/doctree" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" ) func TestTreeShiftTree(t *testing.T) { c := qt.New(t) - tree := doctree.NewTreeShiftTree[string](0, 10) + tree := doctree.NewTreeShiftTree[string](sitesmatrix.Vector{10, 1, 1}) c.Assert(tree, qt.IsNotNil) } + +func BenchmarkTreeShiftTreeSlice(b *testing.B) { + v := sitesmatrix.Vector{10, 10, 10} + t := doctree.NewTreeShiftTree[string](v) + b.Run("New", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for l1 := 0; l1 < v[0]; l1++ { + for l2 := 0; l2 < v[1]; l2++ { + for l3 := 0; l3 < v[2]; l3++ { + _ = doctree.NewTreeShiftTree[string](sitesmatrix.Vector{l1, l2, l3}) + } + } + } + } + }) + b.Run("Shape", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for l1 := 0; l1 < v[0]; l1++ { + for l2 := 0; l2 < v[1]; l2++ { + for l3 := 0; l3 < v[2]; l3++ { + t.Shape(sitesmatrix.Vector{l1, l2, l3}) + } + } + } + } + }) +} diff --git a/hugolib/fileInfo.go b/hugolib/fileInfo.go index a01b37008..567e87006 100644 --- a/hugolib/fileInfo.go +++ b/hugolib/fileInfo.go @@ -23,8 +23,6 @@ import ( type fileInfo struct { *source.File - - overriddenLang string } func (fi *fileInfo) Open() (afero.File, error) { @@ -37,9 +35,6 @@ func (fi *fileInfo) Open() (afero.File, error) { } func (fi *fileInfo) Lang() string { - if fi.overriddenLang != "" { - return fi.overriddenLang - } return fi.File.Lang() } diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index b32b8796f..dd3d97e6c 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -16,6 +16,7 @@ package filesystems import ( + "context" "fmt" "io" "os" @@ -26,21 +27,23 @@ import ( "github.com/bep/overlayfs" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/htesting" - "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/common/loggers" - "github.com/gohugoio/hugo/common/types" "github.com/rogpeppe/go-internal/lockedfile" "github.com/gohugoio/hugo/hugofs/files" + "github.com/gohugoio/hugo/hugofs/hglob" "github.com/gohugoio/hugo/modules" hpaths "github.com/gohugoio/hugo/common/paths" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugolib/paths" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "github.com/spf13/afero" ) @@ -127,7 +130,7 @@ func (b *BaseFs) WatchFilenames() []string { w := hugofs.NewWalkway(hugofs.WalkwayConfig{ Fs: sourceFs, Root: meta.Filename, - WalkFn: func(path string, fi hugofs.FileMetaInfo) error { + WalkFn: func(ctx context.Context, path string, fi hugofs.FileMetaInfo) error { if !fi.IsDir() { return nil } @@ -555,10 +558,9 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { fs := hugofs.NewComponentFs( hugofs.ComponentFsOptions{ - Fs: overlayFs, - Component: componentID, - DefaultContentLanguage: b.p.Cfg.DefaultContentLanguage(), - PathParser: b.p.Cfg.PathParser(), + Fs: overlayFs, + Component: componentID, + Cfg: b.p.Cfg, }, ) @@ -579,10 +581,9 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { contentFs := hugofs.NewComponentFs( hugofs.ComponentFsOptions{ - Fs: b.theBigFs.overlayMountsContent, - Component: files.ComponentFolderContent, - DefaultContentLanguage: b.p.Cfg.DefaultContentLanguage(), - PathParser: b.p.Cfg.PathParser(), + Fs: b.theBigFs.overlayMountsContent, + Component: files.ComponentFolderContent, + Cfg: b.p.Cfg, }, ) @@ -611,7 +612,7 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { func (b *sourceFilesystemsBuilder) createMainOverlayFs(p *paths.Paths) (*filesystemsCollector, error) { var staticFsMap map[string]*overlayfs.OverlayFs if b.p.Cfg.IsMultihost() { - languages := b.p.Cfg.Languages() + languages := b.p.Cfg.Languages().(langs.Languages) staticFsMap = make(map[string]*overlayfs.OverlayFs) for _, l := range languages { staticFsMap[l.Lang] = overlayfs.New(overlayfs.Options{}) @@ -624,8 +625,8 @@ func (b *sourceFilesystemsBuilder) createMainOverlayFs(p *paths.Paths) (*filesys staticPerLanguage: staticFsMap, overlayMounts: overlayfs.New(overlayfs.Options{}), - overlayMountsContent: overlayfs.New(overlayfs.Options{DirsMerger: hugofs.LanguageDirsMerger}), - overlayMountsStatic: overlayfs.New(overlayfs.Options{DirsMerger: hugofs.LanguageDirsMerger}), + overlayMountsContent: overlayfs.New(overlayfs.Options{DirsMerger: hugofs.AppendDirsMerger}), + overlayMountsStatic: overlayfs.New(overlayfs.Options{}), overlayFull: overlayfs.New(overlayfs.Options{}), overlayResources: overlayfs.New(overlayfs.Options{FirstWritable: true}), } @@ -661,6 +662,10 @@ func (b *sourceFilesystemsBuilder) isStaticMount(mnt modules.Mount) bool { return strings.HasPrefix(mnt.Target, files.ComponentFolderStatic) } +func (b *sourceFilesystemsBuilder) isLayoutsMount(mnt modules.Mount) bool { + return strings.HasPrefix(mnt.Target, files.ComponentFolderLayouts) +} + func (b *sourceFilesystemsBuilder) createOverlayFs( collector *filesystemsCollector, mounts []mountsDescriptor, @@ -684,9 +689,9 @@ func (b *sourceFilesystemsBuilder) createOverlayFs( for _, md := range mounts { var ( - fromTo []hugofs.RootMapping - fromToContent []hugofs.RootMapping - fromToStatic []hugofs.RootMapping + fromTo []*hugofs.RootMapping + fromToContent []*hugofs.RootMapping + fromToStatic []*hugofs.RootMapping ) absPathify := func(path string) (string, string) { @@ -702,17 +707,73 @@ func (b *sourceFilesystemsBuilder) createOverlayFs( // the first entry wins. mountWeight := (10 + md.ordinal) * (len(md.Mounts()) - i) - inclusionFilter, err := glob.NewFilenameFilter( - types.ToStringSlicePreserveString(mount.IncludeFiles), - types.ToStringSlicePreserveString(mount.ExcludeFiles), - ) + patterns, hasLegacyIncludes, hasLegacyExcludes := mount.FilesToFilter() + if hasLegacyIncludes { + hugo.Deprecate("module.mounts.includeFiles", "Replaced by the simpler 'files' setting, see https://gohugo.io/configuration/module/#files", "v0.153.0") + } + if hasLegacyExcludes { + hugo.Deprecate("module.mounts.excludeFiles", "Replaced by the simpler 'files' setting, see https://gohugo.io/configuration/module/#files", "v0.153.0") + } + + inclusionFilter, err := hglob.NewFilenameFilterV2(patterns) if err != nil { return err } base, filename := absPathify(mount.Source) - rm := hugofs.RootMapping{ + var matrix *sitesmatrix.IntSets + v := mount.Sites + + needsDefaultsIfNotset := b.isContentMount(mount) + needsDefaultsAndAllLanguagesIfNotSet := b.isStaticMount(mount) + needsAllIfNotSet := b.isLayoutsMount(mount) + + intSetsCfg := sitesmatrix.IntSetsConfig{ + ApplyDefaults: 0, + Globs: v.Matrix, + } + + matrixBuilder := sitesmatrix.NewIntSetsBuilder(b.p.Cfg.ConfiguredDimensions()) + + if !v.Matrix.IsZero() { + matrixBuilder.WithConfig(intSetsCfg) + if !matrixBuilder.GlobFilterMisses.IsZero() { + continue + } + + if needsDefaultsIfNotset { + matrixBuilder.WithDefaultsIfNotSet() + } + if needsAllIfNotSet { + matrixBuilder.WithAllIfNotSet() + } + + matrix = matrixBuilder.Build() + } else if needsAllIfNotSet { + matrixBuilder.WithAllIfNotSet() + matrix = matrixBuilder.Build() + } else if needsDefaultsAndAllLanguagesIfNotSet { + matrixBuilder.WithDefaultsAndAllLanguagesIfNotSet() + matrix = matrixBuilder.Build() + } else if needsDefaultsIfNotset { + matrix = b.p.Cfg.DefaultContentsitesMatrix() + } else { + if needsDefaultsIfNotset { + matrixBuilder.WithDefaultsIfNotSet() + } + matrix = matrixBuilder.Build() + } + + var complements *sitesmatrix.IntSets + if !v.Complements.IsZero() { + intSetsCfg.Globs = v.Complements + intSetsCfg.ApplyDefaults = 0 + matrixBuilder = sitesmatrix.NewIntSetsBuilder(b.p.Cfg.ConfiguredDimensions()).WithConfig(intSetsCfg).WithAllIfNotSet() + complements = matrixBuilder.Build() + } + + rm := &hugofs.RootMapping{ From: mount.Target, To: filename, ToBase: base, @@ -720,21 +781,16 @@ func (b *sourceFilesystemsBuilder) createOverlayFs( ModuleOrdinal: md.ordinal, IsProject: md.isMainProject, Meta: &hugofs.FileMeta{ - Watch: !mount.DisableWatch && md.Watch(), - Weight: mountWeight, - InclusionFilter: inclusionFilter, + Watch: !mount.DisableWatch && md.Watch(), + Weight: mountWeight, + InclusionFilter: inclusionFilter, + SitesMatrix: matrix, + SitesComplements: complements, }, } isContentMount := b.isContentMount(mount) - lang := mount.Lang - if lang == "" && isContentMount { - lang = b.p.Cfg.DefaultContentLanguage() - } - - rm.Meta.Lang = lang - if isContentMount { fromToContent = append(fromToContent, rm) } else if b.isStaticMount(mount) { @@ -770,12 +826,11 @@ func (b *sourceFilesystemsBuilder) createOverlayFs( collector.addRootFs(rmfsStatic) if collector.staticPerLanguage != nil { - for _, l := range b.p.Cfg.Languages() { + for i, l := range b.p.Cfg.Languages().(langs.Languages) { lang := l.Lang - lfs := rmfsStatic.Filter(func(rm hugofs.RootMapping) bool { - rlang := rm.Meta.Lang - return rlang == "" || rlang == lang + lfs := rmfsStatic.Filter(func(rm *hugofs.RootMapping) bool { + return rm.Meta.SitesMatrix.HasLanguage(i) }) bfs := hugofs.NewBasePathFs(lfs, files.ComponentFolderStatic) collector.staticPerLanguage[lang] = collector.staticPerLanguage[lang].Append(bfs) diff --git a/hugolib/filesystems/basefs_test.go b/hugolib/filesystems/basefs_test.go index abe06ac4a..4a6e0bbaf 100644 --- a/hugolib/filesystems/basefs_test.go +++ b/hugolib/filesystems/basefs_test.go @@ -14,6 +14,7 @@ package filesystems_test import ( + "context" "errors" "fmt" "os" @@ -656,7 +657,7 @@ func countFilesAndGetFilenames(fs afero.Fs, dirname string) (int, []string, erro counter := 0 var filenames []string - wf := func(path string, info hugofs.FileMetaInfo) error { + wf := func(ctx context.Context, path string, info hugofs.FileMetaInfo) error { if !info.IsDir() { counter++ } diff --git a/hugolib/hugo_modules_test.go b/hugolib/hugo_modules_test.go index 243447805..895973b0f 100644 --- a/hugolib/hugo_modules_test.go +++ b/hugolib/hugo_modules_test.go @@ -28,8 +28,8 @@ import ( "github.com/spf13/afero" - "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/common/version" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/hugofs" @@ -304,7 +304,7 @@ func TestHugoModulesMatrix(t *testing.T) { } t.Parallel() - if !htesting.IsCI() || hugo.GoMinorVersion() < 12 { + if !htesting.IsCI() || version.GoMinorVersion() < 12 { // https://github.com/golang/go/issues/26794 // There were some concurrent issues with Go modules in < Go 12. t.Skip("skip this on local host and for Go <= 1.11 due to a bug in Go's stdlib") @@ -709,11 +709,9 @@ baseURL = "https://example.org" ` - b := Test(t, files, TestOptWithConfig(func(cfg *IntegrationTestConfig) { + Test(t, files, TestOptWithConfig(func(cfg *IntegrationTestConfig) { cfg.WorkingDir = tempDir })) - - b.Build() } // https://github.com/gohugoio/hugo/issues/6622 diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index e232fd855..5621cd6e6 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -17,6 +17,7 @@ import ( "context" "fmt" "io" + "iter" "strings" "sync" "sync/atomic" @@ -25,8 +26,9 @@ import ( "github.com/bep/logg" "github.com/gohugoio/hugo/cache/dynacache" "github.com/gohugoio/hugo/config/allconfig" - "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/hugofs/hglob" "github.com/gohugoio/hugo/hugolib/doctree" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "github.com/gohugoio/hugo/resources" "github.com/fsnotify/fsnotify" @@ -34,6 +36,7 @@ import ( "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/parser/metadecoders" + "github.com/gohugoio/hugo/common/hsync" "github.com/gohugoio/hugo/common/htime" "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/common/maps" @@ -47,15 +50,21 @@ import ( "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/lazy" "github.com/gohugoio/hugo/resources/page" ) // HugoSites represents the sites to build. Each site represents a language. type HugoSites struct { + // The current sites slice. + // When rendering, this slice will be shifted out. Sites []*Site + // All sites for all versions and roles. + sitesVersionsRoles [][][]*Site + sitesVersionsRolesMap map[sitesmatrix.Vector]*Site + sitesLanguages []*Site // sample set with all languages. + Configs *allconfig.Configs hugoInfo hugo.HugoInfo @@ -86,7 +95,9 @@ type HugoSites struct { // be relatively rare and low volume. translationKeyPages *maps.SliceCache[page.Page] - pageTrees *pageTrees + pageTrees *pageTrees + previousPageTreesWalkContext *doctree.WalkContext[contentNode] // Set for rebuilds only. + previousSeenTerms map[term]sitesmatrix.Vectors // Set for rebuilds only. printUnusedTemplatesInit sync.Once printPathWarningsInit sync.Once @@ -124,6 +135,57 @@ func (p *progressReporter) Start() { p.t = htime.Now() } +func (h *HugoSites) allSites(include func(s *Site) bool) iter.Seq[*Site] { + return func(yield func(s *Site) bool) { + for _, v := range h.sitesVersionsRoles { + for _, r := range v { + for _, s := range r { + if include != nil && !include(s) { + continue + } + if !yield(s) { + return + } + } + } + } + } +} + +// allSiteLanguages will range over the first site in each language that's not skipped. +func (h *HugoSites) allSiteLanguages(include func(s *Site) bool) iter.Seq[*Site] { + return func(yield func(s *Site) bool) { + LOOP: + for _, v := range h.sitesVersionsRoles { + for _, r := range v { + for _, s := range r { + if include != nil && !include(s) { + continue LOOP + } + if !yield(r[0]) { + return + } + continue LOOP + } + } + } + } +} + +func (h *HugoSites) getFirstTaxonomyConfig(s string) (v viewName) { + for _, ss := range h.sitesLanguages { + if v = ss.pageMap.cfg.getTaxonomyConfig(s); !v.IsZero() { + return + } + } + return +} + +// returns one of the sites with the language of the given vector. +func (h *HugoSites) languageSiteForSiteVector(v sitesmatrix.Vector) *Site { + return h.sitesLanguages[v.Language()] +} + // ShouldSkipFileChangeEvent allows skipping filesystem event early before // the build is started. func (h *HugoSites) ShouldSkipFileChangeEvent(ev fsnotify.Event) bool { @@ -140,18 +202,20 @@ func (h *HugoSites) isRebuild() bool { return h.buildCounter.Load() > 0 } -func (h *HugoSites) resolveSite(lang string) *Site { - if lang == "" { - lang = h.Conf.DefaultContentLanguage() - } - - for _, s := range h.Sites { - if s.Lang() == lang { - return s +func (h *HugoSites) resolveFirstSite(matrix sitesmatrix.VectorStore) *Site { + var s *Site + var ok bool + matrix.ForEachVector(func(v sitesmatrix.Vector) bool { + if s, ok = h.sitesVersionsRolesMap[v]; ok { + return false } + return true + }) + if s == nil { + panic(fmt.Sprintf("no site found for matrix %s", matrix)) } - return nil + return s } type buildCounters struct { @@ -201,14 +265,14 @@ func (f *fatalErrorHandler) Done() <-chan bool { type hugoSitesInit struct { // Loads the data from all of the /data folders. - data *lazy.Init + data hsync.FuncResetter // Loads the Git info and CODEOWNERS for all the pages if enabled. - gitInfo *lazy.Init + gitInfo hsync.FuncResetter } func (h *HugoSites) Data() map[string]any { - if _, err := h.init.data.Do(context.Background()); err != nil { + if err := h.init.data.Do(context.Background()); err != nil { h.SendError(fmt.Errorf("failed to load data: %w", err)) return nil } @@ -251,7 +315,7 @@ func (h *HugoSites) RegularPages() page.Pages { } func (h *HugoSites) gitInfoForPage(p page.Page) (*source.GitInfo, error) { - if _, err := h.init.gitInfo.Do(context.Background()); err != nil { + if err := h.init.gitInfo.Do(context.Background()); err != nil { return nil, err } @@ -263,7 +327,7 @@ func (h *HugoSites) gitInfoForPage(p page.Page) (*source.GitInfo, error) { } func (h *HugoSites) codeownersForPage(p page.Page) ([]string, error) { - if _, err := h.init.gitInfo.Do(context.Background()); err != nil { + if err := h.init.gitInfo.Do(context.Background()); err != nil { return nil, err } @@ -443,7 +507,7 @@ func (h *HugoSites) loadGitInfo() error { } // Reset resets the sites and template caches etc., making it ready for a full rebuild. -func (h *HugoSites) reset(config *BuildCfg) { +func (h *HugoSites) reset() { h.fatalErrorHandler = &fatalErrorHandler{ h: h, donec: make(chan bool), @@ -469,10 +533,10 @@ func (h *HugoSites) withSite(fn func(s *Site) error) error { func (h *HugoSites) withPage(fn func(s string, p *pageState) bool) { h.withSite(func(s *Site) error { - w := &doctree.NodeShiftTreeWalker[contentNodeI]{ + w := &doctree.NodeShiftTreeWalker[contentNode]{ Tree: s.pageMap.treePages, LockType: doctree.LockTypeRead, - Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { + Handle: func(s string, n contentNode) (bool, error) { return fn(s, n.(*pageState)), nil }, } @@ -498,7 +562,7 @@ type BuildCfg struct { RecentlyTouched *types.EvictingQueue[string] // Can be set to build only with a sub set of the content source. - ContentInclusionFilter *glob.FilenameFilter + ContentInclusionFilter *hglob.FilenameFilter // Set when the buildlock is already acquired (e.g. the archetype content builder). NoBuildLock bool @@ -580,7 +644,7 @@ func (h *HugoSites) loadData() error { Fs: h.PathSpec.BaseFs.Data.Fs, IgnoreFile: h.SourceSpec.IgnoreFile, PathParser: h.Conf.PathParser(), - WalkFn: func(path string, fi hugofs.FileMetaInfo) error { + WalkFn: func(ctx context.Context, path string, fi hugofs.FileMetaInfo) error { if fi.IsDir() { return nil } diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 12998ba83..3cd30c41d 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -31,7 +31,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugofs/files" - "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/hugofs/hglob" "github.com/gohugoio/hugo/hugolib/doctree" "github.com/gohugoio/hugo/hugolib/pagesfromdata" "github.com/gohugoio/hugo/hugolib/segments" @@ -60,6 +60,9 @@ import ( // Build builds all sites. If filesystem events are provided, // this is considered to be a potential partial rebuild. func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { + if h.isRebuild() && !h.Conf.Watching() { + return errors.New("Build called multiple times when not in watch or server mode (typically with hugolib.Test(t, files).Build(); Build() is already called once by Test)") + } if !h.isRebuild() && terminal.PrintANSIColors(os.Stdout) { // Don't show progress for fast builds. d := debounce.New(250 * time.Millisecond) @@ -153,7 +156,7 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { return fmt.Errorf("initRebuild: %w", err) } } else { - if err := h.initSites(conf); err != nil { + if err := h.initSites(); err != nil { return fmt.Errorf("initSites: %w", err) } } @@ -184,7 +187,7 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { } } - for _, s := range h.Sites { + for s := range h.allSites(nil) { s.state = siteStateReady } @@ -248,8 +251,8 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { // Build lifecycle methods below. // The order listed matches the order of execution. -func (h *HugoSites) initSites(config *BuildCfg) error { - h.reset(config) +func (h *HugoSites) initSites() error { + h.reset() return nil } @@ -258,8 +261,8 @@ func (h *HugoSites) initRebuild(config *BuildCfg) error { return errors.New("rebuild called when not in watch mode") } - h.pageTrees.treePagesResources.WalkPrefixRaw("", func(key string, n contentNodeI) bool { - n.resetBuildState() + h.pageTrees.treePagesResources.WalkPrefixRaw("", func(key string, n contentNode) bool { + cnh.resetBuildState(n) return false }) @@ -267,7 +270,7 @@ func (h *HugoSites) initRebuild(config *BuildCfg) error { s.resetBuildState(config.WhatChanged.needsPagesAssembly) } - h.reset(config) + h.reset() h.resetLogs() return nil @@ -309,22 +312,43 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil } h.translationKeyPages.Reset() - assemblers := make([]*sitePagesAssembler, len(h.Sites)) - // Changes detected during assembly (e.g. aggregate date changes) - for i, s := range h.Sites { - assemblers[i] = &sitePagesAssembler{ - Site: s, + var assemblers []*sitePagesAssembler + // Changes detected during assembly (e.g. aggregate date changes) + for s := range h.allSites(nil) { + assemblers = append(assemblers, &sitePagesAssembler{ + s: s, assembleChanges: bcfg.WhatChanged, ctx: ctx, - } + }) + } + + apa := newAllPagesAssembler( + ctx, + h, + assemblers[0].s.pageMap, + bcfg.WhatChanged, + ) + for _, s := range assemblers { + s.a = apa + } + + if h.Conf.Watching() { + defer func() { + // Store previous walk context to detect cascade changes on next rebuild. + h.previousPageTreesWalkContext = apa.rw.WalkContext + }() + } + + if err := apa.createAllPages(); err != nil { + return err } g, _ := h.workersSite.Start(ctx) for _, s := range assemblers { s := s g.Run(func() error { - return s.assemblePagesStep1(ctx) + return s.assemblePagesStep1() }) } if err := g.Wait(); err != nil { @@ -356,8 +380,8 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil } h.renderFormats = output.Formats{} - for _, s := range h.Sites { - s.s.initRenderFormats() + for s := range h.allSites(nil) { + s.initRenderFormats() h.renderFormats = append(h.renderFormats, s.renderFormats...) } @@ -397,18 +421,17 @@ func (h *HugoSites) render(l logg.LevelLogger, config *BuildCfg) error { } i := 0 - for _, s := range h.Sites { - segmentFilter := s.conf.C.SegmentFilter - if segmentFilter.ShouldExcludeCoarse(segments.SegmentMatcherFields{Lang: s.language.Lang}) { - l.Logf("skip language %q not matching segments set in --renderSegments", s.language.Lang) + + for s := range h.allSites(nil) { + if s.conf.Segments.Config.SegmentFilter.ShouldExcludeCoarse(segments.SegmentQuery{Site: s.siteVector}) { + l.Logf("skip site %s not matching segments set in --renderSegments", s.resolveDimensionNames()) continue } - - siteRenderContext.languageIdx = s.languagei + siteRenderContext.languageIdx = s.siteVector.Language() h.currentSite = s for siteOutIdx, renderFormat := range s.renderFormats { - if segmentFilter.ShouldExcludeCoarse(segments.SegmentMatcherFields{Output: renderFormat.Name, Lang: s.language.Lang}) { - l.Logf("skip output format %q for language %q not matching segments set in --renderSegments", renderFormat.Name, s.language.Lang) + if s.conf.Segments.Config.SegmentFilter.ShouldExcludeCoarse(segments.SegmentQuery{Output: renderFormat.Name, Site: s.siteVector}) { + l.Logf("skip output format %q for site %s not matching segments set in --renderSegments", renderFormat.Name, s.resolveDimensionNames()) continue } @@ -425,7 +448,7 @@ func (h *HugoSites) render(l logg.LevelLogger, config *BuildCfg) error { case <-h.Done(): return nil default: - for _, s2 := range h.Sites { + for s2 := range h.allSites(nil) { if err := s2.preparePagesForRender(s == s2, siteRenderContext.sitesOutIdx); err != nil { return err } @@ -455,6 +478,7 @@ func (h *HugoSites) render(l logg.LevelLogger, config *BuildCfg) error { } } + } return nil @@ -567,7 +591,7 @@ func (s *Site) executeDeferredTemplates(de *deps.DeferredExecutions) error { return nil } - g := rungroup.Run[string](context.Background(), rungroup.Config[string]{ + g := rungroup.Run(context.Background(), rungroup.Config[string]{ NumWorkers: s.h.numWorkers, Handle: func(ctx context.Context, filename string) error { return handleFile(filename) @@ -902,7 +926,7 @@ func (h *HugoSites) processPartialFileEvents(ctx context.Context, l logg.LevelLo } // Compile cache buster. - np := glob.NormalizePath(path.Join(cps.Component, cps.Path)) + np := hglob.NormalizePath(path.Join(cps.Component, cps.Path)) g, err := h.ResourceSpec.BuildConfig().MatchCacheBuster(h.Log, np) if err == nil && g != nil { cacheBusters = append(cacheBusters, g) @@ -952,9 +976,9 @@ func (h *HugoSites) processPartialFileEvents(ctx context.Context, l logg.LevelLo // Remove all pages and resources below. prefix := paths.AddTrailingSlash(pathInfo.Base()) - h.pageTrees.treePages.DeletePrefixAll(prefix) - h.pageTrees.resourceTrees.DeletePrefixAll(prefix) - changes = append(changes, identity.NewGlobIdentity(prefix+"**")) + h.pageTrees.treePages.DeletePrefixRaw(prefix) + h.pageTrees.resourceTrees.DeletePrefixRaw(prefix) + changes = append(changes, hglob.NewGlobIdentity(prefix+"**")) } return err != nil }) @@ -975,17 +999,17 @@ func (h *HugoSites) processPartialFileEvents(ctx context.Context, l logg.LevelLo h.pageTrees.treeTaxonomyEntries.DeletePrefix("") if delete && !isContentDataFile { - _, ok := h.pageTrees.treePages.LongestPrefixAll(pathInfo.Base()) + _, ok := h.pageTrees.treePages.LongestPrefixRaw(pathInfo.Base()) if ok { - h.pageTrees.treePages.DeleteAll(pathInfo.Base()) - h.pageTrees.resourceTrees.DeleteAll(pathInfo.Base()) + h.pageTrees.treePages.DeletePrefixRaw(pathInfo.Base()) + h.pageTrees.resourceTrees.DeletePrefixRaw(pathInfo.Base()) if pathInfo.IsBundle() { // Assume directory removed. - h.pageTrees.treePages.DeletePrefixAll(pathInfo.Base() + "/") - h.pageTrees.resourceTrees.DeletePrefixAll(pathInfo.Base() + "/") + h.pageTrees.treePages.DeletePrefixRaw(pathInfo.Base() + "/") + h.pageTrees.resourceTrees.DeletePrefixRaw(pathInfo.Base() + "/") } } else { - h.pageTrees.resourceTrees.DeleteAll(pathInfo.Base()) + h.pageTrees.resourceTrees.DeletePrefixRaw(pathInfo.Base()) } } @@ -1011,7 +1035,7 @@ func (h *HugoSites) processPartialFileEvents(ctx context.Context, l logg.LevelLo changes = append(changes, identity.GenghisKhan) } if strings.Contains(base, "shortcodes") { - changes = append(changes, identity.NewGlobIdentity(fmt.Sprintf("shortcodes/%s*", pathInfo.BaseNameNoIdentifier()))) + changes = append(changes, hglob.NewGlobIdentity(fmt.Sprintf("shortcodes/%s*", pathInfo.BaseNameNoIdentifier()))) } else { changes = append(changes, pathInfo) } @@ -1252,15 +1276,37 @@ func (s *Site) handleContentAdapterChanges(bi pagesfromdata.BuildInfo, buildConf } for _, p := range bi.DeletedPaths { - pp := path.Join(bi.Path.Base(), p) - if v, ok := s.pageMap.treePages.Delete(pp); ok { - buildConfig.WhatChanged.Add(v.GetIdentity()) + pp := paths.AddLeadingSlash(path.Join(bi.Path.Base(), p.Path)) + df := func(n contentNode) bool { + if len(p.Hashes) == 0 { + return true + } + if nn, ok := n.(contentNodeSourceEntryIDProvider); ok { + if i, ok := nn.nodeSourceEntryID().(uint64); ok && i != 0 { + if _, found := p.Hashes[i]; found { + return true + } + } + } + return false + } + + if v, count := s.pageMap.treePages.DeleteFuncRaw(pp, df); count > 0 { + buildConfig.WhatChanged.Add(cnh.GetIdentity(v)) + } + if v, count := s.pageMap.treeResources.DeleteFuncRaw(pp, df); count > 0 { + buildConfig.WhatChanged.Add(cnh.GetIdentity(v)) + + // A deleted resource may affect its parent page. + if _, v := s.pageMap.treePages.LongestPrefiValueRaw(pp); v != nil { + buildConfig.WhatChanged.Add(cnh.GetIdentity(v)) + } } } } func (h *HugoSites) processContentAdaptersOnRebuild(ctx context.Context, buildConfig *BuildCfg) error { - g := rungroup.Run[*pagesfromdata.PagesFromTemplate](ctx, rungroup.Config[*pagesfromdata.PagesFromTemplate]{ + g := rungroup.Run(ctx, rungroup.Config[*pagesfromdata.PagesFromTemplate]{ NumWorkers: h.numWorkers, Handle: func(ctx context.Context, p *pagesfromdata.PagesFromTemplate) error { bi, err := p.Execute(ctx) diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 4c2bf452c..4d13e3617 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -203,94 +203,6 @@ func checkContent(s *sitesBuilder, filename string, matches ...string) { } } -func TestTranslationsFromContentToNonContent(t *testing.T) { - b := newTestSitesBuilder(t) - b.WithConfigFile("toml", ` - -baseURL = "http://example.com/" - -defaultContentLanguage = "en" - -[languages] -[languages.en] -weight = 10 -contentDir = "content/en" -[languages.nn] -weight = 20 -contentDir = "content/nn" - - -`) - - b.WithContent("en/mysection/_index.md", ` ---- -Title: My Section ---- - -`) - - b.WithContent("en/_index.md", ` ---- -Title: My Home ---- - -`) - - b.WithContent("en/categories/mycat/_index.md", ` ---- -Title: My MyCat ---- - -`) - - b.WithContent("en/categories/_index.md", ` ---- -Title: My categories ---- - -`) - - for _, lang := range []string{"en", "nn"} { - b.WithContent(lang+"/mysection/page.md", ` ---- -Title: My Page -categories: ["mycat"] ---- - -`) - } - - b.Build(BuildCfg{}) - - for _, path := range []string{ - "/", - "/mysection", - "/categories", - "/categories/mycat", - } { - t.Run(path, func(t *testing.T) { - c := qt.New(t) - - s1, _ := b.H.Sites[0].getPage(nil, path) - s2, _ := b.H.Sites[1].getPage(nil, path) - - c.Assert(s1, qt.Not(qt.IsNil)) - c.Assert(s2, qt.Not(qt.IsNil)) - - c.Assert(len(s1.Translations()), qt.Equals, 1) - c.Assert(len(s2.Translations()), qt.Equals, 1) - c.Assert(s1.Translations()[0], qt.Equals, s2) - c.Assert(s2.Translations()[0], qt.Equals, s1) - - m1 := s1.Translations().MergeByLanguage(s2.Translations()) - m2 := s2.Translations().MergeByLanguage(s1.Translations()) - - c.Assert(len(m1), qt.Equals, 1) - c.Assert(len(m2), qt.Equals, 1) - }) - } -} - func writeSource(t testing.TB, fs *hugofs.Fs, filename, content string) { t.Helper() writeToFs(t, fs.Source, filename, content) diff --git a/hugolib/hugo_sites_multihost_test.go b/hugolib/hugo_sites_multihost_test.go index 37f7ab927..738aa4b34 100644 --- a/hugolib/hugo_sites_multihost_test.go +++ b/hugolib/hugo_sites_multihost_test.go @@ -107,9 +107,11 @@ robots|{{ site.Language.Lang }} b.AssertFileContent("public/fr/mysect/mybundle/index.html", "Foo: https://example.fr/mysect/mybundle/foo.1cbec737f863e4922cee63cc2ebbfaafcd1cff8b790d8cfd2e6a5d550b648afa.txt|") // Assets CSS fingerprinted - b.AssertFileContent("public/en/mysect/mybundle/index.html", "CSS: https://example.fr/css/main.5de625c36355cce7c1d5408826a0b21abfb49fb6c0e1f16c945a6f2aef38200c.css|") + // Note that before Hugo v0.149.0 we rendered the project starting with defaultContentLanguage, + // now with a more complex matrix, we have one sort order. + b.AssertFileContent("public/en/mysect/mybundle/index.html", "CSS: https://example.com/docs/css/main.5de625c36355cce7c1d5408826a0b21abfb49fb6c0e1f16c945a6f2aef38200c.css|") b.AssertFileContent("public/en/css/main.5de625c36355cce7c1d5408826a0b21abfb49fb6c0e1f16c945a6f2aef38200c.css", "body { color: red; }") - b.AssertFileContent("public/fr/mysect/mybundle/index.html", "CSS: https://example.fr/css/main.5de625c36355cce7c1d5408826a0b21abfb49fb6c0e1f16c945a6f2aef38200c.css|") + b.AssertFileContent("public/fr/mysect/mybundle/index.html", "CSS: https://example.com/docs/css/main.5de625c36355cce7c1d5408826a0b21abfb49fb6c0e1f16c945a6f2aef38200c.css|") b.AssertFileContent("public/fr/css/main.5de625c36355cce7c1d5408826a0b21abfb49fb6c0e1f16c945a6f2aef38200c.css", "body { color: red; }") } @@ -210,7 +212,7 @@ title: mybundle-en b.AssertFileExists("public/en/mybundle/pixel_hu_58204cbc58507d74.png", true) } -func TestMultihostResourceOneBaseURLWithSuPath(t *testing.T) { +func TestMultihostResourceOneBaseURLWithSubPath(t *testing.T) { files := ` -- hugo.toml -- defaultContentLanguage = "en" diff --git a/hugolib/hugo_sites_test.go b/hugolib/hugo_sites_test.go index 5e1a1504c..6b076ee71 100644 --- a/hugolib/hugo_sites_test.go +++ b/hugolib/hugo_sites_test.go @@ -53,6 +53,32 @@ title: "Bundle De" "Bundle all translations: en|fr|de|$", "Bundle translations: fr|de|$", "Site languages: en|fr|de|$", - "Sites: fr|en|de|$", + "Sites: en|fr|de|$", ) } + +func TestSitesOrder(t *testing.T) { + files := ` +-- hugo.toml -- +defaultContentLanguage = "fr" +defaultContentLanguageInSubdir = true +[languages] +[languages.en] +weight = 1 +[languages.fr] +weight = 2 +[languages.de] +weight = 3 +[roles] +[roles.guest] +weight = 1 +[roles.member] +weight = 2 +-- layouts/home.html -- +Sites: {{ range site.Sites }}{{ .Language.Lang }}|{{ end }}$ +Languages: {{ range site.Languages }}{{ .Lang }}|{{ end }} +` + b := Test(t, files) + // Note that before v0.152.0 the order of these slices where different .Sites pulled the defaultContentLanguage first. + b.AssertFileContent("public/en/index.html", "Sites: en|fr|de|$", "Languages: en|fr|de|") +} diff --git a/hugolib/hugo_smoke_test.go b/hugolib/hugo_smoke_test.go index 09d57bbff..c6508bc04 100644 --- a/hugolib/hugo_smoke_test.go +++ b/hugolib/hugo_smoke_test.go @@ -49,6 +49,260 @@ Home: {{ .Title }} b.AssertFileContent("public/index.html", `Hello`) } +func TestSmoke202509(t *testing.T) { + t.Parallel() + + // Test variants: + // Site with two languages, one with home page content and one without. + // A common translated page bundle but with different dates. + // A text resource in one of the languages. + // Date aggregation. + // A content resource in one of the languages. + // Basic shortcode usage with templates in both languages. + // Test Rotate the language dimension. + // The same content page mounted for all languages. + // RenderString with shortcode. + + files := ` +-- hugo.toml -- +baseURL = "https://example.com" +defaultContentLanguage = "en" +defaultContentLanguageInSubdir = true +[languages.en] +title = "Site title en" +weight = 200 +[languages.nn] +title = "Site title nn" +weight = 100 +[[module.mounts]] +source = 'content/en' +target = 'content' +lang = 'en' +[[module.mounts]] +source = 'content/nn' +target = 'content' +lang = 'nn' +[[module.mounts]] +source = 'content/all' +target = 'content' +[module.mounts.sites.matrix] +languages = ["**"] +-- content/en/_index.md -- +--- +title: "Home in English" +--- +Home Content. +-- content/en/mysection/p1/mytext.txt -- +This is a text resource in English. +-- content/en/mysection/p1/mypage.md -- +--- +title: "mypage en" +--- +mypage en content. +-- content/en/mysection/p1/index.md -- +--- +title: "p1 en" +date: 2023-10-01 +--- +Content p1 en. + +{{< myshortcode >}} +-- content/nn/mysection/p1/index.md -- +--- +title: "p1 nn" +date: 2024-10-01 +--- +Content p1 nn. + +{{< myshortcode >}} +-- content/all/mysection/p2/index.md -- +--- +title: "p2 all" +date: 2022-10-01 +--- +Content p2 all. + +{{< myshortcode >}} +-- layouts/all.html -- +All. {{ .Title }}|Lastmod: {{ .Lastmod.Format "2006-01-02" }}| +Kind: {{ .Kind }}| +Content: {{ .Content }}| +CurrentSection: {{ .CurrentSection.PathInfo.Path }}| +Parent: {{ with .Parent }}{{ .RelPermalink }}{{ end }}| +Home: {{ .Site.Home.Title }}| +Rotate(language): {{ range .Rotate "language" }}{{ .Lang }}|{{ .Title }}|{{ end }}| +mytext.txt: {{ with .Resources.GetMatch "**.txt" }}{{ .Content }}|{{ .RelPermalink }}{{ end }}| +mypage.md: {{ with .Resources.GetMatch "**.md" }}{{ .Content }}|{{ .RelPermalink }}{{ end }}| +RenderString with shortcode: {{ .RenderString "{{< myshortcode >}}" }}| +-- layouts/shortcodes/myshortcode.html -- +myshortcode.html +-- layouts/shortcodes/myshortcode.en.html -- +myshortcode.en.html +` + + b := Test(t, files) + + b.AssertFileContent("public/en/index.html", + "All. Home in English|", // from content file. + "Kind: home|", + "Lastmod: 2023-10-01", + "RenderString with shortcode: myshortcode.en.html", + "Parent: |", + ) + b.AssertFileContent("public/nn/index.html", + "Site title nn|", // from site config. + "Lastmod: 2024-10-01", + "RenderString with shortcode: myshortcode.html", + ) + + b.AssertFileContent("public/nn/mysection/p1/index.html", + "p1 nn|Lastmod: 2024-10-01|\nRotate(language): nn|p1 nn|en|p1 en||", + "mytext.txt: This is a text resource in English.|/en/mysection/p1/mytext.txt|", + "Content p1 nn.", + "mypage.md: |", + "myshortcode.html", + ) + + b.AssertFileContent("public/en/mysection/p1/index.html", + "p1 en|Lastmod: 2023-10-01|\nRotate(language): nn|p1 nn|en|p1 en||", + "mytext.txt: This is a text resource in English.|/en/mysection/p1/mytext.txt|", + "mypage.md:

mypage en content.

", + "Content p1 en.", + "myshortcode.en.html", + "RenderString with shortcode: myshortcode.en.html", + ) + + b.AssertFileContent("public/nn/mysection/p2/index.html", + "myshortcode.html", + "RenderString with shortcode: myshortcode.html", + ) + + b.AssertFileContent("public/en/mysection/p2/index.html", + "myshortcode.en.html", + "RenderString with shortcode: myshortcode.en.html", + ) +} + +func TestSmokeTaxonomies202509(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ["rss", "sitemap", "robotsTXT"] +baseURL = "https://example.com" +defaultContentLanguage = "en" +defaultContentLanguageInSubdir = true +[taxonomies] +category = "categories" +tag = "tags" +[languages.en] +title = "Site title en" +weight = 200 +[languages.nn] +title = "Site title nn" +weight = 100 +[languages.nn.taxonomies] +tag = "tags" +foo = "foos" +[module] +[[module.mounts]] +source = 'content/en' +target = 'content' +[module.mounts.sites.matrix] +languages = 'en' +[[module.mounts]] +source = 'content/nn' +target = 'content' +[module.mounts.sites.matrix] +languages = 'nn' +-- content/en/p1.md -- +--- +title: "p1 en" +date: 2023-10-01 +tags: ["tag1", "tag2"] +categories: ["cat1"] +foos: ["foo2"] +--- +Content p1 en. +-- content/nn/p1.md -- +--- +title: "p1 nn" +date: 2024-10-01 +tags: ["tag1", "tag3"] +categories: ["cat1", "cat2"] +foos: ["foo1"] +--- +-- layouts/all.html -- +All. {{ .Title }}|{{ .Kind }}| +GetTerms tags: {{ range .GetTerms "tags" }}{{ .Name }}|{{ end }}$ +GetTerms categories: {{ range .GetTerms "categories" }}{{ .Name }}|{{ end }}$ +GetTerms foos: {{ range .GetTerms "foos" }}{{ .Name }}|{{ end }}$ +` + + b := Test(t, files) + + b.AssertFileContent("public/en/p1/index.html", "p1 en", "GetTerms tags: tag1|tag2|$", "GetTerms categories: cat1|$", "GetTerms foos: $") + b.AssertFileContent("public/nn/p1/index.html", "p1 nn", "GetTerms tags: tag1|tag3|$", "GetTerms categories: $", "GetTerms foos: foo1|$") + + b.AssertFileContent("public/en/tags/index.html", "All. Tags|taxonomy|") + b.AssertFileContent("public/nn/tags/tag1/index.html", "All. Tag1|term|") + b.AssertFileContent("public/en/tags/tag1/index.html", "All. Tag1|term|") +} + +func TestSmokeEdits202509(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +baseURL = "https://example.com" +disableKinds = ["term", "taxonomy"] +disableLiveReload = true +defaultContentLanguage = "en" +defaultContentLanguageInSubdir = true +[languages.en] +title = "Site title en" +weight = 200 +[languages.nn] +title = "Site title nn" +weight = 100 +[module] +[[module.mounts]] +source = 'content/en' +target = 'content' +[module.mounts.sites.matrix] +languages = ['en'] +[[module.mounts]] +source = 'content/nn' +target = 'content' +[module.mounts.sites.matrix] +languages = ['nn'] +-- content/en/p1/index.md -- +--- +title: "p1 en" +date: 2023-10-01 +--- +Content p1 en. +-- content/nn/p1/index.md -- +--- +title: "p1 nn" +date: 2024-10-01 +--- +Content p1 nn. +-- layouts/all.html -- +All. {{ .Title }}|Lastmod: {{ .Lastmod.Format "2006-01-02" }}|Content: {{ .Content }}| + +` + + b := TestRunning(t, files) + + // b.AssertPublishDir("sDF") + b.AssertFileContent("public/en/p1/index.html", "All. p1 en|") + b.AssertFileContent("public/nn/p1/index.html", "All. p1 nn|") + b.EditFileReplaceAll("public/en/p1/index.html", "p1 en", "p1 en edited").Build() + b.AssertFileContent("public/en/p1/index.html", "All. p1 en edited") + b.EditFileReplaceAll("public/nn/p1/index.html", "p1 nn", "p1 nn|").Build() +} + func TestSmokeOutputFormats(t *testing.T) { t.Parallel() @@ -289,7 +543,7 @@ Content Tag 1. b.AssertFileContent("public/en/posts/p1/index.html", "Single: en|page|/en/posts/p1/|Post 1|

Content 1.

\n|Len Resources: 2|", - "Resources: text|/en/posts/p1/f1.txt|text/plain|map[icon:enicon] - page||application/octet-stream|map[draft:false iscjklanguage:false title:Post Sub 1] -", + "Resources: text|/en/posts/p1/f1.txt|text/plain|map[icon:enicon] - page||application/octet-stream|map[background:post.jpg draft:false iscjklanguage:false title:Post Sub 1] -", "Icon: enicon", "Icon fingerprinted: enicon|/en/posts/p1/f1.e5746577af5cbfc4f34c558051b7955a9a5a795a84f1c6ab0609cb3473a924cb.txt|", "NextInSection: |\nPrevInSection: /en/posts/p2/|Post 2|", diff --git a/hugolib/integrationtest_builder.go b/hugolib/integrationtest_builder.go index f28407fa1..210a53d32 100644 --- a/hugolib/integrationtest_builder.go +++ b/hugolib/integrationtest_builder.go @@ -33,6 +33,8 @@ import ( "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/hugofs/hglob" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "github.com/spf13/afero" "github.com/spf13/cast" "golang.org/x/text/unicode/norm" @@ -217,6 +219,82 @@ type IntegrationTestBuilder struct { builderInit sync.Once } +type IntegrationTestSiteHelper struct { + *qt.C + b *IntegrationTestBuilder + S *Site +} + +type IntegrationTestPageHelper struct { + *qt.C + b *IntegrationTestBuilder + p *pageState +} + +func (s *IntegrationTestPageHelper) siteIntsToMap(matrix, complements sitesmatrix.VectorStore) map[string]map[string][]string { + dconf := s.p.s.Conf.ConfiguredDimensions() + intSetsToMap := func(intSets sitesmatrix.VectorStore) map[string][]string { + var languages, versions, roles []string + + keys1, keys2, keys3 := intSets.KeysSorted() + + for _, v := range keys1 { + languages = append(languages, dconf.ConfiguredLanguages.ResolveName(v)) + } + for _, v := range keys2 { + versions = append(versions, dconf.ConfiguredVersions.ResolveName(v)) + } + for _, v := range keys3 { + roles = append(roles, dconf.ConfiguredRoles.ResolveName(v)) + } + + return map[string][]string{ + "languages": languages, + "versions": versions, + "roles": roles, + } + } + + return map[string]map[string][]string{ + "matrix": intSetsToMap(matrix), + "complements": intSetsToMap(complements), + } +} + +func (s *IntegrationTestPageHelper) MatrixFromPageConfig() map[string]map[string][]string { + pc := s.p.m.pageConfigSource + return s.siteIntsToMap(pc.SitesMatrix, pc.SitesComplements) +} + +func (s *IntegrationTestPageHelper) MatrixFromFile() map[string]map[string][]string { + if s.p.m.f == nil { + return nil + } + m := s.p.m.f.FileInfo().Meta() + return s.siteIntsToMap(m.SitesMatrix, m.SitesComplements) +} + +func (s *IntegrationTestSiteHelper) PageHelper(path string) *IntegrationTestPageHelper { + p, err := s.S.GetPage(path) + s.Assert(err, qt.IsNil) + s.Assert(p, qt.Not(qt.IsNil), qt.Commentf("Page not found: %s", path)) + ps, ok := p.(*pageState) + s.Assert(ok, qt.IsTrue, qt.Commentf("Expected pageState, got %T", p)) + return &IntegrationTestPageHelper{ + C: s.C, + b: s.b, + p: ps, + } +} + +func (s *IntegrationTestSiteHelper) DimensionNames() types.Strings3 { + return types.Strings3{ + s.S.Language().Name(), + s.S.Version().Name(), + s.S.Role().Name(), + } +} + type lockingBuffer struct { sync.Mutex buf bytes.Buffer @@ -248,8 +326,28 @@ func (b *lockingBuffer) Write(p []byte) (n int, err error) { return } +// SiteHelper returns a helper for the given language, version and role. +// Note that a blank value for an argument will use the default value for that dimension. +func (s *IntegrationTestBuilder) SiteHelper(language, version, role string) *IntegrationTestSiteHelper { + s.Helper() + if s.H == nil { + s.Fatal("SiteHelper: no sites available") + } + v := s.H.Conf.ConfiguredDimensions().ResolveVector(types.Strings3{language, version, role}) + site, found := s.H.sitesVersionsRolesMap[v] + if !found { + s.Fatalf("SiteHelper: no site found for vector %v", v) + } + + return &IntegrationTestSiteHelper{ + C: s.C, + b: s, + S: site, + } +} + // AssertLogContains asserts that the last build log contains the given strings. -// Each string can be negated with a "! " prefix. +// Each string can be negated with a hglob.NegationPrefix prefix. func (s *IntegrationTestBuilder) AssertLogContains(els ...string) { s.Helper() for _, el := range els { @@ -264,7 +362,7 @@ func (s *IntegrationTestBuilder) AssertLogContains(els ...string) { } // AssertLogMatches asserts that the last build log matches the given regular expressions. -// The regular expressions can be negated with a "! " prefix. +// The regular expressions can be negated with a hglob.NegationPrefix prefix. func (s *IntegrationTestBuilder) AssertLogMatches(expression string) { s.Helper() var negate bool @@ -278,16 +376,6 @@ func (s *IntegrationTestBuilder) AssertLogMatches(expression string) { s.Assert(re.MatchString(s.lastBuildLog), checker, qt.Commentf(s.lastBuildLog)) } -func (s *IntegrationTestBuilder) AssertBuildCountData(count int) { - s.Helper() - s.Assert(s.H.init.data.InitCount(), qt.Equals, count) -} - -func (s *IntegrationTestBuilder) AssertBuildCountGitInfo(count int) { - s.Helper() - s.Assert(s.H.init.gitInfo.InitCount(), qt.Equals, count) -} - func (s *IntegrationTestBuilder) AssertFileCount(dirname string, expected int) { s.Helper() fs := s.fs.WorkingDirReadOnly @@ -307,9 +395,9 @@ func (s *IntegrationTestBuilder) AssertFileCount(dirname string, expected int) { func (s *IntegrationTestBuilder) negate(match string) (string, bool) { var negate bool - if strings.HasPrefix(match, "! ") { + if strings.HasPrefix(match, hglob.NegationPrefix) { negate = true - match = strings.TrimPrefix(match, "! ") + match = strings.TrimPrefix(match, hglob.NegationPrefix) } return match, negate } @@ -319,7 +407,7 @@ func (s *IntegrationTestBuilder) AssertFileContent(filename string, matches ...s content := strings.TrimSpace(s.FileContent(filename)) for _, m := range matches { - cm := qt.Commentf("File: %s Match %s\nContent:\n%s", filename, m, content) + cm := qt.Commentf("File: %s Expect: %s Got: %s", filename, m, content) lines := strings.Split(m, "\n") for _, match := range lines { match = strings.TrimSpace(match) @@ -379,9 +467,9 @@ func (s *IntegrationTestBuilder) AssertFs(fs afero.Fs, matches ...string) { for _, match := range lines { match = strings.TrimSpace(match) var negate bool - if strings.HasPrefix(match, "! ") { + if strings.HasPrefix(match, hglob.NegationPrefix) { negate = true - match = strings.TrimPrefix(match, "! ") + match = strings.TrimPrefix(match, hglob.NegationPrefix) } if negate { s.Assert(content, qt.Not(qt.Contains), match, cm) @@ -425,9 +513,11 @@ func (s *IntegrationTestBuilder) AssertFileExists(filename string, b bool) { if !b { checker = qt.IsNotNil } + _, err := s.fs.WorkingDirReadOnly.Stat(filename) - if !herrors.IsNotExist(err) { + if err != nil && !herrors.IsNotExist(err) { s.Assert(err, qt.IsNil) + return } s.Assert(err, checker) } @@ -456,13 +546,17 @@ func (s *IntegrationTestBuilder) AssertRenderCountPageBetween(from, to int) { func (s *IntegrationTestBuilder) Build() *IntegrationTestBuilder { s.Helper() _, err := s.BuildE() + + if err != nil && strings.Contains(err.Error(), "error(s)") { + err = fmt.Errorf("%w: %s", err, s.lastBuildLog) + } + if s.Cfg.Verbose || err != nil { - fmt.Println(s.lastBuildLog) if s.H != nil && err == nil { - for _, s := range s.H.Sites { + for s := range s.H.allSites(nil) { m := s.pageMap var buff bytes.Buffer - fmt.Fprintf(&buff, "PageMap for site %q\n\n", s.Language().Lang) + fmt.Fprintf(&buff, "======= PageMap for site %q =======\n\n", s.resolveDimensionNames()) m.debugPrint("", 999, &buff) fmt.Println(buff.String()) } @@ -591,6 +685,15 @@ func (s *IntegrationTestBuilder) RemoveFiles(filenames ...string) *IntegrationTe return s } +func (s *IntegrationTestBuilder) RemovePublishDir() *IntegrationTestBuilder { + s.Helper() + if err := s.fs.PublishDir.RemoveAll(""); err != nil && !herrors.IsNotExist(err) { + s.Fatalf("Failed to remove publish dir: %s", err) + } + + return s +} + func (s *IntegrationTestBuilder) RenameFile(old, new string) *IntegrationTestBuilder { absOldFilename := s.absFilename(old) absNewFilename := s.absFilename(new) @@ -693,8 +796,8 @@ func (s *IntegrationTestBuilder) initBuilder() error { } var w io.Writer - if s.Cfg.LogLevel == logg.LevelTrace { - w = os.Stdout + if s.Cfg.Verbose || s.Cfg.LogLevel == logg.LevelTrace { + w = io.MultiWriter(os.Stdout, &s.logBuff) } else { w = &s.logBuff } @@ -939,7 +1042,7 @@ type IntegrationTestConfig struct { // Note that the CLI for the server does allow for --watch=false, but that is not used in these test. Watching bool - // Will print the log buffer after the build + // Enable verbose logging. Verbose bool // The log level to use. diff --git a/hugolib/language_content_dir_test.go b/hugolib/language_content_dir_test.go index 27039400c..acc0d60ae 100644 --- a/hugolib/language_content_dir_test.go +++ b/hugolib/language_content_dir_test.go @@ -237,6 +237,6 @@ title: p1 (es) b.AssertFileExists("public/es/p1/index.html", true) b.AssertFileExists("public/es/p2/index.html", true) - b.AssertLogContains("INFO Duplicate") - b.AssertLogContains("! WARN Duplicate") + // This assertion was changed for 0.152.0. It was no longer possible/practical to warn about duplicate content paths. + b.AssertLogContains("! Duplicate") } diff --git a/hugolib/page.go b/hugolib/page.go index 9e7ec22c3..5902cbf79 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -16,14 +16,16 @@ package hugolib import ( "context" "fmt" + "iter" "path/filepath" + "slices" "strconv" "strings" "sync/atomic" "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/hugolib/doctree" "github.com/gohugoio/hugo/hugolib/segments" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/output" @@ -52,7 +54,6 @@ var ( _ collections.Grouper = (*pageState)(nil) _ collections.Slicer = (*pageState)(nil) _ identity.DependencyManagerScopedProvider = (*pageState)(nil) - _ contentNodeI = (*pageState)(nil) _ pageContext = (*pageState)(nil) ) @@ -95,12 +96,14 @@ type pageState struct { // Note that this will change between builds for a given Page. pid uint64 + s *Site + // This slice will be of same length as the number of global slice of output // formats (for all sites). pageOutputs []*pageOutput // Used to determine if we can reuse content across output formats. - pageOutputTemplateVariationsState *atomic.Uint32 + pageOutputTemplateVariationsState atomic.Uint32 // This will be shifted out when we start to render a new output format. pageOutputIdx int @@ -128,48 +131,56 @@ func (p *pageState) incrPageOutputTemplateVariation() { p.pageOutputTemplateVariationsState.Add(1) } -func (p *pageState) canReusePageOutputContent() bool { - return p.pageOutputTemplateVariationsState.Load() == 1 +func (ps *pageState) canReusePageOutputContent() bool { + return ps.pageOutputTemplateVariationsState.Load() == 1 } -func (p *pageState) IdentifierBase() string { - return p.Path() +func (ps *pageState) IdentifierBase() string { + return ps.Path() } -func (p *pageState) GetIdentity() identity.Identity { - return p +func (ps *pageState) GetIdentity() identity.Identity { + return ps } -func (p *pageState) ForEeachIdentity(f func(identity.Identity) bool) bool { - return f(p) +func (ps *pageState) ForEeachIdentity(f func(identity.Identity) bool) bool { + return f(ps) } -func (p *pageState) GetDependencyManager() identity.Manager { - return p.dependencyManager +func (ps *pageState) GetDependencyManager() identity.Manager { + return ps.dependencyManager } -func (p *pageState) GetDependencyManagerForScope(scope int) identity.Manager { +func (ps *pageState) GetDependencyManagerForScope(scope int) identity.Manager { switch scope { case pageDependencyScopeDefault: - return p.dependencyManagerOutput + return ps.dependencyManagerOutput case pageDependencyScopeGlobal: - return p.dependencyManager + return ps.dependencyManager default: return identity.NopManager } } -func (p *pageState) GetDependencyManagerForScopesAll() []identity.Manager { - return []identity.Manager{p.dependencyManager, p.dependencyManagerOutput} +func (ps *pageState) GetDependencyManagerForScopesAll() []identity.Manager { + return []identity.Manager{ps.dependencyManager, ps.dependencyManagerOutput} +} + +// Param is a convenience method to do lookups in Page's and Site's Params map, +// in that order. +// +// This method is also implemented on SiteInfo. +func (ps *pageState) Param(key any) (any, error) { + return resource.Param(ps, ps.s.Params(), key) } -func (p *pageState) Key() string { - return "page-" + strconv.FormatUint(p.pid, 10) +func (ps *pageState) Key() string { + return "page-" + strconv.FormatUint(ps.pid, 10) } // RelatedKeywords implements the related.Document interface needed for fast page searches. -func (p *pageState) RelatedKeywords(cfg related.IndexConfig) ([]related.Keyword, error) { - v, found, err := page.NamedPageMetaValue(p, cfg.Name) +func (ps *pageState) RelatedKeywords(cfg related.IndexConfig) ([]related.Keyword, error) { + v, found, err := page.NamedPageMetaValue(ps, cfg.Name) if err != nil { return nil, err } @@ -181,25 +192,25 @@ func (p *pageState) RelatedKeywords(cfg related.IndexConfig) ([]related.Keyword, return cfg.ToKeywords(v) } -func (p *pageState) resetBuildState() { - // Nothing to do for now. +func (ps *pageState) resetBuildState() { + ps.m.prepareRebuild() } -func (p *pageState) skipRender() bool { - b := p.s.conf.C.SegmentFilter.ShouldExcludeFine( - segments.SegmentMatcherFields{ - Path: p.Path(), - Kind: p.Kind(), - Lang: p.Lang(), - Output: p.pageOutput.f.Name, +func (ps *pageState) skipRender() bool { + b := ps.s.conf.Segments.Config.SegmentFilter.ShouldExcludeFine( + segments.SegmentQuery{ + Path: ps.Path(), + Kind: ps.Kind(), + Site: ps.s.siteVector, + Output: ps.pageOutput.f.Name, }, ) return b } -func (po *pageState) isRenderedAny() bool { - for _, o := range po.pageOutputs { +func (ps *pageState) isRenderedAny() bool { + for _, o := range ps.pageOutputs { if o.isRendered() { return true } @@ -207,22 +218,79 @@ func (po *pageState) isRenderedAny() bool { return false } -func (p *pageState) isContentNodeBranch() bool { - return p.IsNode() +// Implements contentNode. + +func (ps *pageState) forEeachContentNode(f func(v sitesmatrix.Vector, n contentNode) bool) bool { + return f(ps.s.siteVector, ps) +} + +func (ps *pageState) contentWeight() int { + if ps.m.f == nil { + return 0 + } + return ps.m.f.FileInfo().Meta().Weight +} + +func (ps *pageState) nodeSourceEntryID() any { + return ps.m.nodeSourceEntryID() +} + +func (ps *pageState) nodeCategoryPage() { + // Marker method. +} + +func (m *pageState) nodeCategorySingle() { + // Marker method. +} + +func (ps *pageState) lookupContentNode(v sitesmatrix.Vector) contentNode { + pc := ps.m.pageConfigSource + if pc.MatchSiteVector(v) { + return ps + } + return nil +} + +func (ps *pageState) lookupContentNodes(vec sitesmatrix.Vector, fallback bool) iter.Seq[contentNodeForSite] { + nop := func(yield func(n contentNodeForSite) bool) {} + pc := ps.m.pageConfigSource + if !fallback { + if !pc.MatchSiteVector(vec) { + return nop + } + return func(yield func(n contentNodeForSite) bool) { + yield(ps) + } + } + + if !pc.MatchLanguageCoarse(vec) { + return nop + } + + if !pc.MatchVersionCoarse(vec) { + return nop + } + if !pc.MatchRoleCoarse(vec) { + return nop + } + + return func(yield func(n contentNodeForSite) bool) { + yield(ps) + } } // Eq returns whether the current page equals the given page. // This is what's invoked when doing `{{ if eq $page $otherPage }}` -func (p *pageState) Eq(other any) bool { +func (ps *pageState) Eq(other any) bool { pp, err := unwrapPage(other) if err != nil { return false } - return p == pp + return ps == pp } -func (p *pageState) HeadingsFiltered(context.Context) tableofcontents.Headings { +func (ps *pageState) HeadingsFiltered(context.Context) tableofcontents.Headings { return nil } @@ -240,204 +308,205 @@ func (p *pageHeadingsFiltered) page() page.Page { } // For internal use by the related content feature. -func (p *pageState) ApplyFilterToHeadings(ctx context.Context, fn func(*tableofcontents.Heading) bool) related.Document { - fragments := p.pageOutput.pco.c().Fragments(ctx) +func (ps *pageState) ApplyFilterToHeadings(ctx context.Context, fn func(*tableofcontents.Heading) bool) related.Document { + fragments := ps.pageOutput.pco.c().Fragments(ctx) headings := fragments.Headings.FilterBy(fn) return &pageHeadingsFiltered{ - pageState: p, + pageState: ps, headings: headings, } } -func (p *pageState) GitInfo() *source.GitInfo { - return p.gitInfo +func (ps *pageState) GitInfo() *source.GitInfo { + return ps.gitInfo } -func (p *pageState) CodeOwners() []string { - return p.codeowners +func (ps *pageState) CodeOwners() []string { + return ps.codeowners } // GetTerms gets the terms defined on this page in the given taxonomy. // The pages returned will be ordered according to the front matter. -func (p *pageState) GetTerms(taxonomy string) page.Pages { - return p.s.pageMap.getTermsForPageInTaxonomy(p.Path(), taxonomy) +func (ps *pageState) GetTerms(taxonomy string) page.Pages { + return ps.s.pageMap.getTermsForPageInTaxonomy(ps.Path(), taxonomy) } -func (p *pageState) MarshalJSON() ([]byte, error) { - return page.MarshalPageToJSON(p) +func (ps *pageState) MarshalJSON() ([]byte, error) { + return page.MarshalPageToJSON(ps) } -func (p *pageState) RegularPagesRecursive() page.Pages { - switch p.Kind() { +func (ps *pageState) RegularPagesRecursive() page.Pages { + switch ps.Kind() { case kinds.KindSection, kinds.KindHome: - return p.s.pageMap.getPagesInSection( + return ps.s.pageMap.getPagesInSection( pageMapQueryPagesInSection{ pageMapQueryPagesBelowPath: pageMapQueryPagesBelowPath{ - Path: p.Path(), - Include: pagePredicates.ShouldListLocal.And(pagePredicates.KindPage), + Path: ps.Path(), + Include: pagePredicates.ShouldListLocal.And(pagePredicates.KindPage).BoolFunc(), }, Recursive: true, }, ) default: - return p.RegularPages() + return ps.RegularPages() } } -func (p *pageState) PagesRecursive() page.Pages { +func (ps *pageState) PagesRecursive() page.Pages { return nil } -func (p *pageState) RegularPages() page.Pages { - switch p.Kind() { +func (ps *pageState) RegularPages() page.Pages { + switch ps.Kind() { case kinds.KindPage: case kinds.KindSection, kinds.KindHome, kinds.KindTaxonomy: - return p.s.pageMap.getPagesInSection( + return ps.s.pageMap.getPagesInSection( pageMapQueryPagesInSection{ pageMapQueryPagesBelowPath: pageMapQueryPagesBelowPath{ - Path: p.Path(), - Include: pagePredicates.ShouldListLocal.And(pagePredicates.KindPage), + Path: ps.Path(), + Include: pagePredicates.ShouldListLocal.And(pagePredicates.KindPage).BoolFunc(), }, }, ) case kinds.KindTerm: - return p.s.pageMap.getPagesWithTerm( + return ps.s.pageMap.getPagesWithTerm( pageMapQueryPagesBelowPath{ - Path: p.Path(), - Include: pagePredicates.ShouldListLocal.And(pagePredicates.KindPage), + Path: ps.Path(), + Include: pagePredicates.ShouldListLocal.And(pagePredicates.KindPage).BoolFunc(), }, ) default: - return p.s.RegularPages() + return ps.s.RegularPages() } return nil } -func (p *pageState) Pages() page.Pages { - switch p.Kind() { +func (ps *pageState) Pages() page.Pages { + switch ps.Kind() { case kinds.KindPage: case kinds.KindSection, kinds.KindHome: - return p.s.pageMap.getPagesInSection( + return ps.s.pageMap.getPagesInSection( pageMapQueryPagesInSection{ pageMapQueryPagesBelowPath: pageMapQueryPagesBelowPath{ - Path: p.Path(), + Path: ps.Path(), KeyPart: "page-section", Include: pagePredicates.ShouldListLocal.And( pagePredicates.KindPage.Or(pagePredicates.KindSection), - ), + ).BoolFunc(), }, }, ) case kinds.KindTerm: - return p.s.pageMap.getPagesWithTerm( + return ps.s.pageMap.getPagesWithTerm( pageMapQueryPagesBelowPath{ - Path: p.Path(), + Path: ps.Path(), }, ) case kinds.KindTaxonomy: - return p.s.pageMap.getPagesInSection( + return ps.s.pageMap.getPagesInSection( pageMapQueryPagesInSection{ pageMapQueryPagesBelowPath: pageMapQueryPagesBelowPath{ - Path: p.Path(), + Path: ps.Path(), KeyPart: "term", - Include: pagePredicates.ShouldListLocal.And(pagePredicates.KindTerm), + Include: pagePredicates.ShouldListLocal.And(pagePredicates.KindTerm).BoolFunc(), }, Recursive: true, }, ) default: - return p.s.Pages() + return ps.s.Pages() } return nil } // RawContent returns the un-rendered source content without // any leading front matter. -func (p *pageState) RawContent() string { - if p.m.content.pi.itemsStep2 == nil { +func (ps *pageState) RawContent() string { + if ps.m.content.pi.itemsStep2 == nil { return "" } - start := p.m.content.pi.posMainContent + start := ps.m.content.pi.posMainContent if start == -1 { start = 0 } - source, err := p.m.content.pi.contentSource(p.m.content) + source, err := ps.m.content.pi.contentSource(ps.m.content) if err != nil { panic(err) } return string(source[start:]) } -func (p *pageState) Resources() resource.Resources { - return p.s.pageMap.getOrCreateResourcesForPage(p) +func (ps *pageState) Resources() resource.Resources { + return ps.s.pageMap.getOrCreateResourcesForPage(ps) } -func (p *pageState) HasShortcode(name string) bool { - if p.m.content.shortcodeState == nil { +func (ps *pageState) HasShortcode(name string) bool { + p := ps.m.content.hasShortcode.Load() + if p == nil { return false } - - return p.m.content.shortcodeState.hasName(name) + hasShortcode := *p + return hasShortcode(name) } -func (p *pageState) Site() page.Site { - return p.sWrapped +func (ps *pageState) Site() page.Site { + return ps.s.siteWrapped } -func (p *pageState) String() string { +func (ps *pageState) String() string { var sb strings.Builder - if p.File() != nil { + if ps.File() != nil { // The forward slashes even on Windows is motivated by // getting stable tests. // This information is meant for getting positional information in logs, // so the direction of the slashes should not matter. - sb.WriteString(filepath.ToSlash(p.File().Filename())) - if p.File().IsContentAdapter() { + sb.WriteString(filepath.ToSlash(ps.File().Filename())) + if ps.File().IsContentAdapter() { // Also include the path. sb.WriteString(":") - sb.WriteString(p.Path()) + sb.WriteString(ps.Path()) } } else { - sb.WriteString(p.Path()) + sb.WriteString(ps.Path()) } return sb.String() } // IsTranslated returns whether this content file is translated to // other language(s). -func (p *pageState) IsTranslated() bool { - return len(p.Translations()) > 0 +func (ps *pageState) IsTranslated() bool { + return len(ps.Translations()) > 0 } // TranslationKey returns the key used to identify a translation of this content. -func (p *pageState) TranslationKey() string { - if p.m.pageConfig.TranslationKey != "" { - return p.m.pageConfig.TranslationKey +func (ps *pageState) TranslationKey() string { + if ps.m.pageConfig.TranslationKey != "" { + return ps.m.pageConfig.TranslationKey } - return p.Path() + return ps.Path() } // AllTranslations returns all translations, including the current Page. -func (p *pageState) AllTranslations() page.Pages { - key := p.Path() + "/" + "translations-all" +func (ps *pageState) AllTranslations() page.Pages { + key := ps.Path() + "/" + "translations-all" // This is called from Translations, so we need to use a different partition, cachePages2, // to avoid potential deadlocks. - pages, err := p.s.pageMap.getOrCreatePagesFromCache(p.s.pageMap.cachePages2, key, func(string) (page.Pages, error) { - if p.m.pageConfig.TranslationKey != "" { + pages, err := ps.s.pageMap.getOrCreatePagesFromCache(ps.s.pageMap.cachePages2, key, func(string) (page.Pages, error) { + if ps.m.pageConfig.TranslationKey != "" { // translationKey set by user. - pas, _ := p.s.h.translationKeyPages.Get(p.m.pageConfig.TranslationKey) - pasc := make(page.Pages, len(pas)) - copy(pasc, pas) + pas, _ := ps.s.h.translationKeyPages.Get(ps.m.pageConfig.TranslationKey) + pasc := slices.Clone(pas) page.SortByLanguage(pasc) return pasc, nil } var pas page.Pages - p.s.pageMap.treePages.ForEeachInDimension(p.Path(), doctree.DimensionLanguage.Index(), - func(n contentNodeI) bool { + + ps.s.pageMap.treePages.ForEeachInDimension(ps.Path(), ps.s.siteVector, sitesmatrix.Language, + func(n contentNode) bool { if n != nil { pas = append(pas, n.(page.Page)) } - return false + return true }, ) @@ -452,13 +521,66 @@ func (p *pageState) AllTranslations() page.Pages { return pages } +// For internal use only. +func (ps *pageState) SiteVector() sitesmatrix.Vector { + return ps.s.siteVector +} + +func (ps *pageState) siteVector() sitesmatrix.Vector { + return ps.s.siteVector +} + +func (ps *pageState) siteVectors() sitesmatrix.VectorIterator { + return ps.s.siteVector +} + +// Rotate returns all pages in the given dimension for this page. +func (ps *pageState) Rotate(dimensionStr string) (page.Pages, error) { + dimensionStr = strings.ToLower(dimensionStr) + key := ps.Path() + "/" + "rotate-" + dimensionStr + d, err := sitesmatrix.ParseDimension(dimensionStr) + if err != nil { + return nil, fmt.Errorf("failed to parse dimension %q: %w", dimensionStr, err) + } + + pages, err := ps.s.pageMap.getOrCreatePagesFromCache(ps.s.pageMap.cachePages2, key, func(string) (page.Pages, error) { + var pas page.Pages + ps.s.pageMap.treePages.ForEeachInDimension(ps.Path(), ps.s.siteVector, d, + func(n contentNode) bool { + if n != nil { + p := n.(page.Page) + pas = append(pas, p) + } + return true + }, + ) + + if dimensionStr == "language" && ps.m.pageConfig.TranslationKey != "" { + // translationKey set by user. + // This is an old construct back from when languages were introduced. + // We keep it for backward compatibility. + // Also see AllTranslations. + pas, _ := ps.s.h.translationKeyPages.Get(ps.m.pageConfig.TranslationKey) + pasc := slices.Clone(pas) + page.SortByLanguage(pasc) + return pasc, nil + } + + pas = pagePredicates.ShouldLink.Filter(pas) + page.SortByDims(pas) + return pas, nil + }) + + return pages, err +} + // Translations returns the translations excluding the current Page. -func (p *pageState) Translations() page.Pages { - key := p.Path() + "/" + "translations" - pages, err := p.s.pageMap.getOrCreatePagesFromCache(nil, key, func(string) (page.Pages, error) { +func (ps *pageState) Translations() page.Pages { + key := ps.Path() + "/" + "translations" + pages, err := ps.s.pageMap.getOrCreatePagesFromCache(nil, key, func(string) (page.Pages, error) { var pas page.Pages - for _, pp := range p.AllTranslations() { - if !pp.Eq(p) { + for _, pp := range ps.AllTranslations() { + if !pp.Eq(ps) { pas = append(pas, pp) } } @@ -472,6 +594,9 @@ func (p *pageState) Translations() page.Pages { func (ps *pageState) initCommonProviders(pp pagePaths) error { if ps.IsPage() { + if ps.s == nil { + panic("no site") + } ps.posNextPrev = &nextPrev{init: ps.s.init.prevNext} ps.posNextPrevSection = &nextPrev{init: ps.s.init.prevNextInSection} ps.InSectionPositioner = newPagePositionInSection(ps.posNextPrevSection) @@ -490,10 +615,10 @@ func (ps *pageState) initCommonProviders(pp pagePaths) error { func (po *pageOutput) GetInternalTemplateBasePathAndDescriptor() (string, tplimpl.TemplateDescriptor) { p := po.p f := po.f + base := p.PathInfo().BaseReTyped(p.m.pageConfig.Type) return base, tplimpl.TemplateDescriptor{ Kind: p.Kind(), - Lang: p.Language().Lang, LayoutFromUser: p.Layout(), OutputFormat: f.Name, MediaType: f.MediaType.Type, @@ -501,8 +626,8 @@ func (po *pageOutput) GetInternalTemplateBasePathAndDescriptor() (string, tplimp } } -func (p *pageState) resolveTemplate(layouts ...string) (*tplimpl.TemplInfo, bool, error) { - dir, d := p.GetInternalTemplateBasePathAndDescriptor() +func (ps *pageState) resolveTemplate(layouts ...string) (*tplimpl.TemplInfo, bool, error) { + dir, d := ps.GetInternalTemplateBasePathAndDescriptor() if len(layouts) > 0 { d.LayoutFromUser = layouts[0] @@ -512,10 +637,11 @@ func (p *pageState) resolveTemplate(layouts ...string) (*tplimpl.TemplInfo, bool q := tplimpl.TemplateQuery{ Path: dir, Category: tplimpl.CategoryLayout, + Sites: ps.s.siteVector, Desc: d, } - tinfo := p.s.TemplateStore.LookupPagesLayout(q) + tinfo := ps.s.TemplateStore.LookupPagesLayout(q) if tinfo == nil { return nil, false, nil } @@ -524,20 +650,79 @@ func (p *pageState) resolveTemplate(layouts ...string) (*tplimpl.TemplInfo, bool } // Must be run after the site section tree etc. is built and ready. -func (p *pageState) initPage() error { - if _, err := p.init.Do(context.Background()); err != nil { - return err - } - return nil -} +func (ps *pageState) initPage() error { + var initErr error + ps.pageInit.Do(func() { + var pp pagePaths + pp, initErr = newPagePaths(ps) + if initErr != nil { + return + } -func (p *pageState) renderResources() error { - for _, r := range p.Resources() { + var outputFormatsForPage output.Formats + var renderFormats output.Formats + if ps.m.standaloneOutputFormat.IsZero() { + outputFormatsForPage = ps.outputFormats() + renderFormats = ps.s.h.renderFormats + } else { + // One of the fixed output format pages, e.g. 404. + outputFormatsForPage = output.Formats{ps.m.standaloneOutputFormat} + renderFormats = outputFormatsForPage + } + + // Prepare output formats for all sites. + // We do this even if this page does not get rendered on + // its own. It may be referenced via one of the site collections etc. + // it will then need an output format. + ps.pageOutputs = make([]*pageOutput, len(renderFormats)) + created := make(map[string]*pageOutput) + shouldRenderPage := !ps.m.noRender() + + for i, f := range renderFormats { + + if po, found := created[f.Name]; found { + ps.pageOutputs[i] = po + continue + } + + render := shouldRenderPage + if render { + _, render = outputFormatsForPage.GetByName(f.Name) + } + + po := newPageOutput(ps, pp, f, render) + + // Create a content provider for the first, + // we may be able to reuse it. + if i == 0 { + var contentProvider *pageContentOutput + contentProvider, initErr = newPageContentOutput(po) + if initErr != nil { + return + } + po.setContentProvider(contentProvider) + } + + ps.pageOutputs[i] = po + created[f.Name] = po + + } + + if initErr = ps.initCommonProviders(pp); initErr != nil { + return + } + }) + + return initErr +} + +func (ps *pageState) renderResources() error { + for _, r := range ps.Resources() { if _, ok := r.(page.Page); ok { - if p.s.h.buildCounter.Load() == 0 { + if ps.s.h.buildCounter.Load() == 0 { // Pages gets rendered with the owning page but we count them here. - p.s.PathSpec.ProcessingStats.Incr(&p.s.PathSpec.ProcessingStats.Pages) + ps.s.PathSpec.ProcessingStats.Incr(&ps.s.PathSpec.ProcessingStats.Pages) } continue } @@ -553,20 +738,20 @@ func (p *pageState) renderResources() error { if err := src.Publish(); err != nil { if !herrors.IsNotExist(err) { - p.s.Log.Errorf("Failed to publish Resource for page %q: %s", p.pathOrTitle(), err) + ps.s.Log.Errorf("Failed to publish Resource for page %q: %s", ps.pathOrTitle(), err) } } else { - p.s.PathSpec.ProcessingStats.Incr(&p.s.PathSpec.ProcessingStats.Files) + ps.s.PathSpec.ProcessingStats.Incr(&ps.s.PathSpec.ProcessingStats.Files) } } return nil } -func (p *pageState) AlternativeOutputFormats() page.OutputFormats { - f := p.outputFormat() +func (ps *pageState) AlternativeOutputFormats() page.OutputFormats { + f := ps.outputFormat() var o page.OutputFormats - for _, of := range p.OutputFormats() { + for _, of := range ps.OutputFormats() { if of.Format.NotAlternative || of.Format.Name == f.Name { continue } @@ -586,59 +771,59 @@ var defaultRenderStringOpts = renderStringOpts{ Markup: "", // Will inherit the page's value when not set. } -func (p *pageMeta) wrapError(err error, sourceFs afero.Fs) error { +func (m *pageMetaSource) wrapError(err error, sourceFs afero.Fs) error { if err == nil { panic("wrapError with nil") } - if p.File() == nil { + if m.f == nil { // No more details to add. - return fmt.Errorf("%q: %w", p.Path(), err) + return fmt.Errorf("%q: %w", m.Path(), err) } - return hugofs.AddFileInfoToError(err, p.File().FileInfo(), sourceFs) + return hugofs.AddFileInfoToError(err, m.f.FileInfo(), sourceFs) } // wrapError adds some more context to the given error if possible/needed -func (p *pageState) wrapError(err error) error { - return p.m.wrapError(err, p.s.h.SourceFs) +func (ps *pageState) wrapError(err error) error { + return ps.m.wrapError(err, ps.s.h.SourceFs) } -func (p *pageState) getPageInfoForError() string { - s := fmt.Sprintf("kind: %q, path: %q", p.Kind(), p.Path()) - if p.File() != nil { - s += fmt.Sprintf(", file: %q", p.File().Filename()) +func (ps *pageState) getPageInfoForError() string { + s := fmt.Sprintf("kind: %q, path: %q", ps.Kind(), ps.Path()) + if ps.File() != nil { + s += fmt.Sprintf(", file: %q", ps.File().Filename()) } return s } -func (p *pageState) getContentConverter() converter.Converter { +func (ps *pageState) getContentConverter() converter.Converter { var err error - p.contentConverterInit.Do(func() { - if p.m.pageConfig.ContentMediaType.IsZero() { + ps.contentConverterInit.Do(func() { + if ps.m.pageConfigSource.ContentMediaType.IsZero() { panic("ContentMediaType not set") } - markup := p.m.pageConfig.ContentMediaType.SubType + markup := ps.m.pageConfigSource.ContentMediaType.SubType if markup == "html" { // Only used for shortcode inner content. markup = "markdown" } - p.contentConverter, err = p.m.newContentConverter(p, markup) + ps.contentConverter, err = ps.m.newContentConverter(ps, markup) }) if err != nil { - p.s.Log.Errorln("Failed to create content converter:", err) + ps.s.Log.Errorln("Failed to create content converter:", err) } - return p.contentConverter + return ps.contentConverter } -func (p *pageState) errorf(err error, format string, a ...any) error { +func (ps *pageState) errorf(err error, format string, a ...any) error { if herrors.UnwrapFileError(err) != nil { // More isn't always better. return err } - args := append([]any{p.Language().Lang, p.pathOrTitle()}, a...) + args := append([]any{ps.Language().Lang, ps.pathOrTitle()}, a...) args = append(args, err) format = "[%s] page %q: " + format + ": %w" if err == nil { @@ -647,71 +832,71 @@ func (p *pageState) errorf(err error, format string, a ...any) error { return fmt.Errorf(format, args...) } -func (p *pageState) outputFormat() (f output.Format) { - if p.pageOutput == nil { +func (ps *pageState) outputFormat() (f output.Format) { + if ps.pageOutput == nil { panic("no pageOutput") } - return p.pageOutput.f + return ps.pageOutput.f } -func (p *pageState) parseError(err error, input []byte, offset int) error { +func (ps *pageState) parseError(err error, input []byte, offset int) error { pos := posFromInput("", input, offset) - return herrors.NewFileErrorFromName(err, p.File().Filename()).UpdatePosition(pos) + return herrors.NewFileErrorFromName(err, ps.File().Filename()).UpdatePosition(pos) } -func (p *pageState) pathOrTitle() string { - if p.File() != nil { - return p.File().Filename() +func (ps *pageState) pathOrTitle() string { + if ps.File() != nil { + return ps.File().Filename() } - if p.Path() != "" { - return p.Path() + if ps.Path() != "" { + return ps.Path() } - return p.Title() + return ps.Title() } -func (p *pageState) posFromInput(input []byte, offset int) text.Position { - return posFromInput(p.pathOrTitle(), input, offset) +func (ps *pageState) posFromInput(input []byte, offset int) text.Position { + return posFromInput(ps.pathOrTitle(), input, offset) } -func (p *pageState) posOffset(offset int) text.Position { - return p.posFromInput(p.m.content.mustSource(), offset) +func (ps *pageState) posOffset(offset int) text.Position { + return ps.posFromInput(ps.m.content.mustSource(), offset) } // shiftToOutputFormat is serialized. The output format idx refers to the // full set of output formats for all sites. // This is serialized. -func (p *pageState) shiftToOutputFormat(isRenderingSite bool, idx int) error { - if err := p.initPage(); err != nil { +func (ps *pageState) shiftToOutputFormat(isRenderingSite bool, idx int) error { + if err := ps.initPage(); err != nil { return err } - if len(p.pageOutputs) == 1 { + if len(ps.pageOutputs) == 1 { idx = 0 } - p.pageOutputIdx = idx - p.pageOutput = p.pageOutputs[idx] - if p.pageOutput == nil { + ps.pageOutputIdx = idx + ps.pageOutput = ps.pageOutputs[idx] + if ps.pageOutput == nil { panic(fmt.Sprintf("pageOutput is nil for output idx %d", idx)) } // Reset any built paginator. This will trigger when re-rendering pages in // server mode. - if isRenderingSite && p.pageOutput.paginator != nil && p.pageOutput.paginator.current != nil { - p.pageOutput.paginator.reset() + if isRenderingSite && ps.pageOutput.paginator != nil && ps.pageOutput.paginator.current != nil { + ps.pageOutput.paginator.reset() } if isRenderingSite { - cp := p.pageOutput.pco - if cp == nil && p.canReusePageOutputContent() { + cp := ps.pageOutput.pco + if cp == nil && ps.canReusePageOutputContent() { // Look for content to reuse. - for i := range p.pageOutputs { + for i := range ps.pageOutputs { if i == idx { continue } - po := p.pageOutputs[i] + po := ps.pageOutputs[i] if po.pco != nil { cp = po.pco @@ -722,12 +907,12 @@ func (p *pageState) shiftToOutputFormat(isRenderingSite bool, idx int) error { if cp == nil { var err error - cp, err = newPageContentOutput(p.pageOutput) + cp, err = newPageContentOutput(ps.pageOutput) if err != nil { return err } } - p.pageOutput.setContentProvider(cp) + ps.pageOutput.setContentProvider(cp) } else { // We attempt to assign pageContentOutputs while preparing each site // for rendering and before rendering each site. This lets us share @@ -735,21 +920,22 @@ func (p *pageState) shiftToOutputFormat(isRenderingSite bool, idx int) error { // unexpectedly calls a method of a ContentProvider that is not yet // initialized, we assign a LazyContentProvider that performs the // initialization just in time. - if lcp, ok := (p.pageOutput.ContentProvider.(*page.LazyContentProvider)); ok { + if lcp, ok := (ps.pageOutput.ContentProvider.(*page.LazyContentProvider)); ok { lcp.Reset() } else { - lcp = page.NewLazyContentProvider(func() (page.OutputFormatContentProvider, error) { - cp, err := newPageContentOutput(p.pageOutput) + lcp = page.NewLazyContentProvider(func(context.Context) page.OutputFormatContentProvider { + cp, err := newPageContentOutput(ps.pageOutput) if err != nil { - return nil, err + panic(err) } - return cp, nil + return cp }) - p.pageOutput.contentRenderer = lcp - p.pageOutput.ContentProvider = lcp - p.pageOutput.MarkupProvider = lcp - p.pageOutput.PageRenderProvider = lcp - p.pageOutput.TableOfContentsProvider = lcp + + ps.pageOutput.contentRenderer = lcp + ps.pageOutput.ContentProvider = lcp + ps.pageOutput.MarkupProvider = lcp + ps.pageOutput.PageRenderProvider = lcp + ps.pageOutput.TableOfContentsProvider = lcp } } diff --git a/hugolib/page__common.go b/hugolib/page__common.go index d7584155c..bbaa2e584 100644 --- a/hugolib/page__common.go +++ b/hugolib/page__common.go @@ -18,7 +18,6 @@ import ( "github.com/gohugoio/hugo/common/hstore" "github.com/gohugoio/hugo/compare" - "github.com/gohugoio/hugo/lazy" "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/navigation" "github.com/gohugoio/hugo/resources/page" @@ -43,16 +42,14 @@ func (p *pageCommon) getNextPrevInSection() *nextPrev { } type pageCommon struct { - s *Site + s *Site // TODO(bep) get rid of this. m *pageMeta - sWrapped page.Site - // Lazily initialized dependencies. - init *lazy.Init + pageInit sync.Once // Store holds state that survives server rebuilds. - store *hstore.Scratch + store func() *hstore.Scratch // All of these represents the common parts of a page.Page navigation.PageMenusProvider @@ -99,12 +96,16 @@ type pageCommon struct { // Internal use page.RelatedDocsHandlerProvider + pageContentConverter +} + +type pageContentConverter struct { contentConverterInit sync.Once contentConverter converter.Converter } func (p *pageCommon) Store() *hstore.Scratch { - return p.store + return p.store() } // See issue 13016. diff --git a/hugolib/page__content.go b/hugolib/page__content.go index f47979d61..ba164e549 100644 --- a/hugolib/page__content.go +++ b/hugolib/page__content.go @@ -19,9 +19,9 @@ import ( "fmt" "html/template" "io" - "path/filepath" "strconv" "strings" + "sync/atomic" "unicode/utf8" maps0 "maps" @@ -33,6 +33,7 @@ import ( "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/predicate" "github.com/gohugoio/hugo/common/types/hstring" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/markup" @@ -64,108 +65,85 @@ type pageContentReplacement struct { source pageparser.Item } -func (m *pageMeta) parseFrontMatter(h *HugoSites, pid uint64) (*contentParseInfo, error) { - var ( - sourceKey string - openSource hugio.OpenReadSeekCloser - isFromContentAdapter = m.pageConfig.IsFromContentAdapter - ) +func (m *pageMetaSource) parseFrontMatter( + h *HugoSites, + sid uint64, +) error { + var sourceKey string - if m.f != nil && !isFromContentAdapter { - sourceKey = filepath.ToSlash(m.f.Filename()) - if !isFromContentAdapter { - meta := m.f.FileInfo().Meta() - openSource = func() (hugio.ReadSeekCloser, error) { - r, err := meta.Open() - if err != nil { - return nil, fmt.Errorf("failed to open file %q: %w", meta.Filename, err) - } - return r, nil - } - } - } else if isFromContentAdapter { - openSource = m.pageConfig.Content.ValueAsOpenReadSeekCloser() + if sourceKey == "" { + sourceKey = strconv.FormatUint(sid, 10) } - if sourceKey == "" { - sourceKey = strconv.FormatUint(pid, 10) + var filename string + if m.f != nil { + filename = m.f.Filename() } - pi := &contentParseInfo{ - h: h, - pid: pid, - sourceKey: sourceKey, - openSource: openSource, + m.pi = &contentParseInfo{ + h: h, + sid: sid, + sourceKey: sourceKey, + openSource: m.openSource, + shortcodeParseInfo: newShortcodeHandler(filename, h.Deps), } - source, err := pi.contentSource(m) + source, err := m.pi.contentSource(m) if err != nil { - return nil, err + return err } items, err := pageparser.ParseBytes( source, pageparser.Config{ - NoFrontMatter: isFromContentAdapter, + NoFrontMatter: m.noFrontMatter, }, ) if err != nil { - return nil, err + return err } - pi.itemsStep1 = items - - if isFromContentAdapter { - // No front matter. - return pi, nil - } + m.pi.itemsStep1 = items - if err := pi.mapFrontMatter(source); err != nil { - return nil, err + if err := m.pi.parseSource(source, m.noFrontMatter); err != nil { + return err } - return pi, nil + return nil } -func (m *pageMeta) newCachedContent(h *HugoSites, pi *contentParseInfo) (*cachedContent, error) { - var filename string - if m.f != nil { - filename = m.f.Filename() +func (m *pageMeta) newCachedContent(s *Site) (*cachedContent, error) { + if m.pageMetaSource.pi == nil { + panic("pageMeta.pageMetaSource.pi must be set before creating cachedContent") } c := &cachedContent{ - pm: m.s.pageMap, - StaleInfo: m, - shortcodeState: newShortcodeHandler(filename, m.s), - pi: pi, - enableEmoji: m.s.conf.EnableEmoji, - scopes: maps.NewCache[string, *cachedContentScope](), - } - - source, err := c.pi.contentSource(m) - if err != nil { - return nil, err - } - - if err := c.parseContentFile(source); err != nil { - return nil, err + pm: s.pageMap, + StaleInfo: m, + pi: m.pi, + enableEmoji: s.conf.EnableEmoji, + scopes: maps.NewCache[string, *cachedContentScope](), } + var hasName predicate.P[string] = m.pi.shortcodeParseInfo.hasName + c.hasShortcode.Store(&hasName) return c, nil } +// Content cached for a page instance. type cachedContent struct { pm *pageMap resource.StaleInfo - shortcodeState *shortcodeHandler - // Parsed content. pi *contentParseInfo enableEmoji bool + // Whether the content has a given shortcode name. + hasShortcode atomic.Pointer[predicate.P[string]] + scopes *maps.Cache[string, *cachedContentScope] } @@ -181,10 +159,11 @@ func (c *cachedContent) getOrCreateScope(scope string, pco *pageContentOutput) * return cs } +// contentParseInfo is shared between all Page instances created from the same source. type contentParseInfo struct { h *HugoSites - pid uint64 + sid uint64 sourceKey string // The source bytes. @@ -207,20 +186,23 @@ type contentParseInfo struct { // *shortcode, pageContentReplacement or pageparser.Item itemsStep2 []any + + // The shortcode handler. + shortcodeParseInfo *shortcodeParseInfo } -func (p *contentParseInfo) AddBytes(item pageparser.Item) { - p.itemsStep2 = append(p.itemsStep2, item) +func (pi *contentParseInfo) AddBytes(item pageparser.Item) { + pi.itemsStep2 = append(pi.itemsStep2, item) } -func (p *contentParseInfo) AddReplacement(val []byte, source pageparser.Item) { - p.itemsStep2 = append(p.itemsStep2, pageContentReplacement{val: val, source: source}) +func (pi *contentParseInfo) AddReplacement(val []byte, source pageparser.Item) { + pi.itemsStep2 = append(pi.itemsStep2, pageContentReplacement{val: val, source: source}) } -func (p *contentParseInfo) AddShortcode(s *shortcode) { - p.itemsStep2 = append(p.itemsStep2, s) +func (pi *contentParseInfo) AddShortcode(s *shortcode) { + pi.itemsStep2 = append(pi.itemsStep2, s) if s.insertPlaceholder() { - p.hasNonMarkdownShortcode = true + pi.hasNonMarkdownShortcode = true } } @@ -268,91 +250,12 @@ func (c *cachedContent) IsZero() bool { return len(c.pi.itemsStep2) == 0 } -func (c *cachedContent) parseContentFile(source []byte) error { - if source == nil || c.pi.openSource == nil { - return nil - } - - return c.pi.mapItemsAfterFrontMatter(source, c.shortcodeState) -} - -func (c *contentParseInfo) parseFrontMatter(it pageparser.Item, iter *pageparser.Iterator, source []byte) error { - if c.frontMatter != nil { +func (pi *contentParseInfo) parseSource(source []byte, skipFrontMatter bool) error { + if len(pi.itemsStep1) == 0 { return nil } - f := pageparser.FormatFromFrontMatterType(it.Type) - var err error - c.frontMatter, err = metadecoders.Default.UnmarshalToMap(it.Val(source), f) - if err != nil { - fe := herrors.UnwrapFileError(err) - if fe == nil { - fe = herrors.NewFileError(err) - } - pos := fe.Position() - - // Offset the starting position of front matter. - offset := iter.LineNumber(source) - 1 - - pos.LineNumber += offset - - fe.UpdatePosition(pos) - fe.SetFilename("") // It will be set later. - return fe - } - - return nil -} - -func (rn *contentParseInfo) failMap(source []byte, err error, i pageparser.Item) error { - if fe, ok := err.(herrors.FileError); ok { - return fe - } - - pos := posFromInput("", source, i.Pos()) - - return herrors.NewFileErrorFromPos(err, pos) -} - -func (rn *contentParseInfo) mapFrontMatter(source []byte) error { - if len(rn.itemsStep1) == 0 { - return nil - } - iter := pageparser.NewIterator(rn.itemsStep1) - -Loop: - for { - it := iter.Next() - switch { - case it.IsFrontMatter(): - if err := rn.parseFrontMatter(it, iter, source); err != nil { - return err - } - next := iter.Peek() - if !next.IsDone() { - rn.posMainContent = next.Pos() - } - // Done. - break Loop - case it.IsEOF(): - break Loop - case it.IsError(): - return rn.failMap(source, it.Err, it) - default: - - } - } - - return nil -} - -func (rn *contentParseInfo) mapItemsAfterFrontMatter( - source []byte, - s *shortcodeHandler, -) error { - if len(rn.itemsStep1) == 0 { - return nil - } + s := pi.shortcodeParseInfo fail := func(err error, i pageparser.Item) error { if fe, ok := err.(herrors.FileError); ok { @@ -364,7 +267,7 @@ func (rn *contentParseInfo) mapItemsAfterFrontMatter( return herrors.NewFileErrorFromPos(err, pos) } - iter := pageparser.NewIterator(rn.itemsStep1) + iter := pageparser.NewIterator(pi.itemsStep1) // the parser is guaranteed to return items in proper order or fail, so … // … it's safe to keep some "global" state @@ -377,7 +280,15 @@ Loop: switch { case it.Type == pageparser.TypeIgnore: case it.IsFrontMatter(): - // Ignore. + if !skipFrontMatter { + if err := pi.parseFrontMatter(it, iter, source); err != nil { + return err + } + } + next := iter.Peek() + if !next.IsDone() { + pi.posMainContent = next.Pos() + } case it.Type == pageparser.TypeLeadSummaryDivider: posBody := -1 f := func(item pageparser.Item) bool { @@ -393,11 +304,11 @@ Loop: } iter.PeekWalk(f) - rn.hasSummaryDivider = true + pi.hasSummaryDivider = true // The content may be rendered by Goldmark or similar, // and we need to track the summary. - rn.AddReplacement(internalSummaryDividerPre, it) + pi.AddReplacement(internalSummaryDividerPre, it) // Handle shortcode case it.IsLeftShortcodeDelim(): @@ -412,7 +323,7 @@ Loop: currShortcode.pos = it.Pos() currShortcode.length = iter.Current().Pos() - it.Pos() if currShortcode.placeholder == "" { - currShortcode.placeholder = createShortcodePlaceholder("s", rn.pid, currShortcode.ordinal) + currShortcode.placeholder = createShortcodePlaceholder("s", pi.sid, currShortcode.ordinal) } if currShortcode.name != "" { @@ -424,19 +335,47 @@ Loop: currShortcode.params = s } - currShortcode.placeholder = createShortcodePlaceholder("s", rn.pid, ordinal) + currShortcode.placeholder = createShortcodePlaceholder("s", pi.sid, ordinal) ordinal++ s.shortcodes = append(s.shortcodes, currShortcode) - rn.AddShortcode(currShortcode) + pi.AddShortcode(currShortcode) case it.IsEOF(): break Loop case it.IsError(): return fail(it.Err, it) default: - rn.AddBytes(it) + pi.AddBytes(it) + } + } + + return nil +} + +func (pi *contentParseInfo) parseFrontMatter(it pageparser.Item, iter *pageparser.Iterator, source []byte) error { + if pi.frontMatter != nil { + return nil + } + + f := pageparser.FormatFromFrontMatterType(it.Type) + var err error + pi.frontMatter, err = metadecoders.Default.UnmarshalToMap(it.Val(source), f) + if err != nil { + fe := herrors.UnwrapFileError(err) + if fe == nil { + fe = herrors.NewFileError(err) } + pos := fe.Position() + + // Offset the starting position of front matter. + offset := iter.LineNumber(source) - 1 + + pos.LineNumber += offset + + fe.UpdatePosition(pos) + fe.SetFilename("") // It will be set later. + return fe } return nil @@ -450,12 +389,12 @@ func (c *cachedContent) mustSource() []byte { return source } -func (c *contentParseInfo) contentSource(s resource.StaleInfo) ([]byte, error) { - key := c.sourceKey +func (pi *contentParseInfo) contentSource(s resource.StaleInfo) ([]byte, error) { + key := pi.sourceKey versionv := s.StaleVersion() - v, err := c.h.cacheContentSource.GetOrCreate(key, func(string) (*resources.StaleValue[[]byte], error) { - b, err := c.readSourceAll() + v, err := pi.h.cacheContentSource.GetOrCreate(key, func(string) (*resources.StaleValue[[]byte], error) { + b, err := pi.readSourceAll() if err != nil { return nil, err } @@ -474,11 +413,11 @@ func (c *contentParseInfo) contentSource(s resource.StaleInfo) ([]byte, error) { return v.Value, nil } -func (c *contentParseInfo) readSourceAll() ([]byte, error) { - if c.openSource == nil { +func (pi *contentParseInfo) readSourceAll() ([]byte, error) { + if pi.openSource == nil { return []byte{}, nil } - r, err := c.openSource() + r, err := pi.openSource() if err != nil { return nil, err } @@ -525,6 +464,7 @@ func (c *cachedContentScope) contentRendered(ctx context.Context) (contentSummar cp := c.pco ctx = tpl.Context.DependencyScope.Set(ctx, pageDependencyScopeGlobal) key := c.pi.sourceKey + "/" + c.keyScope(ctx) + versionv := c.version(cp) v, err := c.pm.cacheContentRendered.GetOrCreate(key, func(string) (*resources.StaleValue[contentSummary], error) { @@ -604,7 +544,7 @@ func (c *cachedContentScope) contentRendered(ctx context.Context) (contentSummar var result contentSummary if c.pi.hasSummaryDivider { s := string(b) - summarized := page.ExtractSummaryFromHTMLWithDivider(cp.po.p.m.pageConfig.ContentMediaType, s, internalSummaryDividerBase) + summarized := page.ExtractSummaryFromHTMLWithDivider(cp.po.p.m.pageConfigSource.ContentMediaType, s, internalSummaryDividerBase) result.summary = page.Summary{ Text: template.HTML(summarized.Summary()), Type: page.SummaryTypeManual, @@ -619,7 +559,7 @@ func (c *cachedContentScope) contentRendered(ctx context.Context) (contentSummar if !c.pi.hasSummaryDivider && cp.po.p.m.pageConfig.Summary == "" { numWords := cp.po.p.s.conf.SummaryLength isCJKLanguage := cp.po.p.m.pageConfig.IsCJKLanguage - summary := page.ExtractSummaryFromHTML(cp.po.p.m.pageConfig.ContentMediaType, string(result.content), numWords, isCJKLanguage) + summary := page.ExtractSummaryFromHTML(cp.po.p.m.pageConfigSource.ContentMediaType, string(result.content), numWords, isCJKLanguage) result.summary = page.Summary{ Text: template.HTML(summary.Summary()), Type: page.SummaryTypeAuto, @@ -640,7 +580,7 @@ func (c *cachedContentScope) contentRendered(ctx context.Context) (contentSummar if err != nil { return nil, err } - html := cp.po.p.s.ContentSpec.TrimShortHTML(b.Bytes(), cp.po.p.m.pageConfig.Content.Markup) + html := cp.po.p.s.ContentSpec.TrimShortHTML(b.Bytes(), cp.po.p.m.pageConfigSource.Content.Markup) rs.Value.summary = page.Summary{ Text: helpers.BytesToHTML(html), Type: page.SummaryTypeFrontMatter, @@ -691,7 +631,7 @@ func (c *cachedContentScope) contentToC(ctx context.Context) (contentTableOfCont } po := cp.po p := po.p - ct.contentPlaceholders, err = c.shortcodeState.prepareShortcodesForPage(ctx, po, false) + ct.contentPlaceholders, err = c.pi.shortcodeParseInfo.prepareShortcodesForPage(po, false) if err != nil { return nil, err } @@ -704,13 +644,14 @@ func (c *cachedContentScope) contentToC(ctx context.Context) (contentTableOfCont maps0.Copy(ct.contentPlaceholders, ct2.contentPlaceholders) if p.s.conf.Internal.Watch { - for _, s := range cp2.po.p.m.content.shortcodeState.shortcodes { + for _, s := range cp2.po.p.m.content.pi.shortcodeParseInfo.shortcodes { cp.trackDependency(s.templ) } } // Transfer shortcode names so HasShortcode works for shortcodes from included pages. - cp.po.p.m.content.shortcodeState.transferNames(cp2.po.p.m.content.shortcodeState) + combined := cp.po.p.m.content.hasShortcode.Load().Or(*cp2.po.p.m.content.hasShortcode.Load()) + cp.po.p.m.content.hasShortcode.Store(&combined) if cp2.po.p.pageOutputTemplateVariationsState.Load() > 0 { cp.po.p.incrPageOutputTemplateVariation() } @@ -728,7 +669,7 @@ func (c *cachedContentScope) contentToC(ctx context.Context) (contentTableOfCont p.incrPageOutputTemplateVariation() } - isHTML := cp.po.p.m.pageConfig.ContentMediaType.IsHTML() + isHTML := cp.po.p.m.pageConfigSource.ContentMediaType.IsHTML() if !isHTML { createAndSetToC := func(tocProvider converter.TableOfContentsProvider) error { @@ -951,7 +892,7 @@ func (c *cachedContentScope) RenderString(ctx context.Context, args ...any) (tem conv := pco.po.p.getContentConverter() - if opts.Markup != "" && opts.Markup != pco.po.p.m.pageConfig.ContentMediaType.SubType { + if opts.Markup != "" && opts.Markup != pco.po.p.m.pageConfigSource.ContentMediaType.SubType { var err error conv, err = pco.po.p.m.newContentConverter(pco.po.p, opts.Markup) if err != nil { @@ -963,7 +904,7 @@ func (c *cachedContentScope) RenderString(ctx context.Context, args ...any) (tem parseInfo := &contentParseInfo{ h: pco.po.p.s.h, - pid: pco.po.p.pid, + sid: pco.po.p.pid, } if pageparser.HasShortcode(contentToRender) { @@ -977,12 +918,12 @@ func (c *cachedContentScope) RenderString(ctx context.Context, args ...any) (tem return "", err } - s := newShortcodeHandler(pco.po.p.pathOrTitle(), pco.po.p.s) - if err := parseInfo.mapItemsAfterFrontMatter(contentToRenderb, s); err != nil { + parseInfo.shortcodeParseInfo = newShortcodeHandler(pco.po.p.pathOrTitle(), pco.po.p.s.h.Deps) + if err := parseInfo.parseSource(contentToRenderb, true); err != nil { return "", err } - placeholders, err := s.prepareShortcodesForPage(ctx, pco.po, true) + placeholders, err := parseInfo.shortcodeParseInfo.prepareShortcodesForPage(pco.po, true) if err != nil { return "", err } @@ -1022,7 +963,7 @@ func (c *cachedContentScope) RenderString(ctx context.Context, args ...any) (tem return repl, nil } // This should not happen. - return nil, fmt.Errorf("unknown shortcode token %q", token) + return nil, fmt.Errorf("RenderString: unknown shortcode token %q", token) } rendered, err = expandShortcodeTokens(ctx, rendered, tokenHandler) @@ -1035,7 +976,8 @@ func (c *cachedContentScope) RenderString(ctx context.Context, args ...any) (tem } // We need a consolidated view in $page.HasShortcode - pco.po.p.m.content.shortcodeState.transferNames(s) + combined := pco.po.p.m.content.hasShortcode.Load().Or(parseInfo.shortcodeParseInfo.hasName) + pco.po.p.m.content.hasShortcode.Store(&combined) } else { c, err := pco.renderContentWithConverter(ctx, conv, []byte(contentToRender), false) @@ -1047,7 +989,7 @@ func (c *cachedContentScope) RenderString(ctx context.Context, args ...any) (tem } if opts.Display == "inline" { - markup := pco.po.p.m.pageConfig.Content.Markup + markup := pco.po.p.m.pageConfigSource.Content.Markup if opts.Markup != "" { markup = pco.po.p.s.ContentSpec.ResolveMarkup(opts.Markup) } diff --git a/hugolib/page__data.go b/hugolib/page__data.go index 256d8a97f..29faf9262 100644 --- a/hugolib/page__data.go +++ b/hugolib/page__data.go @@ -21,19 +21,18 @@ import ( "github.com/gohugoio/hugo/resources/page" ) -type pageData struct { - *pageState +type dataFunc func() any - dataInit sync.Once - data page.Data +func (f dataFunc) Data() any { + return f() } -func (p *pageData) Data() any { - p.dataInit.Do(func() { - p.data = make(page.Data) +func newDataFunc(p *pageState) dataFunc { + return sync.OnceValue(func() any { + data := make(page.Data) if p.Kind() == kinds.KindPage { - return + return data } switch p.Kind() { @@ -41,23 +40,23 @@ func (p *pageData) Data() any { path := p.Path() name := p.s.pageMap.cfg.getTaxonomyConfig(path) term := p.s.Taxonomies()[name.plural].Get(strings.TrimPrefix(path, name.pluralTreeKey)) - p.data[name.singular] = term - p.data["Singular"] = name.singular - p.data["Plural"] = name.plural - p.data["Term"] = p.m.term + data[name.singular] = term + data["Singular"] = name.singular + data["Plural"] = name.plural + data["Term"] = p.m.term case kinds.KindTaxonomy: viewCfg := p.s.pageMap.cfg.getTaxonomyConfig(p.Path()) - p.data["Singular"] = viewCfg.singular - p.data["Plural"] = viewCfg.plural - p.data["Terms"] = p.s.Taxonomies()[viewCfg.plural] + data["Singular"] = viewCfg.singular + data["Plural"] = viewCfg.plural + data["Terms"] = p.s.Taxonomies()[viewCfg.plural] // keep the following just for legacy reasons - p.data["OrderedIndex"] = p.data["Terms"] - p.data["Index"] = p.data["Terms"] + data["OrderedIndex"] = data["Terms"] + data["Index"] = data["Terms"] } // Assign the function to the map to make sure it is lazily initialized - p.data["pages"] = p.Pages - }) + data["pages"] = p.Pages - return p.data + return data + }) } diff --git a/hugolib/page__menus.go b/hugolib/page__menus.go index 1666036ce..4951dc6d9 100644 --- a/hugolib/page__menus.go +++ b/hugolib/page__menus.go @@ -30,22 +30,20 @@ type pageMenus struct { } func (p *pageMenus) HasMenuCurrent(menuID string, me *navigation.MenuEntry) bool { - p.p.s.init.menus.Do(context.Background()) + _ = p.p.s.init.menus.Value(context.Background()) p.init() return p.q.HasMenuCurrent(menuID, me) } func (p *pageMenus) IsMenuCurrent(menuID string, inme *navigation.MenuEntry) bool { - p.p.s.init.menus.Do(context.Background()) + _ = p.p.s.init.menus.Value(context.Background()) p.init() return p.q.IsMenuCurrent(menuID, inme) } func (p *pageMenus) Menus() navigation.PageMenus { - // There is a reverse dependency here. initMenus will, once, build the - // site menus and update any relevant page. - p.p.s.init.menus.Do(context.Background()) - + // There is a reverse dependency here. + _ = p.p.s.init.menus.Value(context.Background()) return p.menus() } diff --git a/hugolib/page__meta.go b/hugolib/page__meta.go index 00251186a..4da455de1 100644 --- a/hugolib/page__meta.go +++ b/hugolib/page__meta.go @@ -14,7 +14,6 @@ package hugolib import ( - "context" "fmt" "path/filepath" "regexp" @@ -25,12 +24,19 @@ import ( "github.com/bep/logg" "github.com/gobuffalo/flect" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/hugofs/files" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/markup/converter" + "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/source" "github.com/gohugoio/hugo/common/hashing" + "github.com/gohugoio/hugo/common/hiter" + "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/common/maps" @@ -48,62 +54,343 @@ import ( var cjkRe = regexp.MustCompile(`\p{Han}|\p{Hangul}|\p{Hiragana}|\p{Katakana}`) -type pageMeta struct { - term string // Set for kind == KindTerm. - singular string // Set for kind == KindTerm and kind == KindTaxonomy. +func (m *pageMetaSource) GetIdentity() identity.Identity { + return m.pathInfo +} + +func (m *pageMetaSource) Path() string { + return m.pathInfo.Base() +} + +func (m *pageMetaSource) PathInfo() *paths.Path { + return m.pathInfo +} + +func (m *pageMetaSource) forEeachContentNode(f func(v sitesmatrix.Vector, n contentNode) bool) bool { + return f(sitesmatrix.Vector{}, m) +} + +type pageMetaSource struct { + pathInfo *paths.Path // Always set. This the canonical path to the Page. + f *source.File + pi *contentParseInfo + contentAdapterSourceEntryHash uint64 - resource.Staler - *pageMetaParams + sitesMatrixBase sitesmatrix.VectorIterator + sitesMatrixBaseOnly bool // Set for standalone pages, e.g. robotsTXT. standaloneOutputFormat output.Format + resources.AtomicStaler + + pageConfigSource *pagemeta.PageConfigEarly + cascadeCompiled *page.PageMatcherParamsConfigs + + resourcePath string // Set for bundled pages; path relative to its bundle root. + bundled bool // Set if this page is bundled inside another. + noFrontMatter bool + openSource func() (hugio.ReadSeekCloser, error) +} + +func (m *pageMetaSource) String() string { + if m.f != nil { + return fmt.Sprintf("pageMetaSource(%s, %s)", m.f.FileInfo().Meta().PathInfo, m.f.FileInfo().Meta().Filename) + } + return fmt.Sprintf("pageMetaSource(%s)", m.pathInfo) +} + +func (m *pageMetaSource) nodeCategoryPage() { + // Marker method. +} + +func (m *pageMetaSource) nodeCategorySingle() { + // Marker method. +} + +func (m *pageMetaSource) nodeSourceEntryID() any { + if m.f != nil && !m.f.IsContentAdapter() { + return m.f.FileInfo().Meta().Filename + } + return m.contentAdapterSourceEntryHash +} + +func (m *pageMetaSource) setCascadeFromMap(frontmatter map[string]any, defaultSitesMatrix sitesmatrix.VectorStore, configuredDimensions *sitesmatrix.ConfiguredDimensions, logger loggers.Logger) error { + const ( + pageMetaKeyCascade = "cascade" + ) + + // Check for any cascade define on itself. + if cv, found := frontmatter[pageMetaKeyCascade]; found { + var err error + cascade, err := page.DecodeCascadeConfig(cv) + if err != nil { + return err + } + if err := cascade.InitConfig(logger, defaultSitesMatrix, configuredDimensions); err != nil { + return err + } + + m.cascadeCompiled = cascade + } + return nil +} + +var _ contentNodeCascadeProvider = (*pageState)(nil) - resourcePath string // Set for bundled pages; path relative to its bundle root. - bundled bool // Set if this page is bundled inside another. +type pageMeta struct { + // Shared between all dimensions of this page. + *pageMetaSource - pathInfo *paths.Path // Always set. This the canonical path to the Page. - f *source.File + pageConfig *pagemeta.PageConfigLate + + term string // Set for kind == KindTerm. + singular string // Set for kind == KindTerm and kind == KindTaxonomy. content *cachedContent // The source and the parsed page content. - s *Site // The site this page belongs to. + datesOriginal pagemeta.Dates // Original dates for rebuilds. } -// Prepare for a rebuild of the data passed in from front matter. -func (m *pageMeta) setMetaPostPrepareRebuild() { - params := xmaps.Clone(m.paramsOriginal) - m.pageMetaParams.pageConfig = pagemeta.ClonePageConfigForRebuild(m.pageMetaParams.pageConfig, params) +func (p *pageState) getCascade() *page.PageMatcherParamsConfigs { + return p.m.cascadeCompiled } -type pageMetaParams struct { - setMetaPostCount int - setMetaPostCascadeChanged bool +func (m *pageMetaSource) initEarly(h *HugoSites, cascades *page.PageMatcherParamsConfigs) error { + if err := m.doInitEarly(h, cascades); err != nil { + return m.wrapError(err, h.SourceFs) + } + return nil +} + +func (m *pageMetaSource) doInitEarly(h *HugoSites, cascades *page.PageMatcherParamsConfigs) error { + if err := m.initFrontMatter(h); err != nil { + return err + } + + if m.pageConfigSource.Kind == "" { + // Resolve page kind. + m.pageConfigSource.Kind = kinds.KindSection + if m.pathInfo.Base() == "" { + m.pageConfigSource.Kind = kinds.KindHome + } else if m.pathInfo.IsBranchBundle() { + // A section, taxonomy or term. + tc := h.getFirstTaxonomyConfig(m.Path()) + if !tc.IsZero() { + // Either a taxonomy or a term. + if tc.pluralTreeKey == m.Path() { + m.pageConfigSource.Kind = kinds.KindTaxonomy + } else { + m.pageConfigSource.Kind = kinds.KindTerm + } + } + } else if m.f != nil { + m.pageConfigSource.Kind = kinds.KindPage + } + } - pageConfig *pagemeta.PageConfig + sitesMatrixBase := m.sitesMatrixBase + if sitesMatrixBase == nil && m.f != nil { + sitesMatrixBase = m.f.FileInfo().Meta().SitesMatrix + } - // These are only set in watch mode. - datesOriginal pagemeta.Dates - paramsOriginal map[string]any // contains the original params as defined in the front matter. - cascadeOriginal *maps.Ordered[page.PageMatcher, page.PageMatcherParamsConfig] // contains the original cascade as defined in the front matter. + if m.sitesMatrixBaseOnly && sitesMatrixBase == nil { + panic("sitesMatrixBaseOnly set, but no base sites matrix") + } + + var fim *hugofs.FileMeta + if m.f != nil { + fim = m.f.FileInfo().Meta() + } + if err := m.pageConfigSource.CompileEarly(m.pathInfo, cascades, h.Conf, fim, sitesMatrixBase, m.sitesMatrixBaseOnly); err != nil { + return err + } + + if cnh.isBranchNode(m) && m.pageConfigSource.Frontmatter != nil { + if err := m.setCascadeFromMap(m.pageConfigSource.Frontmatter, m.pageConfigSource.SitesMatrix, h.Conf.ConfiguredDimensions(), h.Log); err != nil { + return err + } + } + return nil } -func (m *pageMetaParams) init(preserveOriginal bool) { - if preserveOriginal { - if m.pageConfig.IsFromContentAdapter { - m.paramsOriginal = xmaps.Clone(m.pageConfig.ContentAdapterData) - } else { - m.paramsOriginal = xmaps.Clone(m.pageConfig.Params) +func (m *pageMetaSource) initFrontMatter(h *HugoSites) error { + if m.pathInfo == nil { + if err := m.initPathInfo(h); err != nil { + return err + } + } + + if m.pageConfigSource == nil { + m.pageConfigSource = &pagemeta.PageConfigEarly{} + } + + // Read front matter. + if err := m.parseFrontMatter(h, pageSourceIDCounter.Add(1)); err != nil { + return err + } + var ext string + if m.f != nil { + if !m.f.IsContentAdapter() { + ext = m.f.Ext() + } + } + + if err := m.pageConfigSource.SetMetaPreFromMap(ext, m.pi.frontMatter, h.Log, h.Conf); err != nil { + return err + } + + return nil +} + +func (m *pageMetaSource) initPathInfo(h *HugoSites) error { + pcfg := m.pageConfigSource + if pcfg.Path != "" { + s := pcfg.Path + // Paths from content adapters should never have any extension. + if pcfg.IsFromContentAdapter || !paths.HasExt(s) { + var ( + isBranch bool + isBranchSet bool + ext string = pcfg.ContentMediaType.FirstSuffix.Suffix + ) + if pcfg.Kind != "" { + isBranch = kinds.IsBranch(pcfg.Kind) + isBranchSet = true + } + + if !pcfg.IsFromContentAdapter { + if m.pathInfo != nil { + if !isBranchSet { + isBranch = m.pathInfo.IsBranchBundle() + } + if m.pathInfo.Ext() != "" { + ext = m.pathInfo.Ext() + } + } else if m.f != nil { + pi := m.f.FileInfo().Meta().PathInfo + if !isBranchSet { + isBranch = pi.IsBranchBundle() + } + if pi.Ext() != "" { + ext = pi.Ext() + } + } + } + if isBranch { + s += "/_index." + ext + } else { + s += "/index." + ext + } + + } + m.pathInfo = h.Conf.PathParser().Parse(files.ComponentFolderContent, s) + } else { + if m.f != nil { + m.pathInfo = m.f.FileInfo().Meta().PathInfo + } + + if m.pathInfo == nil { + panic(fmt.Sprintf("missing pathInfo in %v", m)) + } + } + return nil +} + +func (m *pageMeta) initLate(s *Site) error { + var tc viewName + + if m.pageConfigSource.Kind == kinds.KindTerm || m.pageConfigSource.Kind == kinds.KindTaxonomy { + if tc.IsZero() { + tc = s.pageMap.cfg.getTaxonomyConfig(m.Path()) + } + if tc.IsZero() { + return fmt.Errorf("no taxonomy configuration found for %q", m.Path()) + } + m.singular = tc.singular + + if m.pageConfigSource.Kind == kinds.KindTerm { + m.term = paths.TrimLeading(strings.TrimPrefix(m.pathInfo.Unnormalized().Base(), tc.pluralTreeKey)) + } + } + + return nil +} + +// bookmark1 +func (h *HugoSites) newPageMetaSourceFromFile(fi hugofs.FileMetaInfo) (*pageMetaSource, error) { + p, err := func() (*pageMetaSource, error) { + meta := fi.Meta() + openSource := func() (hugio.ReadSeekCloser, error) { + r, err := meta.Open() + if err != nil { + return nil, fmt.Errorf("failed to open file %q: %w", meta.Filename, err) + } + return r, nil + } + + p := &pageMetaSource{ + f: source.NewFileInfo(fi), + pathInfo: fi.Meta().PathInfo, + openSource: openSource, } - m.cascadeOriginal = m.pageConfig.CascadeCompiled.Clone() + + return p, nil + }() + if err != nil { + return nil, hugofs.AddFileInfoToError(err, fi, h.SourceFs) + } + + return p, err +} + +func (h *HugoSites) newPageMetaSourceForContentAdapter(fi hugofs.FileMetaInfo, sitesMatrixBase sitesmatrix.VectorIterator, pc *pagemeta.PageConfigEarly) (*pageMetaSource, error) { + p := &pageMetaSource{ + f: source.NewFileInfo(fi), + noFrontMatter: true, + pageConfigSource: pc, + contentAdapterSourceEntryHash: pc.SourceEntryHash, + sitesMatrixBase: sitesMatrixBase, + openSource: pc.Content.ValueAsOpenReadSeekCloser(), + } + + return p, p.initPathInfo(h) +} + +func (s *Site) newPageFromPageMetasource(ms *pageMetaSource, cascades *page.PageMatcherParamsConfigs) (*pageState, error) { + m, err := s.newPageMetaFromPageMetasource(ms) + if err != nil { + return nil, err + } + p, err := s.newPageFromPageMeta(m, cascades) + if err != nil { + return nil, err + } + + return p, nil +} + +func (s *Site) newPageMetaFromPageMetasource(ms *pageMetaSource) (*pageMeta, error) { + m := &pageMeta{ + pageMetaSource: ms, + pageConfig: &pagemeta.PageConfigLate{ + Params: make(maps.Params), + }, } + return m, nil +} + +// Prepare for a rebuild of the data passed in from front matter. +func (m *pageMeta) prepareRebuild() { + // Restore orignal date values so we can repeat aggregation. + m.pageConfig.Dates = m.datesOriginal } -func (p *pageMeta) Aliases() []string { - return p.pageConfig.Aliases +func (m *pageMeta) Aliases() []string { + return m.pageConfig.Aliases } -func (p *pageMeta) BundleType() string { - switch p.pathInfo.Type() { +func (m *pageMeta) BundleType() string { + switch m.pathInfo.Type() { case paths.TypeLeaf: return "leaf" case paths.TypeBranch: @@ -113,251 +400,158 @@ func (p *pageMeta) BundleType() string { } } -func (p *pageMeta) Date() time.Time { - return p.pageConfig.Dates.Date +func (m *pageMeta) Date() time.Time { + return m.pageConfig.Dates.Date } -func (p *pageMeta) PublishDate() time.Time { - return p.pageConfig.Dates.PublishDate +func (m *pageMeta) PublishDate() time.Time { + return m.pageConfig.Dates.PublishDate } -func (p *pageMeta) Lastmod() time.Time { - return p.pageConfig.Dates.Lastmod +func (m *pageMeta) Lastmod() time.Time { + return m.pageConfig.Dates.Lastmod } -func (p *pageMeta) ExpiryDate() time.Time { - return p.pageConfig.Dates.ExpiryDate +func (m *pageMeta) ExpiryDate() time.Time { + return m.pageConfig.Dates.ExpiryDate } -func (p *pageMeta) Description() string { - return p.pageConfig.Description +func (m *pageMeta) Description() string { + return m.pageConfig.Description } -func (p *pageMeta) Lang() string { - return p.s.Lang() +func (m *pageMeta) Draft() bool { + return m.pageConfig.Draft } -func (p *pageMeta) Draft() bool { - return p.pageConfig.Draft +func (m *pageMeta) File() *source.File { + return m.f } -func (p *pageMeta) File() *source.File { - return p.f +func (m *pageMeta) IsHome() bool { + return m.Kind() == kinds.KindHome } -func (p *pageMeta) IsHome() bool { - return p.Kind() == kinds.KindHome +func (m *pageMeta) Keywords() []string { + return m.pageConfig.Keywords } -func (p *pageMeta) Keywords() []string { - return p.pageConfig.Keywords +func (m *pageMetaSource) Kind() string { + return m.pageConfigSource.Kind } -func (p *pageMeta) Kind() string { - return p.pageConfig.Kind +func (m *pageMeta) Layout() string { + return m.pageConfig.Layout } -func (p *pageMeta) Layout() string { - return p.pageConfig.Layout -} - -func (p *pageMeta) LinkTitle() string { - if p.pageConfig.LinkTitle != "" { - return p.pageConfig.LinkTitle +func (m *pageMeta) LinkTitle() string { + if m.pageConfig.LinkTitle != "" { + return m.pageConfig.LinkTitle } - return p.Title() + return m.Title() } -func (p *pageMeta) Name() string { - if p.resourcePath != "" { - return p.resourcePath +func (m *pageMeta) Name() string { + if m.resourcePath != "" { + return m.resourcePath } - if p.pageConfig.Kind == kinds.KindTerm { - return p.pathInfo.Unnormalized().BaseNameNoIdentifier() + if m.Kind() == kinds.KindTerm { + return m.pathInfo.Unnormalized().BaseNameNoIdentifier() } - return p.Title() + return m.Title() } -func (p *pageMeta) IsNode() bool { - return !p.IsPage() +func (m *pageMeta) IsNode() bool { + return !m.IsPage() } -func (p *pageMeta) IsPage() bool { - return p.Kind() == kinds.KindPage +func (m *pageMeta) IsPage() bool { + return m.Kind() == kinds.KindPage } -// Param is a convenience method to do lookups in Page's and Site's Params map, -// in that order. -// -// This method is also implemented on SiteInfo. -// TODO(bep) interface -func (p *pageMeta) Param(key any) (any, error) { - return resource.Param(p, p.s.Params(), key) +func (m *pageMeta) Params() maps.Params { + return m.pageConfig.Params } -func (p *pageMeta) Params() maps.Params { - return p.pageConfig.Params -} - -func (p *pageMeta) Path() string { - return p.pathInfo.Base() +func (m *pageMeta) Path() string { + if m.IsHome() { + // Base returns an empty string for the home page, + // which is correct as the key to the page tree. + return "/" + } + return m.pathInfo.Base() } -func (p *pageMeta) PathInfo() *paths.Path { - return p.pathInfo +func (m *pageMeta) PathInfo() *paths.Path { + return m.pathInfo } -func (p *pageMeta) IsSection() bool { - return p.Kind() == kinds.KindSection +func (m *pageMeta) IsSection() bool { + return m.Kind() == kinds.KindSection } -func (p *pageMeta) Section() string { - return p.pathInfo.Section() +func (m *pageMeta) Section() string { + return m.pathInfo.Section() } -func (p *pageMeta) Sitemap() config.SitemapConfig { - return p.pageConfig.Sitemap +func (m *pageMeta) Sitemap() config.SitemapConfig { + return m.pageConfig.Sitemap } -func (p *pageMeta) Title() string { - return p.pageConfig.Title +func (m *pageMeta) Title() string { + return m.pageConfig.Title } const defaultContentType = "page" -func (p *pageMeta) Type() string { - if p.pageConfig.Type != "" { - return p.pageConfig.Type +func (m *pageMeta) Type() string { + if m.pageConfig.Type != "" { + return m.pageConfig.Type } - if sect := p.Section(); sect != "" { + if sect := m.Section(); sect != "" { return sect } return defaultContentType } -func (p *pageMeta) Weight() int { - return p.pageConfig.Weight -} - -func (p *pageMeta) setMetaPre(pi *contentParseInfo, logger loggers.Logger, conf config.AllProvider) error { - frontmatter := pi.frontMatter - - if frontmatter != nil { - pcfg := p.pageConfig - // Needed for case insensitive fetching of params values - maps.PrepareParams(frontmatter) - pcfg.Params = frontmatter - // Check for any cascade define on itself. - if cv, found := frontmatter["cascade"]; found { - var err error - cascade, err := page.DecodeCascade(logger, true, cv) - if err != nil { - return err - } - pcfg.CascadeCompiled = cascade - } - - // Look for path, lang and kind, all of which values we need early on. - if v, found := frontmatter["path"]; found { - pcfg.Path = paths.ToSlashPreserveLeading(cast.ToString(v)) - pcfg.Params["path"] = pcfg.Path - } - if v, found := frontmatter["lang"]; found { - lang := strings.ToLower(cast.ToString(v)) - if _, ok := conf.PathParser().LanguageIndex[lang]; ok { - pcfg.Lang = lang - pcfg.Params["lang"] = pcfg.Lang - } - } - if v, found := frontmatter["kind"]; found { - s := cast.ToString(v) - if s != "" { - pcfg.Kind = kinds.GetKindMain(s) - if pcfg.Kind == "" { - return fmt.Errorf("unknown kind %q in front matter", s) - } - pcfg.Params["kind"] = pcfg.Kind - } - } - } else if p.pageMetaParams.pageConfig.Params == nil { - p.pageConfig.Params = make(maps.Params) - } - - p.pageMetaParams.init(conf.Watching()) - - return nil +func (m *pageMeta) Weight() int { + return m.pageConfig.Weight } -func (ps *pageState) setMetaPost(cascade *maps.Ordered[page.PageMatcher, page.PageMatcherParamsConfig]) error { - ps.m.setMetaPostCount++ - var cascadeHashPre uint64 - if ps.m.setMetaPostCount > 1 { - cascadeHashPre = hashing.HashUint64(ps.m.pageConfig.CascadeCompiled) - ps.m.pageConfig.CascadeCompiled = ps.m.cascadeOriginal.Clone() - - } - - // Apply cascades first so they can be overridden later. - if cascade != nil { - if ps.m.pageConfig.CascadeCompiled != nil { - cascade.Range(func(k page.PageMatcher, v page.PageMatcherParamsConfig) bool { - vv, found := ps.m.pageConfig.CascadeCompiled.Get(k) - if !found { - ps.m.pageConfig.CascadeCompiled.Set(k, v) - } else { - // Merge - for ck, cv := range v.Params { - if _, found := vv.Params[ck]; !found { - vv.Params[ck] = cv - } - } - for ck, cv := range v.Fields { - if _, found := vv.Fields[ck]; !found { - vv.Fields[ck] = cv - } - } - } - return true - }) - cascade = ps.m.pageConfig.CascadeCompiled +func (ps *pageState) setMetaPost(cascades *page.PageMatcherParamsConfigs) error { + if ps.m.pageConfigSource.Frontmatter != nil { + if ps.m.pageConfigSource.IsFromContentAdapter { + ps.m.pageConfig.ContentAdapterData = xmaps.Clone(ps.m.pageConfigSource.Frontmatter) } else { - ps.m.pageConfig.CascadeCompiled = cascade + ps.m.pageConfig.Params = xmaps.Clone(ps.m.pageConfigSource.Frontmatter) } } - if cascade == nil { - cascade = ps.m.pageConfig.CascadeCompiled + if ps.m.pageConfig.Params == nil { + ps.m.pageConfig.Params = make(maps.Params) + } + if ps.m.pageConfigSource.IsFromContentAdapter && ps.m.pageConfig.ContentAdapterData == nil { + ps.m.pageConfig.ContentAdapterData = make(maps.Params) } - if ps.m.setMetaPostCount > 1 { - ps.m.setMetaPostCascadeChanged = cascadeHashPre != hashing.HashUint64(ps.m.pageConfig.CascadeCompiled) - if !ps.m.setMetaPostCascadeChanged { + // Cascade defined on itself has higher priority than inherited ones. + allCascades := hiter.Concat(ps.m.cascadeCompiled.All(), cascades.All()) - // No changes, restore any value that may be changed by aggregation. - ps.m.pageConfig.Dates = ps.m.datesOriginal - return nil + for v := range allCascades { + if !v.Target.Match(ps.Kind(), ps.Path(), ps.s.Conf.Environment(), ps.s.siteVector) { + continue } - ps.m.setMetaPostPrepareRebuild() - } - - // Cascade is also applied to itself. - var err error - cascade.Range(func(k page.PageMatcher, v page.PageMatcherParamsConfig) bool { - if !k.Matches(ps) { - return true - } for kk, vv := range v.Params { if _, found := ps.m.pageConfig.Params[kk]; !found { ps.m.pageConfig.Params[kk] = vv } } - for kk, vv := range v.Fields { - if ps.m.pageConfig.IsFromContentAdapter { + if ps.m.pageConfigSource.IsFromContentAdapter { if _, found := ps.m.pageConfig.ContentAdapterData[kk]; !found { ps.m.pageConfig.ContentAdapterData[kk] = vv } @@ -367,18 +561,13 @@ func (ps *pageState) setMetaPost(cascade *maps.Ordered[page.PageMatcher, page.Pa } } } - return true - }) - - if err != nil { - return err } if err := ps.setMetaPostParams(); err != nil { return err } - if err := ps.m.applyDefaultValues(); err != nil { + if err := ps.m.applyDefaultValues(ps.s); err != nil { return err } @@ -388,49 +577,48 @@ func (ps *pageState) setMetaPost(cascade *maps.Ordered[page.PageMatcher, page.Pa return nil } -func (p *pageState) setMetaPostParams() error { - pm := p.m +func (ps *pageState) setMetaPostParams() error { + pm := ps.m var mtime time.Time var contentBaseName string - var ext string var isContentAdapter bool - if p.File() != nil { - isContentAdapter = p.File().IsContentAdapter() - contentBaseName = p.File().ContentBaseName() - if p.File().FileInfo() != nil { - mtime = p.File().FileInfo().ModTime() - } - if !isContentAdapter { - ext = p.File().Ext() + if ps.File() != nil { + isContentAdapter = ps.File().IsContentAdapter() + contentBaseName = ps.File().ContentBaseName() + if ps.File().FileInfo() != nil { + mtime = ps.File().FileInfo().ModTime() } } var gitAuthorDate time.Time - if p.gitInfo != nil { - gitAuthorDate = p.gitInfo.AuthorDate + if ps.gitInfo != nil { + gitAuthorDate = ps.gitInfo.AuthorDate } descriptor := &pagemeta.FrontMatterDescriptor{ - PageConfig: pm.pageConfig, - BaseFilename: contentBaseName, - ModTime: mtime, - GitAuthorDate: gitAuthorDate, - Location: langs.GetLocation(pm.s.Language()), - PathOrTitle: p.pathOrTitle(), + PageConfigEarly: pm.pageConfigSource, + PageConfigLate: pm.pageConfig, + BaseFilename: contentBaseName, + ModTime: mtime, + GitAuthorDate: gitAuthorDate, + Location: langs.GetLocation(ps.s.Language()), + PathOrTitle: ps.pathOrTitle(), } if isContentAdapter { - if err := pm.pageConfig.Compile(ext, p.s.Log, p.s.conf.OutputFormats.Config, p.s.conf.MediaTypes.Config); err != nil { + if err := pm.pageConfig.Compile(pm.pageConfigSource, ps.s.Log, ps.s.conf.OutputFormats.Config); err != nil { return err } } + pcfg := pm.pageConfig + // Handle the date separately // TODO(bep) we need to "do more" in this area so this can be split up and // more easily tested without the Page, but the coupling is strong. - err := pm.s.frontmatterHandler.HandleDates(descriptor) + err := ps.s.frontmatterHandler.HandleDates(descriptor) if err != nil { - p.s.Log.Errorf("Failed to handle dates for page %q: %s", p.pathOrTitle(), err) + ps.s.Log.Errorf("Failed to handle dates for page %q: %s", ps.pathOrTitle(), err) } if isContentAdapter { @@ -438,13 +626,15 @@ func (p *pageState) setMetaPostParams() error { return nil } + var sitemapSet bool + var buildConfig any var isNewBuildKeyword bool - if v, ok := pm.pageConfig.Params["_build"]; ok { + if v, ok := pcfg.Params["_build"]; ok { hugo.Deprecate("The \"_build\" front matter key", "Use \"build\" instead. See https://gohugo.io/content-management/build-options.", "0.145.0") buildConfig = v } else { - buildConfig = pm.pageConfig.Params["build"] + buildConfig = pcfg.Params["build"] isNewBuildKeyword = true } pm.pageConfig.Build, err = pagemeta.DecodeBuildConfig(buildConfig) @@ -464,14 +654,6 @@ params: return fmt.Errorf("failed to decode build config in front matter: %s%s", err, msgDetail) } - var sitemapSet bool - - pcfg := pm.pageConfig - params := pcfg.Params - if params == nil { - panic("params not set for " + p.Title()) - } - var draft, published, isCJKLanguage *bool var userParams map[string]any for k, v := range pcfg.Params { @@ -496,7 +678,7 @@ params: continue } - if pm.s.frontmatterHandler.IsDateKey(loki) { + if ps.s.frontmatterHandler.IsDateKey(loki) { continue } @@ -508,39 +690,39 @@ params: switch loki { case "title": pcfg.Title = cast.ToString(v) - params[loki] = pcfg.Title + pcfg.Params[loki] = pcfg.Title case "linktitle": pcfg.LinkTitle = cast.ToString(v) - params[loki] = pcfg.LinkTitle + pcfg.Params[loki] = pcfg.LinkTitle case "summary": pcfg.Summary = cast.ToString(v) - params[loki] = pcfg.Summary + pcfg.Params[loki] = pcfg.Summary case "description": pcfg.Description = cast.ToString(v) - params[loki] = pcfg.Description + pcfg.Params[loki] = pcfg.Description case "slug": // Don't start or end with a - pcfg.Slug = strings.Trim(cast.ToString(v), "-") - params[loki] = pm.Slug() + pcfg.Params[loki] = pm.Slug() case "url": url := cast.ToString(v) if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { - return fmt.Errorf("URLs with protocol (http*) not supported: %q. In page %q", url, p.pathOrTitle()) + return fmt.Errorf("URLs with protocol (http*) not supported: %q. In page %q", url, ps.pathOrTitle()) } pcfg.URL = url - params[loki] = url + pcfg.Params[loki] = url case "type": pcfg.Type = cast.ToString(v) - params[loki] = pcfg.Type + pcfg.Params[loki] = pcfg.Type case "keywords": pcfg.Keywords = cast.ToStringSlice(v) - params[loki] = pcfg.Keywords + pcfg.Params[loki] = pcfg.Keywords case "headless": // Legacy setting for leaf bundles. // This is since Hugo 0.63 handled in a more general way for all // pages. isHeadless := cast.ToBool(v) - params[loki] = isHeadless + pcfg.Params[loki] = isHeadless if isHeadless { pm.pageConfig.Build.List = pagemeta.Never pm.pageConfig.Build.Render = pagemeta.Never @@ -557,13 +739,10 @@ params: *draft = cast.ToBool(v) case "layout": pcfg.Layout = cast.ToString(v) - params[loki] = pcfg.Layout - case "markup": - pcfg.Content.Markup = cast.ToString(v) - params[loki] = pcfg.Content.Markup + pcfg.Params[loki] = pcfg.Layout case "weight": pcfg.Weight = cast.ToInt(v) - params[loki] = pcfg.Weight + pcfg.Params[loki] = pcfg.Weight case "aliases": pcfg.Aliases = cast.ToStringSlice(v) for i, alias := range pcfg.Aliases { @@ -572,9 +751,9 @@ params: } pcfg.Aliases[i] = filepath.ToSlash(alias) } - params[loki] = pcfg.Aliases + pcfg.Params[loki] = pcfg.Aliases case "sitemap": - pcfg.Sitemap, err = config.DecodeSitemap(p.s.conf.Sitemap, maps.ToStringMap(v)) + pcfg.Sitemap, err = config.DecodeSitemap(ps.s.conf.Sitemap, maps.ToStringMap(v)) if err != nil { return fmt.Errorf("failed to decode sitemap config in front matter: %s", err) } @@ -584,7 +763,7 @@ params: *isCJKLanguage = cast.ToBool(v) case "translationkey": pcfg.TranslationKey = cast.ToString(v) - params[loki] = pcfg.TranslationKey + pcfg.Params[loki] = pcfg.TranslationKey case "resources": var resources []map[string]any handled := true @@ -632,55 +811,59 @@ params: for i, u := range vv { a[i] = cast.ToString(u) } - params[loki] = a + pcfg.Params[loki] = a } else { - params[loki] = vv + pcfg.Params[loki] = vv } } else { - params[loki] = []string{} + pcfg.Params[loki] = []string{} } default: - params[loki] = vv + pcfg.Params[loki] = vv } } } for k, v := range userParams { - params[strings.ToLower(k)] = v + pcfg.Params[strings.ToLower(k)] = v } if !sitemapSet { - pcfg.Sitemap = p.s.conf.Sitemap + pcfg.Sitemap = ps.s.conf.Sitemap } if draft != nil && published != nil { pcfg.Draft = *draft - p.m.s.Log.Warnf("page %q has both draft and published settings in its frontmatter. Using draft.", p.File().Filename()) + ps.s.Log.Warnf("page %q has both draft and published settings in its frontmatter. Using draft.", ps.File().Filename()) } else if draft != nil { pcfg.Draft = *draft } else if published != nil { pcfg.Draft = !*published } - params["draft"] = pcfg.Draft + pcfg.Params["draft"] = pcfg.Draft if isCJKLanguage != nil { pcfg.IsCJKLanguage = *isCJKLanguage - } else if p.s.conf.HasCJKLanguage && p.m.content.pi.openSource != nil { - if cjkRe.Match(p.m.content.mustSource()) { + } else if ps.s.conf.HasCJKLanguage && ps.m.content.pi.openSource != nil { + if cjkRe.Match(ps.m.content.mustSource()) { pcfg.IsCJKLanguage = true } else { pcfg.IsCJKLanguage = false } } - params["iscjklanguage"] = pcfg.IsCJKLanguage + pcfg.Params["iscjklanguage"] = pcfg.IsCJKLanguage - if err := pcfg.Init(false); err != nil { + if err := pm.pageConfigSource.Init(false); err != nil { return err } - if err := pcfg.Compile(ext, p.s.Log, p.s.conf.OutputFormats.Config, p.s.conf.MediaTypes.Config); err != nil { + if err := pcfg.Init(); err != nil { + return err + } + + if err := pcfg.Compile(pm.pageConfigSource, ps.s.Log, ps.s.conf.OutputFormats.Config); err != nil { return err } @@ -689,13 +872,13 @@ params: // shouldList returns whether this page should be included in the list of pages. // global indicates site.Pages etc. -func (p *pageMeta) shouldList(global bool) bool { - if p.isStandalone() { +func (m *pageMeta) shouldList(global bool) bool { + if m.isStandalone() { // Never list 404, sitemap and similar. return false } - switch p.pageConfig.Build.List { + switch m.pageConfig.Build.List { case pagemeta.Always: return true case pagemeta.Never: @@ -706,102 +889,102 @@ func (p *pageMeta) shouldList(global bool) bool { return false } -func (p *pageMeta) shouldListAny() bool { - return p.shouldList(true) || p.shouldList(false) +func (m *pageMeta) shouldListAny() bool { + return m.shouldList(true) || m.shouldList(false) } -func (p *pageMeta) isStandalone() bool { - return !p.standaloneOutputFormat.IsZero() +func (m *pageMeta) isStandalone() bool { + return !m.standaloneOutputFormat.IsZero() } -func (p *pageMeta) shouldBeCheckedForMenuDefinitions() bool { - if !p.shouldList(false) { +func (m *pageMeta) shouldBeCheckedForMenuDefinitions() bool { + if !m.shouldList(false) { return false } - return p.pageConfig.Kind == kinds.KindHome || p.pageConfig.Kind == kinds.KindSection || p.pageConfig.Kind == kinds.KindPage + return m.Kind() == kinds.KindHome || m.Kind() == kinds.KindSection || m.Kind() == kinds.KindPage } -func (p *pageMeta) noRender() bool { - return p.pageConfig.Build.Render != pagemeta.Always +func (m *pageMeta) noRender() bool { + return m.pageConfig.Build.Render != pagemeta.Always } -func (p *pageMeta) noLink() bool { - return p.pageConfig.Build.Render == pagemeta.Never +func (m *pageMeta) noLink() bool { + return m.pageConfig.Build.Render == pagemeta.Never } -func (p *pageMeta) applyDefaultValues() error { - if p.pageConfig.Build.IsZero() { - p.pageConfig.Build, _ = pagemeta.DecodeBuildConfig(nil) +func (m *pageMeta) applyDefaultValues(s *Site) error { + if m.pageConfig.Build.IsZero() { + m.pageConfig.Build, _ = pagemeta.DecodeBuildConfig(nil) } - if !p.s.conf.IsKindEnabled(p.Kind()) { - (&p.pageConfig.Build).Disable() + if !s.conf.IsKindEnabled(m.Kind()) { + (&m.pageConfig.Build).Disable() } - if p.pageConfig.Content.Markup == "" { - if p.File() != nil { + if m.pageConfigSource.Content.Markup == "" { + if m.File() != nil { // Fall back to file extension - p.pageConfig.Content.Markup = p.s.ContentSpec.ResolveMarkup(p.File().Ext()) + m.pageConfigSource.Content.Markup = s.ContentSpec.ResolveMarkup(m.File().Ext()) } - if p.pageConfig.Content.Markup == "" { - p.pageConfig.Content.Markup = "markdown" + if m.pageConfigSource.Content.Markup == "" { + m.pageConfigSource.Content.Markup = "markdown" } } - if p.pageConfig.Title == "" && p.f == nil { - switch p.Kind() { + if m.pageConfig.Title == "" && m.f == nil { + switch m.Kind() { case kinds.KindHome: - p.pageConfig.Title = p.s.Title() + m.pageConfig.Title = s.Title() case kinds.KindSection: - sectionName := p.pathInfo.Unnormalized().BaseNameNoIdentifier() - if p.s.conf.PluralizeListTitles { + sectionName := m.pathInfo.Unnormalized().BaseNameNoIdentifier() + if s.conf.PluralizeListTitles { sectionName = flect.Pluralize(sectionName) } - if p.s.conf.CapitalizeListTitles { - sectionName = p.s.conf.C.CreateTitle(sectionName) + if s.conf.CapitalizeListTitles { + sectionName = s.conf.C.CreateTitle(sectionName) } - p.pageConfig.Title = sectionName + m.pageConfig.Title = sectionName case kinds.KindTerm: - if p.term != "" { - if p.s.conf.CapitalizeListTitles { - p.pageConfig.Title = p.s.conf.C.CreateTitle(p.term) + if m.term != "" { + if s.conf.CapitalizeListTitles { + m.pageConfig.Title = s.conf.C.CreateTitle(m.term) } else { - p.pageConfig.Title = p.term + m.pageConfig.Title = m.term } } else { panic("term not set") } case kinds.KindTaxonomy: - if p.s.conf.CapitalizeListTitles { - p.pageConfig.Title = strings.Replace(p.s.conf.C.CreateTitle(p.pathInfo.Unnormalized().BaseNameNoIdentifier()), "-", " ", -1) + if s.conf.CapitalizeListTitles { + m.pageConfig.Title = strings.Replace(s.conf.C.CreateTitle(m.pathInfo.Unnormalized().BaseNameNoIdentifier()), "-", " ", -1) } else { - p.pageConfig.Title = strings.Replace(p.pathInfo.Unnormalized().BaseNameNoIdentifier(), "-", " ", -1) + m.pageConfig.Title = strings.Replace(m.pathInfo.Unnormalized().BaseNameNoIdentifier(), "-", " ", -1) } case kinds.KindStatus404: - p.pageConfig.Title = "404 Page not found" + m.pageConfig.Title = "404 Page not found" } } return nil } -func (p *pageMeta) newContentConverter(ps *pageState, markup string) (converter.Converter, error) { +func (m *pageMeta) newContentConverter(ps *pageState, markup string) (converter.Converter, error) { if ps == nil { panic("no Page provided") } - cp := p.s.ContentSpec.Converters.Get(markup) + cp := ps.s.ContentSpec.Converters.Get(markup) if cp == nil { return converter.NopConverter, fmt.Errorf("no content renderer found for markup %q, page: %s", markup, ps.getPageInfoForError()) } var filename string var path string - if p.f != nil { - filename = p.f.Filename() - path = p.f.Path() + if m.f != nil { + filename = m.f.Filename() + path = m.f.Path() } else { - path = p.Path() + path = m.Path() } doc := newPageForRenderHook(ps) @@ -821,7 +1004,7 @@ func (p *pageMeta) newContentConverter(ps *pageState, markup string) (converter. converter.DocumentContext{ Document: doc, DocumentLookup: documentLookup, - DocumentID: hashing.XxHashFromStringHexEncoded(p.Path()), + DocumentID: hashing.XxHashFromStringHexEncoded(ps.Path()), DocumentName: path, Filename: filename, }, @@ -834,15 +1017,15 @@ func (p *pageMeta) newContentConverter(ps *pageState, markup string) (converter. } // The output formats this page will be rendered to. -func (m *pageMeta) outputFormats() output.Formats { - if len(m.pageConfig.ConfiguredOutputFormats) > 0 { - return m.pageConfig.ConfiguredOutputFormats +func (ps *pageState) outputFormats() output.Formats { + if len(ps.m.pageConfig.ConfiguredOutputFormats) > 0 { + return ps.m.pageConfig.ConfiguredOutputFormats } - return m.s.conf.C.KindOutputFormats[m.Kind()] + return ps.s.conf.C.KindOutputFormats[ps.Kind()] } -func (p *pageMeta) Slug() string { - return p.pageConfig.Slug +func (m *pageMeta) Slug() string { + return m.pageConfig.Slug } func getParam(m resource.ResourceParamsProvider, key string, stringToLower bool) any { @@ -879,69 +1062,3 @@ func getParam(m resource.ResourceParamsProvider, key string, stringToLower bool) func getParamToLower(m resource.ResourceParamsProvider, key string) any { return getParam(m, key, true) } - -func (ps *pageState) initLazyProviders() error { - ps.init.Add(func(ctx context.Context) (any, error) { - pp, err := newPagePaths(ps) - if err != nil { - return nil, err - } - - var outputFormatsForPage output.Formats - var renderFormats output.Formats - - if ps.m.standaloneOutputFormat.IsZero() { - outputFormatsForPage = ps.m.outputFormats() - renderFormats = ps.s.h.renderFormats - } else { - // One of the fixed output format pages, e.g. 404. - outputFormatsForPage = output.Formats{ps.m.standaloneOutputFormat} - renderFormats = outputFormatsForPage - } - - // Prepare output formats for all sites. - // We do this even if this page does not get rendered on - // its own. It may be referenced via one of the site collections etc. - // it will then need an output format. - ps.pageOutputs = make([]*pageOutput, len(renderFormats)) - created := make(map[string]*pageOutput) - shouldRenderPage := !ps.m.noRender() - - for i, f := range renderFormats { - - if po, found := created[f.Name]; found { - ps.pageOutputs[i] = po - continue - } - - render := shouldRenderPage - if render { - _, render = outputFormatsForPage.GetByName(f.Name) - } - - po := newPageOutput(ps, pp, f, render) - - // Create a content provider for the first, - // we may be able to reuse it. - if i == 0 { - contentProvider, err := newPageContentOutput(po) - if err != nil { - return nil, err - } - po.setContentProvider(contentProvider) - } - - ps.pageOutputs[i] = po - created[f.Name] = po - - } - - if err := ps.initCommonProviders(pp); err != nil { - return nil, err - } - - return nil, nil - }) - - return nil -} diff --git a/hugolib/page__new.go b/hugolib/page__new.go index 6b78dd083..dc3c53e49 100644 --- a/hugolib/page__new.go +++ b/hugolib/page__new.go @@ -16,251 +16,105 @@ package hugolib import ( "fmt" "strings" + "sync" "sync/atomic" - "github.com/gohugoio/hugo/hugofs/files" - "github.com/gohugoio/hugo/resources" - "github.com/gohugoio/hugo/common/constants" "github.com/gohugoio/hugo/common/hstore" - "github.com/gohugoio/hugo/common/maps" - "github.com/gohugoio/hugo/common/paths" - - "github.com/gohugoio/hugo/lazy" - "github.com/gohugoio/hugo/resources/kinds" "github.com/gohugoio/hugo/resources/page" - "github.com/gohugoio/hugo/resources/page/pagemeta" ) -var pageIDCounter atomic.Uint64 +var ( + pageIDCounter atomic.Uint64 + pageSourceIDCounter atomic.Uint64 +) -func (h *HugoSites) newPage(m *pageMeta) (*pageState, *paths.Path, error) { - p, pth, err := h.doNewPage(m) +func (s *Site) newPageFromPageMeta(m *pageMeta, cascades *page.PageMatcherParamsConfigs) (*pageState, error) { + p, err := s.doNewPageFromPageMeta(m, cascades) if err != nil { - // Make sure that any partially created page part is marked as stale. - m.MarkStale() + return nil, m.wrapError(err, s.SourceFs) } - - if p != nil && pth != nil && p.IsHome() && pth.IsLeafBundle() { - msg := "Using %s in your content's root directory is usually incorrect for your home page. " - msg += "You should use %s instead. If you don't rename this file, your home page will be " - msg += "treated as a leaf bundle, meaning it won't be able to have any child pages or sections." - h.Log.Warnidf(constants.WarnHomePageIsLeafBundle, msg, pth.PathNoLeadingSlash(), strings.ReplaceAll(pth.PathNoLeadingSlash(), "index", "_index")) - } - - return p, pth, err + return p, nil } -func (h *HugoSites) doNewPage(m *pageMeta) (*pageState, *paths.Path, error) { - m.Staler = &resources.AtomicStaler{} - if m.pageMetaParams == nil { - m.pageMetaParams = &pageMetaParams{ - pageConfig: &pagemeta.PageConfig{}, - } - } - if m.pageConfig.Params == nil { - m.pageConfig.Params = maps.Params{} +func (s *Site) doNewPageFromPageMeta(m *pageMeta, cascades *page.PageMatcherParamsConfigs) (*pageState, error) { + if err := m.initLate(s); err != nil { + return nil, err } - pid := pageIDCounter.Add(1) - pi, err := m.parseFrontMatter(h, pid) + // Parse the rest of the page content. + var err error + m.content, err = m.newCachedContent(s) if err != nil { - return nil, nil, err + return nil, m.wrapError(err, s.SourceFs) } - if err := m.setMetaPre(pi, h.Log, h.Conf); err != nil { - return nil, nil, m.wrapError(err, h.BaseFs.SourceFs) - } - pcfg := m.pageConfig - if pcfg.Lang != "" { - if h.Conf.IsLangDisabled(pcfg.Lang) { - return nil, nil, nil - } + ps := &pageState{ + pid: pid, + s: s, + pageOutput: nopPageOutput, + Staler: m, + dependencyManager: s.Conf.NewIdentityManager(), + pageCommon: &pageCommon{ + store: sync.OnceValue(func() *hstore.Scratch { return hstore.NewScratch() }), // Rarely used. + Positioner: page.NopPage, + InSectionPositioner: page.NopPage, + ResourceNameTitleProvider: m, + ResourceParamsProvider: m, + PageMetaProvider: m, + PageMetaInternalProvider: m, + FileProvider: m, + OutputFormatsProvider: page.NopPage, + ResourceTypeProvider: pageTypesProvider, + MediaTypeProvider: pageTypesProvider, + RefProvider: page.NopPage, + ShortcodeInfoProvider: page.NopPage, + LanguageProvider: s, + RelatedDocsHandlerProvider: s, + m: m, + s: s, + }, } - if pcfg.Path != "" { - s := m.pageConfig.Path - // Paths from content adapters should never have any extension. - if pcfg.IsFromContentAdapter || !paths.HasExt(s) { - var ( - isBranch bool - isBranchSet bool - ext string = m.pageConfig.ContentMediaType.FirstSuffix.Suffix - ) - if pcfg.Kind != "" { - isBranch = kinds.IsBranch(pcfg.Kind) - isBranchSet = true - } - - if !pcfg.IsFromContentAdapter { - if m.pathInfo != nil { - if !isBranchSet { - isBranch = m.pathInfo.IsBranchBundle() - } - if m.pathInfo.Ext() != "" { - ext = m.pathInfo.Ext() - } - } else if m.f != nil { - pi := m.f.FileInfo().Meta().PathInfo - if !isBranchSet { - isBranch = pi.IsBranchBundle() - } - if pi.Ext() != "" { - ext = pi.Ext() - } - } - } - - if isBranch { - s += "/_index." + ext - } else { - s += "/index." + ext - } - - } - m.pathInfo = h.Conf.PathParser().Parse(files.ComponentFolderContent, s) - } else if m.pathInfo == nil { - if m.f != nil { - m.pathInfo = m.f.FileInfo().Meta().PathInfo - } - - if m.pathInfo == nil { - panic(fmt.Sprintf("missing pathInfo in %v", m)) - } + if ps.IsHome() && ps.PathInfo().IsLeafBundle() { + msg := "Using %s in your content's root directory is usually incorrect for your home page. " + msg += "You should use %s instead. If you don't rename this file, your home page will be " + msg += "treated as a leaf bundle, meaning it won't be able to have any child pages or sections." + ps.s.Log.Warnidf(constants.WarnHomePageIsLeafBundle, msg, ps.PathInfo().PathNoLeadingSlash(), strings.ReplaceAll(ps.PathInfo().PathNoLeadingSlash(), "index", "_index")) } - ps, err := func() (*pageState, error) { - if m.s == nil { - // Identify the Site/language to associate this Page with. - var lang string - if pcfg.Lang != "" { - lang = pcfg.Lang - } else if m.f != nil { - meta := m.f.FileInfo().Meta() - lang = meta.Lang - } else { - lang = m.pathInfo.Lang() - } - - m.s = h.resolveSite(lang) - - if m.s == nil { - return nil, fmt.Errorf("no site found for language %q", lang) - } - } - - var tc viewName - // Identify Page Kind. - if m.pageConfig.Kind == "" { - m.pageConfig.Kind = kinds.KindSection - if m.pathInfo.Base() == "/" { - m.pageConfig.Kind = kinds.KindHome - } else if m.pathInfo.IsBranchBundle() { - // A section, taxonomy or term. - tc = m.s.pageMap.cfg.getTaxonomyConfig(m.Path()) - if !tc.IsZero() { - // Either a taxonomy or a term. - if tc.pluralTreeKey == m.Path() { - m.pageConfig.Kind = kinds.KindTaxonomy - } else { - m.pageConfig.Kind = kinds.KindTerm - } - } - } else if m.f != nil { - m.pageConfig.Kind = kinds.KindPage - } - } - - if m.pageConfig.Kind == kinds.KindTerm || m.pageConfig.Kind == kinds.KindTaxonomy { - if tc.IsZero() { - tc = m.s.pageMap.cfg.getTaxonomyConfig(m.Path()) - } - if tc.IsZero() { - return nil, fmt.Errorf("no taxonomy configuration found for %q", m.Path()) - } - m.singular = tc.singular - if m.pageConfig.Kind == kinds.KindTerm { - m.term = paths.TrimLeading(strings.TrimPrefix(m.pathInfo.Unnormalized().Base(), tc.pluralTreeKey)) - } - } - - if m.pageConfig.Kind == kinds.KindPage && !m.s.conf.IsKindEnabled(m.pageConfig.Kind) { - return nil, nil - } - - // Parse the rest of the page content. - m.content, err = m.newCachedContent(h, pi) + if m.f != nil { + gi, err := s.h.gitInfoForPage(ps) if err != nil { - return nil, m.wrapError(err, h.SourceFs) - } - - ps := &pageState{ - pid: pid, - pageOutput: nopPageOutput, - pageOutputTemplateVariationsState: &atomic.Uint32{}, - Staler: m, - dependencyManager: m.s.Conf.NewIdentityManager(m.Path()), - pageCommon: &pageCommon{ - FileProvider: m, - store: hstore.NewScratch(), - Positioner: page.NopPage, - InSectionPositioner: page.NopPage, - ResourceNameTitleProvider: m, - ResourceParamsProvider: m, - PageMetaProvider: m, - PageMetaInternalProvider: m, - OutputFormatsProvider: page.NopPage, - ResourceTypeProvider: pageTypesProvider, - MediaTypeProvider: pageTypesProvider, - RefProvider: page.NopPage, - ShortcodeInfoProvider: page.NopPage, - LanguageProvider: m.s, - - RelatedDocsHandlerProvider: m.s, - init: lazy.New(), - m: m, - s: m.s, - sWrapped: page.WrapSite(m.s), - }, - } - - if m.f != nil { - gi, err := m.s.h.gitInfoForPage(ps) - if err != nil { - return nil, fmt.Errorf("failed to load Git data: %w", err) - } - ps.gitInfo = gi - owners, err := m.s.h.codeownersForPage(ps) - if err != nil { - return nil, fmt.Errorf("failed to load CODEOWNERS: %w", err) - } - ps.codeowners = owners + return nil, fmt.Errorf("failed to load Git data: %w", err) } - - ps.pageMenus = &pageMenus{p: ps} - ps.PageMenusProvider = ps.pageMenus - ps.GetPageProvider = pageSiteAdapter{s: m.s, p: ps} - ps.GitInfoProvider = ps - ps.TranslationsProvider = ps - ps.ResourceDataProvider = &pageData{pageState: ps} - ps.RawContentProvider = ps - ps.ChildCareProvider = ps - ps.TreeProvider = pageTree{p: ps} - ps.Eqer = ps - ps.TranslationKeyProvider = ps - ps.ShortcodeInfoProvider = ps - ps.AlternativeOutputFormatsProvider = ps - - if err := ps.initLazyProviders(); err != nil { - return nil, ps.wrapError(err) + ps.gitInfo = gi + owners, err := s.h.codeownersForPage(ps) + if err != nil { + return nil, fmt.Errorf("failed to load CODEOWNERS: %w", err) } - return ps, nil - }() + ps.codeowners = owners + } - if ps == nil { - return nil, nil, err + ps.pageMenus = &pageMenus{p: ps} + ps.PageMenusProvider = ps.pageMenus + ps.GetPageProvider = pageSiteAdapter{s: s, p: ps} + ps.GitInfoProvider = ps + ps.TranslationsProvider = ps + ps.ResourceDataProvider = newDataFunc(ps) + ps.RawContentProvider = ps + ps.ChildCareProvider = ps + ps.TreeProvider = pageTree{p: ps} + ps.Eqer = ps + ps.TranslationKeyProvider = ps + ps.ShortcodeInfoProvider = ps + ps.AlternativeOutputFormatsProvider = ps + + // Combine the cascade map with front matter. + if err = ps.setMetaPost(cascades); err != nil { + return nil, err } - return ps, ps.PathInfo(), err + return ps, nil } diff --git a/hugolib/page__output.go b/hugolib/page__output.go index b8086bb48..2c5e1ca04 100644 --- a/hugolib/page__output.go +++ b/hugolib/page__output.go @@ -22,6 +22,10 @@ import ( "github.com/gohugoio/hugo/resources/resource" ) +var paginatorNotSupported = page.PaginatorNotSupportedFunc(func() error { + return fmt.Errorf("pagination not supported for this page") +}) + func newPageOutput( ps *pageState, pp pagePaths, @@ -46,9 +50,7 @@ func newPageOutput( pag = newPagePaginator(ps) paginatorProvider = pag } else { - paginatorProvider = page.PaginatorNotSupportedFunc(func() error { - return fmt.Errorf("pagination not supported for this page: %s", ps.getPageInfoForError()) - }) + paginatorProvider = paginatorNotSupported } providers := struct { @@ -71,7 +73,7 @@ func newPageOutput( TableOfContentsProvider: page.NopPage, render: render, paginator: pag, - dependencyManagerOutput: ps.s.Conf.NewIdentityManager((ps.Path() + "/" + f.Name)), + dependencyManagerOutput: ps.s.Conf.NewIdentityManager(), } return po diff --git a/hugolib/page__paths.go b/hugolib/page__paths.go index 62206cb15..85b79238d 100644 --- a/hugolib/page__paths.go +++ b/hugolib/page__paths.go @@ -15,6 +15,7 @@ package hugolib import ( "net/url" + "path" "strings" "github.com/gohugoio/hugo/output" @@ -37,7 +38,7 @@ func newPagePaths(ps *pageState) (pagePaths, error) { if ps.m.isStandalone() { outputFormats = output.Formats{ps.m.standaloneOutputFormat} } else { - outputFormats = pm.outputFormats() + outputFormats = ps.outputFormats() if len(outputFormats) == 0 { return pagePaths{}, nil } @@ -113,8 +114,8 @@ func createTargetPathDescriptor(p *pageState) (page.TargetPathDescriptor, error) d := s.Deps pm := p.m alwaysInSubDir := p.Kind() == kinds.KindSitemap - pageInfoPage := p.PathInfo() + pageInfoCurrentSection := p.CurrentSection().PathInfo() if p.s.Conf.DisablePathToLower() { pageInfoPage = pageInfoPage.Unnormalized() @@ -139,8 +140,31 @@ func createTargetPathDescriptor(p *pageState) (page.TargetPathDescriptor, error) desc.BaseName = pageInfoPage.BaseNameNoIdentifier() } - desc.PrefixFilePath = s.getLanguageTargetPathLang(alwaysInSubDir) - desc.PrefixLink = s.getLanguagePermalinkLang(alwaysInSubDir) + addPrefix := func(filePath, link string) { + if filePath != "" { + if desc.PrefixFilePath != "" { + desc.PrefixFilePath += "/" + filePath + } else { + desc.PrefixFilePath += filePath + } + } + + if link != "" { + if desc.PrefixLink != "" { + desc.PrefixLink += "/" + link + } else { + desc.PrefixLink += link + } + } + } + + // Add path prefixes. + // Add any role first, as that is a natural candidate for external ACL checks. + rolePrefix := s.getPrefixRole() + addPrefix(rolePrefix, rolePrefix) + versionPrefix := s.getPrefixVersion() + addPrefix(versionPrefix, versionPrefix) + addPrefix(s.getLanguageTargetPathLang(alwaysInSubDir), s.getLanguagePermalinkLang(alwaysInSubDir)) if desc.URL != "" && strings.IndexByte(desc.URL, ':') >= 0 { // Attempt to parse and expand an url @@ -162,11 +186,7 @@ func createTargetPathDescriptor(p *pageState) (page.TargetPathDescriptor, error) if opath != "" { opath, _ = url.QueryUnescape(opath) - if strings.HasSuffix(opath, "//") { - // When rewriting the _index of the section the permalink config is applied to, - // we get double slashes at the end sometimes; clear them up here - opath = strings.TrimSuffix(opath, "/") - } + opath = path.Clean(opath) desc.ExpandedPermalink = opath diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go index 15e9a890c..b6e9f6602 100644 --- a/hugolib/page__per_output.go +++ b/hugolib/page__per_output.go @@ -352,10 +352,12 @@ func (pco *pageContentOutput) initRenderHooks() error { Path: base, Category: tplimpl.CategoryMarkup, Desc: layoutDescriptor, + Sites: pco.po.p.s.siteVector, Consider: consider, } v := pco.po.p.s.TemplateStore.LookupPagesLayout(q) + return v, v != nil } diff --git a/hugolib/page__position.go b/hugolib/page__position.go index d55ebeb07..681ad15c9 100644 --- a/hugolib/page__position.go +++ b/hugolib/page__position.go @@ -16,8 +16,8 @@ package hugolib import ( "context" + "github.com/gohugoio/hugo/common/hsync" "github.com/gohugoio/hugo/common/hugo" - "github.com/gohugoio/hugo/lazy" "github.com/gohugoio/hugo/resources/page" ) @@ -30,7 +30,7 @@ func newPagePositionInSection(n *nextPrev) pagePositionInSection { } type nextPrev struct { - init *lazy.Init + init hsync.FuncResetter prevPage page.Page nextPage page.Page } diff --git a/hugolib/page__ref.go b/hugolib/page__ref.go index e55a8a3e4..6badbd3b8 100644 --- a/hugolib/page__ref.go +++ b/hugolib/page__ref.go @@ -87,7 +87,7 @@ func (p pageRef) ref(argsm map[string]any, source any) (string, error) { return "", nil } - return s.refLink(args.Path, source, false, args.OutputFormat) + return s.siteRefLinker.refLink(args.Path, source, false, args.OutputFormat) } func (p pageRef) relRef(argsm map[string]any, source any) (string, error) { @@ -104,7 +104,7 @@ func (p pageRef) relRef(argsm map[string]any, source any) (string, error) { return "", nil } - return s.refLink(args.Path, source, true, args.OutputFormat) + return s.siteRefLinker.refLink(args.Path, source, true, args.OutputFormat) } type refArgs struct { diff --git a/hugolib/page__tree.go b/hugolib/page__tree.go index cccfb8904..599683758 100644 --- a/hugolib/page__tree.go +++ b/hugolib/page__tree.go @@ -31,7 +31,7 @@ type pageTree struct { } func (pt pageTree) IsAncestor(other any) bool { - n, ok := other.(contentNodeI) + n, ok := other.(contentNode) if !ok { return false } @@ -44,7 +44,7 @@ func (pt pageTree) IsAncestor(other any) bool { } func (pt pageTree) IsDescendant(other any) bool { - n, ok := other.(contentNodeI) + n, ok := other.(contentNode) if !ok { return false } @@ -62,16 +62,23 @@ func (pt pageTree) CurrentSection() page.Page { } dir := pt.p.m.pathInfo.Dir() + if dir == "/" { + if pt.p.s.home == nil { + panic(fmt.Sprintf("[%v] home page is nil for %q", pt.p.s.siteVector, pt.p.Path())) + } return pt.p.s.home } - _, n := pt.p.s.pageMap.treePages.LongestPrefix(dir, true, func(n contentNodeI) bool { return n.isContentNodeBranch() }) + _, n := pt.p.s.pageMap.treePages.LongestPrefix(dir, false, func(n contentNode) bool { + return cnh.isBranchNode(n) + }) + if n != nil { return n.(page.Page) } - panic(fmt.Sprintf("CurrentSection not found for %q in lang %s", pt.p.Path(), pt.p.Lang())) + panic(fmt.Sprintf("CurrentSection not found for %q for %s", pt.p.Path(), pt.p.s.resolveDimensionNames())) } func (pt pageTree) FirstSection() page.Page { @@ -81,7 +88,7 @@ func (pt pageTree) FirstSection() page.Page { } for { - k, n := pt.p.s.pageMap.treePages.LongestPrefix(s, true, func(n contentNodeI) bool { return n.isContentNodeBranch() }) + k, n := pt.p.s.pageMap.treePages.LongestPrefix(s, false, func(n contentNode) bool { return cnh.isBranchNode(n) }) if n == nil { return nil } @@ -125,11 +132,11 @@ func (pt pageTree) Parent() page.Page { } for { - _, n := pt.p.s.pageMap.treePages.LongestPrefix(dir, true, nil) + _, n := pt.p.s.pageMap.treePages.LongestPrefix(dir, false, nil) if n == nil { return pt.p.s.home } - if pt.p.m.bundled || n.isContentNodeBranch() { + if pt.p.m.bundled || cnh.isBranchNode(n) { return n.(page.Page) } dir = paths.Dir(dir) @@ -142,6 +149,7 @@ func (pt pageTree) Ancestors() page.Pages { for parent != nil { ancestors = append(ancestors, parent) parent = parent.Parent() + } return ancestors } @@ -155,12 +163,12 @@ func (pt pageTree) Sections() page.Pages { tree = pt.p.s.pageMap.treePages ) - w := &doctree.NodeShiftTreeWalker[contentNodeI]{ + w := &doctree.NodeShiftTreeWalker[contentNode]{ Tree: tree, Prefix: prefix, } - w.Handle = func(ss string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { - if !n.isContentNodeBranch() { + w.Handle = func(ss string, n contentNode) (bool, error) { + if !cnh.isBranchNode(n) { return false, nil } if currentBranchPrefix == "" || !strings.HasPrefix(ss, currentBranchPrefix) { diff --git a/hugolib/page_test.go b/hugolib/page_test.go index 91499da8f..e8f8ced02 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -83,14 +83,6 @@ The [best static site generator][hugo].[^1] [hugo]: http://gohugo.io/ [^1]: Many people say so. -` - simplePageWithShortcodeInSummary = `--- -title: Simple ---- -Summary Next Line. {{
}}. -More text here. - -Some more text ` simplePageWithSummaryDelimiterSameLine = `--- @@ -130,13 +122,6 @@ More then 70 words. ` - simplePageWithMainEnglishWithCJKRunesSummary = "In Chinese, 好 means good. In Chinese, 好 means good. " + - "In Chinese, 好 means good. In Chinese, 好 means good. " + - "In Chinese, 好 means good. In Chinese, 好 means good. " + - "In Chinese, 好 means good. In Chinese, 好 means good. " + - "In Chinese, 好 means good. In Chinese, 好 means good. " + - "In Chinese, 好 means good. In Chinese, 好 means good. " + - "In Chinese, 好 means good. In Chinese, 好 means good." simplePageWithIsCJKLanguageFalse = `--- title: Simple @@ -154,13 +139,6 @@ More then 70 words. ` - simplePageWithIsCJKLanguageFalseSummary = "In Chinese, 好的啊 means good. In Chinese, 好的呀 means good. " + - "In Chinese, 好的啊 means good. In Chinese, 好的呀 means good. " + - "In Chinese, 好的啊 means good. In Chinese, 好的呀 means good. " + - "In Chinese, 好的啊 means good. In Chinese, 好的呀 means good. " + - "In Chinese, 好的啊 means good. In Chinese, 好的呀 means good. " + - "In Chinese, 好的啊 means good. In Chinese, 好的呀 means good. " + - "In Chinese, 好的啊 means good. In Chinese, 好的呀呀 means good enough." simplePageWithLongContent = `--- title: Simple @@ -1126,6 +1104,7 @@ Content b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Configs: configs}).WithNothingAdded() b.Build(BuildCfg{SkipRender: true}) + c.Assert(len(b.H.Sites), qt.Equals, 1) s := b.H.Sites[0] c.Assert(len(s.RegularPages()), qt.Equals, 2) @@ -1402,6 +1381,41 @@ Resources: {{ range .Resources }}{{ .RelPermalink }}|{{ .Content }}|{{ end }}| ) } +func TestTranslationKeyRotate(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ['home','rss','section','sitemap','taxonomy'] +defaultContentLanguage = 'en' +defaultContentLanguageInSubdir = true +[languages] +[languages.en] +weight = 1 +[languages.pt] +weight = 2 +-- content/foo.md -- +--- +title: Foo +translationkey: "mykey" +--- +-- content/bar.md -- +--- +title: Bar +translationkey: "mykey" +--- +-- layouts/all.html -- +Rotate(language): {{ with .Rotate "language" }}{{ range . }}{{ template "printp" . }}|{{ end }}{{ end }}$ +{{ define "printp" }}{{ .RelPermalink }}:{{ with .Site }}{{ template "prints" . }}{{ end }}{{ end }} +{{ define "prints" }}/l:{{ .Language.Name }}/v:{{ .Version.Name }}/r:{{ .Role.Name }}{{ end }} +` + + b := Test(t, files) + + b.AssertFileContent("public/en/foo/index.html", "Rotate(language): /en/bar/:/l:en/v:v1/r:guest|/en/foo/:/l:en/v:v1/r:guest|$") + b.AssertFileContent("public/en/bar/index.html", "Rotate(language): /en/bar/:/l:en/v:v1/r:guest|/en/foo/:/l:en/v:v1/r:guest|$") +} + func TestChompBOM(t *testing.T) { t.Parallel() c := qt.New(t) @@ -1944,32 +1958,6 @@ Hugo: h-Home| ) } -// See #12484 -func TestPageFrontMatterDeprecatePathKindLang(t *testing.T) { - // This cannot be parallel as it depends on output from the global logger. - - files := ` --- hugo.toml -- -disableKinds = ["taxonomy", "term", "home", "section"] --- content/p1.md -- ---- -title: "p1" -kind: "page" -lang: "en" -path: "mypath" ---- --- layouts/_default/single.html -- -Title: {{ .Title }} -` - b := Test(t, files, TestOptWarn()) - b.AssertFileContent("public/mypath/index.html", "p1") - b.AssertLogContains( - "deprecated: kind in front matter was deprecated", - "deprecated: lang in front matter was deprecated", - "deprecated: path in front matter was deprecated", - ) -} - // Issue 13538 func TestHomePageIsLeafBundle(t *testing.T) { t.Parallel() diff --git a/hugolib/pagebundler_test.go b/hugolib/pagebundler_test.go index e5521412b..dcdd7af4f 100644 --- a/hugolib/pagebundler_test.go +++ b/hugolib/pagebundler_test.go @@ -36,6 +36,41 @@ import ( qt "github.com/frankban/quicktest" ) +func TestPageBundlerBasic(t *testing.T) { + files := ` +-- hugo.toml -- +-- content/mybundle/index.md -- +--- +title: "My Bundle" +--- +-- content/mybundle/p1.md -- +--- +title: "P1" +--- +-- content/mybundle/p1.html -- +--- +title: "P1 HTML" +--- +-- content/mybundle/data.txt -- +Data txt. +-- content/mybundle/sub/data.txt -- +Data sub txt. +-- content/mybundle/sub/index.md -- +-- content/mysection/_index.md -- +--- +title: "My Section" +--- +-- content/mysection/data.txt -- +Data section txt. +-- layouts/all.html -- +All. {{ .Title }}|{{ .RelPermalink }}|{{ .Kind }}|{{ .BundleType }}| +` + + b := Test(t, files) + + b.AssertFileContent("public/mybundle/index.html", "My Bundle|/mybundle/|page|leaf|") +} + func TestPageBundlerBundleInRoot(t *testing.T) { t.Parallel() files := ` @@ -227,6 +262,66 @@ Len Sites: {{ len .Site.Sites }}| b.Assert(len(b.H.Sites), qt.Equals, 1) } +func TestMultilingualDisableLanguageMounts(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +baseURL = "https://example.com" +disableKinds = ["taxonomy", "term"] +defaultContentLanguage = "en" +defaultContentLanguageInSubdir = true +[[module.mounts]] +source = 'content/nn' +target = 'content' +[module.mounts.sites.matrix] +languages = "nn" +[[module.mounts]] +source = 'content/en' +target = 'content' +[module.mounts.sites.matrix] +languages = "en" + +[languages] +[languages.en] +weight = 1 +[languages.nn] +weight = 2 +disabled = true +-- content/en/mysect/_index.md -- +--- +title: "My Sect En" +--- +-- content/en/mysect/p1/index.md -- +--- +title: "P1" +--- +P1 +-- content/nn/mysect/_index.md -- +--- +title: "My Sect Nn" +--- +-- content/nn/mysect/p1/index.md -- +--- +title: "P1nn" +--- +P1nn +-- layouts/index.html -- +Len RegularPages: {{ len .Site.RegularPages }}|RegularPages: {{ range site.RegularPages }}{{ .RelPermalink }}: {{ .Title }}|{{ end }}| +Len Pages: {{ len .Site.Pages }}| +Len Sites: {{ len .Site.Sites }}| +-- layouts/_default/single.html -- +{{ .Title }}|{{ .Content }}|{{ .Lang }}| + +` + b := Test(t, files) + + b.AssertFileContent("public/en/index.html", "Len RegularPages: 1|") + b.AssertFileContent("public/en/mysect/p1/index.html", "P1|

P1

\n|en|") + b.AssertFileExists("public/public/nn/mysect/p1/index.html", false) + b.Assert(len(b.H.Sites), qt.Equals, 1) +} + func TestPageBundlerHeadless(t *testing.T) { t.Parallel() diff --git a/hugolib/pagecollections.go b/hugolib/pagecollections.go index f1038deff..5c29baddb 100644 --- a/hugolib/pagecollections.go +++ b/hugolib/pagecollections.go @@ -30,14 +30,14 @@ import ( // pageFinder provides ways to find a Page in a Site. type pageFinder struct { - pageMap *pageMap + pm *pageMap } func newPageFinder(m *pageMap) *pageFinder { if m == nil { panic("must provide a pageMap") } - c := &pageFinder{pageMap: m} + c := &pageFinder{pm: m} return c } @@ -115,7 +115,7 @@ func (c *pageFinder) getPageForRefs(ref ...string) (page.Page, error) { const defaultContentExt = ".md" -func (c *pageFinder) getContentNode(context page.Page, isReflink bool, ref string) (contentNodeI, error) { +func (c *pageFinder) getContentNode(context page.Page, isReflink bool, ref string) (contentNode, error) { ref = paths.ToSlashTrimTrailing(ref) inRef := ref if ref == "" { @@ -148,8 +148,8 @@ func (c *pageFinder) getContentNode(context page.Page, isReflink bool, ref strin return nil, nil } -func (c *pageFinder) getContentNodeForRef(context page.Page, isReflink, hadExtension bool, inRef, ref string) (contentNodeI, error) { - s := c.pageMap.s +func (c *pageFinder) getContentNodeForRef(context page.Page, isReflink, hadExtension bool, inRef, ref string) (contentNode, error) { + s := c.pm.s contentPathParser := s.Conf.PathParser() if context != nil && !strings.HasPrefix(ref, "/") { @@ -171,7 +171,7 @@ func (c *pageFinder) getContentNodeForRef(context page.Page, isReflink, hadExten relPath, _ := contentPathParser.ParseBaseAndBaseNameNoIdentifier(files.ComponentFolderContent, rel) - n, err := c.getContentNodeFromPath(relPath, ref) + n, err := c.getContentNodeFromPath(relPath) if n != nil || err != nil { return n, err } @@ -191,7 +191,7 @@ func (c *pageFinder) getContentNodeForRef(context page.Page, isReflink, hadExten relPath, nameNoIdentifier := contentPathParser.ParseBaseAndBaseNameNoIdentifier(files.ComponentFolderContent, ref) - n, err := c.getContentNodeFromPath(relPath, ref) + n, err := c.getContentNodeFromPath(relPath) if n != nil || err != nil { return n, err @@ -213,7 +213,7 @@ func (c *pageFinder) getContentNodeForRef(context page.Page, isReflink, hadExten return nil, nil } - n = c.pageMap.pageReverseIndex.Get(nameNoIdentifier) + n = c.pm.pageReverseIndex.Get(nameNoIdentifier) if n == ambiguousContentNode { return nil, fmt.Errorf("page reference %q is ambiguous", inRef) } @@ -221,8 +221,8 @@ func (c *pageFinder) getContentNodeForRef(context page.Page, isReflink, hadExten return n, nil } -func (c *pageFinder) getContentNodeFromRefReverseLookup(ref string, fi hugofs.FileMetaInfo) (contentNodeI, error) { - s := c.pageMap.s +func (c *pageFinder) getContentNodeFromRefReverseLookup(ref string, fi hugofs.FileMetaInfo) (contentNode, error) { + s := c.pm.s meta := fi.Meta() dir := meta.Filename if !fi.IsDir() { @@ -239,15 +239,15 @@ func (c *pageFinder) getContentNodeFromRefReverseLookup(ref string, fi hugofs.Fi // There may be multiple matches, but we will only use the first one. for _, pc := range pcs { pi := s.Conf.PathParser().Parse(pc.Component, pc.Path) - if n := c.pageMap.treePages.Get(pi.Base()); n != nil { + if n := c.pm.treePages.Get(pi.Base()); n != nil { return n, nil } } return nil, nil } -func (c *pageFinder) getContentNodeFromPath(s string, ref string) (contentNodeI, error) { - n := c.pageMap.treePages.Get(s) +func (c *pageFinder) getContentNodeFromPath(s string) (contentNode, error) { + n := c.pm.treePages.Get(s) if n != nil { return n, nil } diff --git a/hugolib/pages_capture.go b/hugolib/pages_capture.go index 172c9e17b..7de458a6f 100644 --- a/hugolib/pages_capture.go +++ b/hugolib/pages_capture.go @@ -24,7 +24,6 @@ import ( "time" "github.com/bep/logg" - "github.com/gohugoio/hugo/common/hstrings" "github.com/gohugoio/hugo/common/paths" "github.com/gohugoio/hugo/common/rungroup" "github.com/spf13/afero" @@ -84,13 +83,13 @@ type pagesCollector struct { // It may be restricted by filenames set on the collector (partial build). func (c *pagesCollector) Collect() (collectErr error) { var ( - numWorkers = c.h.numWorkers - numFilesProcessedTotal atomic.Uint64 - numPagesProcessedTotal atomic.Uint64 - numResourcesProcessed atomic.Uint64 - numFilesProcessedLast uint64 - fileBatchTimer = time.Now() - fileBatchTimerMu sync.Mutex + numWorkers = c.h.numWorkers + numFilesProcessedTotal atomic.Uint64 + numPageSourcesProcessedTotal atomic.Uint64 + numResourceSourcesProcessed atomic.Uint64 + numFilesProcessedLast uint64 + fileBatchTimer = time.Now() + fileBatchTimerMu sync.Mutex ) l := c.infoLogger.WithField("substep", "collect") @@ -104,8 +103,8 @@ func (c *pagesCollector) Collect() (collectErr error) { logg.Fields{ logg.Field{Name: "files", Value: numFilesProcessedBatch}, logg.Field{Name: "files_total", Value: numFilesProcessedTotal.Load()}, - logg.Field{Name: "pages_total", Value: numPagesProcessedTotal.Load()}, - logg.Field{Name: "resources_total", Value: numResourcesProcessed.Load()}, + logg.Field{Name: "pagesources_total", Value: numPageSourcesProcessedTotal.Load()}, + logg.Field{Name: "resourcesources_total", Value: numResourceSourcesProcessed.Load()}, }, "", ) @@ -121,16 +120,17 @@ func (c *pagesCollector) Collect() (collectErr error) { c.g = rungroup.Run(c.ctx, rungroup.Config[hugofs.FileMetaInfo]{ NumWorkers: numWorkers, Handle: func(ctx context.Context, fi hugofs.FileMetaInfo) error { - numPages, numResources, err := c.m.AddFi(fi, c.buildConfig) + numPageSources, numResourceSources, err := c.m.AddFi(fi, c.buildConfig) if err != nil { return hugofs.AddFileInfoToError(err, fi, c.h.SourceFs) } numFilesProcessedTotal.Add(1) - numPagesProcessedTotal.Add(numPages) - numResourcesProcessed.Add(numResources) + numPageSourcesProcessedTotal.Add(numPageSources) + numResourceSourcesProcessed.Add(numResourceSources) if numFilesProcessedTotal.Load()%1000 == 0 { logFilesProcessed(false) } + return nil }, }) @@ -255,6 +255,8 @@ func (c *pagesCollector) collectDir(dirPath *paths.Path, isDir bool, inFilter fu } func (c *pagesCollector) collectDirDir(path string, root hugofs.FileMetaInfo, inFilter func(fim hugofs.FileMetaInfo) bool) error { + ctx := context.Background() + filter := func(fim hugofs.FileMetaInfo) bool { if inFilter != nil { return inFilter(fim) @@ -262,7 +264,7 @@ func (c *pagesCollector) collectDirDir(path string, root hugofs.FileMetaInfo, in return true } - preHook := func(dir hugofs.FileMetaInfo, path string, readdir []hugofs.FileMetaInfo) ([]hugofs.FileMetaInfo, error) { + preHook := func(ctx context.Context, dir hugofs.FileMetaInfo, path string, readdir []hugofs.FileMetaInfo) ([]hugofs.FileMetaInfo, error) { filtered := readdir[:0] for _, fi := range readdir { if filter(fi) { @@ -312,13 +314,13 @@ func (c *pagesCollector) collectDirDir(path string, root hugofs.FileMetaInfo, in } if firstPi.IsLeafBundle() { - if err := c.handleBundleLeaf(dir, first, path, readdir); err != nil { + if err := c.handleBundleLeaf(ctx, dir, path, readdir); err != nil { return nil, err } return nil, filepath.SkipDir } - seen := map[hstrings.Strings2]hugofs.FileMetaInfo{} + // seen := map[types.Strings2]hugofs.FileMetaInfo{} for _, fi := range readdir { if fi.IsDir() { continue @@ -327,27 +329,10 @@ func (c *pagesCollector) collectDirDir(path string, root hugofs.FileMetaInfo, in pi := fi.Meta().PathInfo meta := fi.Meta() - // Filter out duplicate page or resource. - // These would eventually have been filtered out as duplicates when - // inserting them into the document store, - // but doing it here will preserve a consistent ordering. - baseLang := hstrings.Strings2{pi.Base(), meta.Lang} - if fi2, ok := seen[baseLang]; ok { - if c.h.Configs.Base.PrintPathWarnings && !c.h.isRebuild() { - c.logger.Infof("Duplicate content path: %q file: %q file: %q", pi.Base(), fi2.Meta().Filename, meta.Filename) - } - continue - } - seen[baseLang] = fi - if pi == nil { panic(fmt.Sprintf("no path info for %q", meta.Filename)) } - if meta.Lang == "" { - panic("lang not set") - } - if err := c.g.Enqueue(fi); err != nil { return nil, err } @@ -359,7 +344,7 @@ func (c *pagesCollector) collectDirDir(path string, root hugofs.FileMetaInfo, in var postHook hugofs.WalkHook - wfn := func(path string, fi hugofs.FileMetaInfo) error { + wfn := func(ctx context.Context, path string, fi hugofs.FileMetaInfo) error { return nil } @@ -370,6 +355,7 @@ func (c *pagesCollector) collectDirDir(path string, root hugofs.FileMetaInfo, in Info: root, Fs: c.fs, IgnoreFile: c.h.SourceSpec.IgnoreFile, + Ctx: ctx, PathParser: c.h.Conf.PathParser(), HookPre: preHook, HookPost: postHook, @@ -379,36 +365,12 @@ func (c *pagesCollector) collectDirDir(path string, root hugofs.FileMetaInfo, in return w.Walk() } -func (c *pagesCollector) handleBundleLeaf(dir, bundle hugofs.FileMetaInfo, inPath string, readdir []hugofs.FileMetaInfo) error { - bundlePi := bundle.Meta().PathInfo - seen := map[hstrings.Strings2]bool{} - - walk := func(path string, info hugofs.FileMetaInfo) error { +func (c *pagesCollector) handleBundleLeaf(ctx context.Context, dir hugofs.FileMetaInfo, inPath string, readdir []hugofs.FileMetaInfo) error { + walk := func(ctx context.Context, path string, info hugofs.FileMetaInfo) error { if info.IsDir() { return nil } - pi := info.Meta().PathInfo - - if info != bundle { - // Everything inside a leaf bundle is a Resource, - // even the content pages. - // Note that we do allow index.md as page resources, but not in the bundle root. - if !pi.IsLeafBundle() || pi.Dir() != bundlePi.Dir() { - paths.ModifyPathBundleTypeResource(pi) - } - } - - // Filter out duplicate page or resource. - // These would eventually have been filtered out as duplicates when - // inserting them into the document store, - // but doing it here will preserve a consistent ordering. - baseLang := hstrings.Strings2{pi.Base(), info.Meta().Lang} - if seen[baseLang] { - return nil - } - seen[baseLang] = true - return c.g.Enqueue(info) } @@ -420,6 +382,7 @@ func (c *pagesCollector) handleBundleLeaf(dir, bundle hugofs.FileMetaInfo, inPat Logger: c.logger, Info: dir, DirEntries: readdir, + Ctx: ctx, IgnoreFile: c.h.SourceSpec.IgnoreFile, PathParser: c.h.Conf.PathParser(), WalkFn: walk, diff --git a/hugolib/pagesfromdata/pagesfromgotmpl.go b/hugolib/pagesfromdata/pagesfromgotmpl.go index 2cb01ae1d..db2931503 100644 --- a/hugolib/pagesfromdata/pagesfromgotmpl.go +++ b/hugolib/pagesfromdata/pagesfromgotmpl.go @@ -64,55 +64,62 @@ type pagesFromDataTemplateContext struct { p *PagesFromTemplate } -func (p *pagesFromDataTemplateContext) toPathMap(v any) (string, map[string]any, error) { +func (p *pagesFromDataTemplateContext) toPathSitesMap(v any) (string, map[string]any, map[string]any, error) { m, err := maps.ToStringMapE(v) if err != nil { - return "", nil, err + return "", nil, nil, err } path, err := cast.ToStringE(m["path"]) if err != nil { - return "", nil, fmt.Errorf("invalid path %q", path) + return "", nil, nil, fmt.Errorf("invalid path %q", path) } - return path, m, nil + + sites := maps.ToStringMap(m["sites"]) + + return path, sites, m, nil } func (p *pagesFromDataTemplateContext) AddPage(v any) (string, error) { - path, m, err := p.toPathMap(v) + path, sites, m, err := p.toPathSitesMap(v) if err != nil { return "", err } - if !p.p.buildState.checkHasChangedAndSetSourceInfo(path, m) { + hash, hasChanged := p.p.buildState.checkHasChangedAndSetSourceInfo(path, sites, m) + if !hasChanged { return "", nil } - pd := pagemeta.DefaultPageConfig - pd.IsFromContentAdapter = true - pd.ContentAdapterData = m + pe := &pagemeta.PageConfigEarly{ + IsFromContentAdapter: true, + Frontmatter: m, + SourceEntryHash: hash, + } // The rest will be handled after the cascade is calculated and applied. - if err := mapstructure.WeakDecode(pd.ContentAdapterData, &pd.PageConfigEarly); err != nil { + if err := mapstructure.WeakDecode(pe.Frontmatter, pe); err != nil { err = fmt.Errorf("failed to decode page map: %w", err) return "", err } - if err := pd.Init(true); err != nil { + if err := pe.Init(true); err != nil { return "", err } p.p.buildState.NumPagesAdded++ - return "", p.p.HandlePage(p.p, &pd) + return "", p.p.HandlePage(p.p, pe) } func (p *pagesFromDataTemplateContext) AddResource(v any) (string, error) { - path, m, err := p.toPathMap(v) + path, sites, m, err := p.toPathSitesMap(v) if err != nil { return "", err } - if !p.p.buildState.checkHasChangedAndSetSourceInfo(path, m) { + hash, hasChanged := p.p.buildState.checkHasChangedAndSetSourceInfo(path, sites, m) + if !hasChanged { return "", nil } @@ -120,6 +127,7 @@ func (p *pagesFromDataTemplateContext) AddResource(v any) (string, error) { if err := mapstructure.WeakDecode(m, &rd); err != nil { return "", err } + rd.ContentAdapterSourceEntryHash = hash p.p.buildState.NumResourcesAdded++ @@ -143,6 +151,11 @@ func (p *pagesFromDataTemplateContext) EnableAllLanguages() string { return "" } +func (p *pagesFromDataTemplateContext) EnableAllDimensions() string { + p.p.buildState.EnableAllDimensions = true + return "" +} + func NewPagesFromTemplate(opts PagesFromTemplateOptions) *PagesFromTemplate { return &PagesFromTemplate{ PagesFromTemplateOptions: opts, @@ -162,7 +175,7 @@ type PagesFromTemplateOptions struct { Watching bool - HandlePage func(pt *PagesFromTemplate, p *pagemeta.PageConfig) error + HandlePage func(pt *PagesFromTemplate, p *pagemeta.PageConfigEarly) error HandleResource func(pt *PagesFromTemplate, p *pagemeta.ResourceConfig) error GoTmplFi hugofs.FileMetaInfo @@ -194,21 +207,23 @@ func (b *PagesFromTemplate) StaleVersion() uint32 { } type BuildInfo struct { - NumPagesAdded uint64 - NumResourcesAdded uint64 - EnableAllLanguages bool - ChangedIdentities []identity.Identity - DeletedPaths []string - Path *paths.Path + NumPagesAdded uint64 + NumResourcesAdded uint64 + EnableAllLanguages bool + EnableAllDimensions bool + ChangedIdentities []identity.Identity + DeletedPaths []PathHashes + Path *paths.Path } type BuildState struct { StaleVersion uint32 - EnableAllLanguages bool + EnableAllLanguages bool + EnableAllDimensions bool - // Paths deleted in the current build. - DeletedPaths []string + // PathHashes deleted in the current build. + DeletedPaths []PathHashes // Changed identities in the current build. ChangedIdentities []identity.Identity @@ -224,20 +239,40 @@ func (b *BuildState) hash(v any) uint64 { return hashing.HashUint64(v) } -func (b *BuildState) checkHasChangedAndSetSourceInfo(changedPath string, v any) bool { - h := b.hash(v) - si, found := b.sourceInfosPrevious.Get(changedPath) - if found { - b.sourceInfosCurrent.Set(changedPath, si) - if si.hash == h { - return false +type sourceInfo struct { + siteHashes map[uint64]uint64 +} + +func (b *BuildState) checkHasChangedAndSetSourceInfo(changedPath string, sites map[string]any, v any) (uint64, bool) { + hv := b.hash(v) + hsites := b.hash(sites) + + si, _ := b.sourceInfosCurrent.GetOrCreate(changedPath, func() (*sourceInfo, error) { + return &sourceInfo{ + siteHashes: make(map[uint64]uint64), + }, nil + }) + + if h, found := si.siteHashes[hsites]; found && h == hv { + return hv, false + } + + if psi, found := b.sourceInfosPrevious.Get(changedPath); found { + if h, found := psi.siteHashes[hsites]; found && h == hv { + // Not changed. + si.siteHashes[hsites] = hv + return hv, false } - } else { - si = &sourceInfo{} - b.sourceInfosCurrent.Set(changedPath, si) } - si.hash = h - return true + + // It has changed. + si.siteHashes[hsites] = hv + return hv, true +} + +type PathHashes struct { + Path string + Hashes map[uint64]struct{} } func (b *BuildState) resolveDeletedPaths() { @@ -245,15 +280,26 @@ func (b *BuildState) resolveDeletedPaths() { b.DeletedPaths = nil return } - var paths []string - b.sourceInfosPrevious.ForEeach(func(k string, _ *sourceInfo) bool { - if _, found := b.sourceInfosCurrent.Get(k); !found { - paths = append(paths, k) + var pathsHashes []PathHashes + b.sourceInfosPrevious.ForEeach(func(k string, pv *sourceInfo) bool { + if cv, found := b.sourceInfosCurrent.Get(k); !found { + pathsHashes = append(pathsHashes, PathHashes{Path: k, Hashes: map[uint64]struct{}{}}) + } else { + deleted := map[uint64]struct{}{} + for k, ph := range pv.siteHashes { + ch, found := cv.siteHashes[k] + if !found || ch != ph { + deleted[ph] = struct{}{} + } + } + if len(deleted) > 0 { + pathsHashes = append(pathsHashes, PathHashes{Path: k, Hashes: deleted}) + } } return true }) - b.DeletedPaths = paths + b.DeletedPaths = pathsHashes } func (b *BuildState) PrepareNextBuild() { @@ -266,10 +312,6 @@ func (b *BuildState) PrepareNextBuild() { b.NumResourcesAdded = 0 } -type sourceInfo struct { - hash uint64 -} - func (p PagesFromTemplate) CloneForSite(s page.Site) *PagesFromTemplate { // We deliberately make them share the same DependencyManager and Store. p.PagesFromTemplateOptions.Site = s @@ -324,12 +366,13 @@ func (p *PagesFromTemplate) Execute(ctx context.Context) (BuildInfo, error) { } bi := BuildInfo{ - NumPagesAdded: p.buildState.NumPagesAdded, - NumResourcesAdded: p.buildState.NumResourcesAdded, - EnableAllLanguages: p.buildState.EnableAllLanguages, - ChangedIdentities: p.buildState.ChangedIdentities, - DeletedPaths: p.buildState.DeletedPaths, - Path: p.GoTmplFi.Meta().PathInfo, + NumPagesAdded: p.buildState.NumPagesAdded, + NumResourcesAdded: p.buildState.NumResourcesAdded, + EnableAllLanguages: p.buildState.EnableAllLanguages, + EnableAllDimensions: p.buildState.EnableAllDimensions, + ChangedIdentities: p.buildState.ChangedIdentities, + DeletedPaths: p.buildState.DeletedPaths, + Path: p.GoTmplFi.Meta().PathInfo, } return bi, nil diff --git a/hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go b/hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go index ae8b072c8..556fb84a9 100644 --- a/hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go +++ b/hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go @@ -198,14 +198,6 @@ baseURL = "https://example.com" b.Assert(err.Error(), qt.Contains, "error calling AddPage: empty path is reserved for the home page") }) - t.Run("AddPage, lang set", func(t *testing.T) { - files := strings.ReplaceAll(filesTemplate, "DICT", `(dict "kind" "page" "path" "p1" "lang" "en")`) - b, err := hugolib.TestE(t, files) - b.Assert(err, qt.IsNotNil) - b.Assert(err.Error(), qt.Contains, "_content.gotmpl:1:4") - b.Assert(err.Error(), qt.Contains, "error calling AddPage: lang must not be set") - }) - t.Run("Site methods not ready", func(t *testing.T) { filesTemplate := ` -- hugo.toml -- @@ -591,73 +583,6 @@ disableKinds = ['home','section','rss','sitemap','taxonomy','term'] ) } -func TestPagesFromGoTmplPathWarningsPathPage(t *testing.T) { - t.Parallel() - - files := ` --- hugo.toml -- -baseURL = "https://example.com" -disableKinds = ['home','section','rss','sitemap','taxonomy','term'] -printPathWarnings = true --- content/_content.gotmpl -- -{{ .AddPage (dict "path" "p1" "title" "p1" ) }} -{{ .AddPage (dict "path" "p2" "title" "p2" ) }} --- content/p1.md -- ---- -title: "p1" ---- --- layouts/_default/single.html -- -{{ .Title }}| -` - - b := hugolib.Test(t, files, hugolib.TestOptWarn()) - - b.AssertFileContent("public/p1/index.html", "p1|") - - b.AssertLogContains("Duplicate content path") - - files = strings.ReplaceAll(files, `"path" "p1"`, `"path" "p1new"`) - - b = hugolib.Test(t, files, hugolib.TestOptWarn()) - - b.AssertLogContains("! WARN") -} - -func TestPagesFromGoTmplPathWarningsPathResource(t *testing.T) { - t.Parallel() - - files := ` --- hugo.toml -- -baseURL = "https://example.com" -disableKinds = ['home','section','rss','sitemap','taxonomy','term'] -printPathWarnings = true --- content/_content.gotmpl -- -{{ .AddResource (dict "path" "p1/data1.yaml" "content" (dict "value" "data1" ) ) }} -{{ .AddResource (dict "path" "p1/data2.yaml" "content" (dict "value" "data2" ) ) }} - --- content/p1/index.md -- ---- -title: "p1" ---- --- content/p1/data1.yaml -- -value: data1 --- layouts/_default/single.html -- -{{ .Title }}| -` - - b := hugolib.Test(t, files, hugolib.TestOptWarn()) - - b.AssertFileContent("public/p1/index.html", "p1|") - - b.AssertLogContains("Duplicate resource path") - - files = strings.ReplaceAll(files, `"path" "p1/data1.yaml"`, `"path" "p1/data1new.yaml"`) - - b = hugolib.Test(t, files, hugolib.TestOptWarn()) - - b.AssertLogContains("! WARN") -} - func TestPagesFromGoTmplShortcodeNoPreceddingCharacterIssue12544(t *testing.T) { t.Parallel() diff --git a/hugolib/paginator_test.go b/hugolib/paginator_test.go index 2fb87956f..2a5da05fa 100644 --- a/hugolib/paginator_test.go +++ b/hugolib/paginator_test.go @@ -178,7 +178,7 @@ Paginator: {{ .Paginator }} ` b, err := TestE(t, files) b.Assert(err, qt.IsNotNil) - b.Assert(err.Error(), qt.Contains, `error calling Paginator: pagination not supported for this page: kind: "page"`) + b.Assert(err.Error(), qt.Contains, `error calling Paginator: pagination not supported for this page`) } func TestNilPointerErrorMessage(t *testing.T) { diff --git a/hugolib/params_test.go b/hugolib/params_test.go index 385b39a6f..2fa4f7752 100644 --- a/hugolib/params_test.go +++ b/hugolib/params_test.go @@ -57,33 +57,6 @@ Summary: {{ .Summary }}| ) } -func TestFrontMatterParamsLangNoCascade(t *testing.T) { - t.Parallel() - - files := ` --- hugo.toml -- -baseURL = "https://example.org/" -disableKinds = ["taxonomy", "term"] -defaultContentLanguage = "en" -defaultContentLanguageInSubdir = true -[languages] -[languages.en] -weight = 1 -[languages.nn] -weight = 2 --- content/_index.md -- -+++ -[[cascade]] -background = 'yosemite.jpg' -lang = 'nn' -+++ - -` - - b, err := TestE(t, files) - b.Assert(err, qt.IsNotNil) -} - // Issue 11970. func TestFrontMatterBuildIsHugoKeyword(t *testing.T) { t.Parallel() diff --git a/hugolib/paths/paths.go b/hugolib/paths/paths.go index 60ec873f9..e761e1246 100644 --- a/hugolib/paths/paths.go +++ b/hugolib/paths/paths.go @@ -18,6 +18,7 @@ import ( "strings" hpaths "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/modules" @@ -65,8 +66,8 @@ func New(fs *hugofs.Fs, cfg config.AllProvider) (*Paths, error) { } var multihostTargetBasePaths []string - if cfg.IsMultihost() && len(cfg.Languages()) > 1 { - for _, l := range cfg.Languages() { + if cfg.IsMultihost() && len(cfg.Languages().(langs.Languages)) > 1 { + for _, l := range cfg.Languages().(langs.Languages) { multihostTargetBasePaths = append(multihostTargetBasePaths, hpaths.ToSlashPreserveLeading(l.Lang)) } } @@ -100,7 +101,7 @@ func (p *Paths) Lang() string { if p == nil || p.Cfg.Language() == nil { return "" } - return p.Cfg.Language().Lang + return p.Cfg.Language().(*langs.Language).Lang } func (p *Paths) GetTargetLanguageBasePath() string { diff --git a/hugolib/rebuild_test.go b/hugolib/rebuild_test.go index 38752b90e..657ed5609 100644 --- a/hugolib/rebuild_test.go +++ b/hugolib/rebuild_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/bep/logg" "github.com/fortytw2/leaktest" qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/common/types" @@ -744,6 +745,103 @@ Prev: {{ with .NextInSection }}{{ .Title }}{{ end }}| b.AssertFileContent("public/s/p6/index.html", "Next: P7 Edited|") } +const rebuildBasicFiles = ` +-- hugo.toml -- +baseURL = "https://example.com/" +disableLiveReload = true +disableKinds = ["sitemap", "robotstxt", "404", "rss"] +-- content/_index.md -- +--- +title: "Home" +cascade: + target: + kind: "page" + date: 2019-05-06 + params: + myparam: "myvalue" +--- +-- content/mysection/_index.md -- +--- +title: "My Section" +date: 2024-02-01 +--- +-- content/mysection/p1.md -- +--- +title: "P1" +date: 2020-01-01 +--- +-- content/mysection/p2.md -- +--- +title: "P2" +date: 2021-02-01 +--- +-- content/mysection/p3.md -- +--- +title: "P3" +--- +-- layouts/all.html -- +all. {{ .Title }}|Param: {{ .Params.myparam }}|Lastmod: {{ .Lastmod.Format "2006-01-02" }}| +` + +func TestRebuildEditPageTitle(t *testing.T) { + t.Parallel() + b := TestRunning(t, rebuildBasicFiles) + b.AssertFileContent("public/index.html", "all. Home|Param: |Lastmod: 2024-02-01|") + b.AssertFileContent("public/mysection/p1/index.html", "P1|Param: myvalue|", "Lastmod: 2020-01-01|") + b.EditFileReplaceAll("content/mysection/p1.md", "P1", "P1edit").Build() + b.AssertRenderCountPage(2) + b.AssertFileContent("public/mysection/p1/index.html", "all. P1edit|Param: myvalue|") +} + +func TestRebuildEditPageDate(t *testing.T) { + t.Parallel() + b := TestRunning(t, rebuildBasicFiles) + b.AssertFileContent("public/index.html", "all. Home|Param: |Lastmod: 2024-02-01|") + b.EditFileReplaceAll("content/mysection/p2.md", "2021-02-01", "2025-04-02").Build() + b.AssertRenderCountPage(2) + b.AssertFileContent("public/index.html", "all. Home|Param: |Lastmod: 2025-04-02|") + b.AssertFileContent("public/mysection/p2/index.html", "all. P2|Param: myvalue|Lastmod: 2025-04-02|") +} + +func TestRebuildEditHomeCascadeEditParam(t *testing.T) { + t.Parallel() + b := TestRunning(t, rebuildBasicFiles) + b.AssertRenderCountPage(7) + b.EditFileReplaceAll("content/_index.md", "myvalue", "myvalue-edit").Build() + b.AssertRenderCountPage(7) + b.AssertFileContent("public/mysection/p1/index.html", "all. P1|Param: myvalue-edit|") +} + +func TestRebuildEditHomeCascadeRemoveParam(t *testing.T) { + t.Parallel() + b := TestRunning(t, rebuildBasicFiles) + b.AssertRenderCountPage(7) + b.EditFileReplaceAll("content/_index.md", `myparam: "myvalue"`, "").Build() + b.AssertRenderCountPage(7) + b.AssertFileContent("public/mysection/p1/index.html", "all. P1|Param: |") +} + +func TestRebuildEditHomeCascadeEditDate(t *testing.T) { + t.Parallel() + b := TestRunning(t, rebuildBasicFiles) + b.AssertRenderCountPage(7) + b.AssertFileContent("public/mysection/p3/index.html", "all. P3|Param: myvalue|Lastmod: 2019-05-06") + b.EditFileReplaceAll("content/_index.md", "2019-05-06", "2025-05-06").Build() + b.AssertRenderCountPage(7) + b.AssertFileContent("public/index.html", "all. Home|Param: |Lastmod: 2025-05-06|") + b.AssertFileContent("public/mysection/index.html", "all. My Section|Param: |Lastmod: 2024-02-01|") + b.AssertFileContent("public/mysection/p3/index.html", "all. P3|Param: myvalue|Lastmod: 2025-05-06") +} + +func TestRebuildEditSectionRemoveDate(t *testing.T) { + t.Parallel() + b := TestRunning(t, rebuildBasicFiles) + b.AssertFileContent("public/mysection/index.html", "all. My Section|Param: |Lastmod: 2024-02-01|") + b.EditFileReplaceAll("content/mysection/_index.md", "date: 2024-02-01", "").Build() + b.AssertRenderCountPage(5) + b.AssertFileContent("public/mysection/index.html", "all. My Section|Param: |Lastmod: 2021-02-01|") +} + func TestRebuildVariations(t *testing.T) { // t.Parallel() not supported, see https://github.com/fortytw2/leaktest/issues/4 // This leaktest seems to be a little bit shaky on Travis. @@ -828,6 +926,7 @@ Len RegularPagesRecursive: {{ len .RegularPagesRecursive }} Site.Lastmod: {{ .Site.Lastmod.Format "2006-01-02" }}| Paginate: {{ range (.Paginate .Site.RegularPages).Pages }}{{ .RelPermalink }}|{{ .Title }}|{{ end }}$ -- layouts/_default/single.html -- +Single: .Site: {{ .Site }} Single: {{ .Title }}|{{ .Content }}| Single Partial Cached: {{ partialCached "pcached" . }}| Page.Lastmod: {{ .Lastmod.Format "2006-01-02" }}| @@ -846,6 +945,7 @@ Partial P1. Partial Pcached. -- layouts/shortcodes/include.html -- {{ $p := site.GetPage (.Get 0)}} +.Page.Site: {{ .Page.Site }} :: site: {{ site }} {{ with $p }} Shortcode Include: {{ .Title }}| {{ end }} @@ -857,8 +957,6 @@ Shortcode Partial P1: {{ partial "p1" . }}| Codeblock Include: {{ .Title }}| {{ end }} - - ` b := NewIntegrationTestBuilder( @@ -866,11 +964,11 @@ Codeblock Include: {{ .Title }}| T: t, TxtarString: files, Running: true, + Verbose: false, BuildCfg: BuildCfg{ testCounters: &buildCounters{}, }, - // Verbose: true, - // LogLevel: logg.LevelTrace, + LogLevel: logg.LevelWarn, }, ).Build() @@ -1545,17 +1643,17 @@ title: "B nn" B nn. -- content/p1/f1.nn.txt -- F1 nn --- layouts/_default/single.html -- -Single: {{ .Title }}|{{ .Content }}|Bundled File: {{ with .Resources.GetMatch "f1.*" }}{{ .Content }}{{ end }}|Bundled Page: {{ with .Resources.GetMatch "b.*" }}{{ .Content }}{{ end }}| +-- layouts/_default/all.html -- +All: {{ .Title }}|{{ .Kind }}|{{ .Content }}|Bundled File: {{ with .Resources.GetMatch "f1.*" }}{{ .Content }}{{ end }}|Bundled Page: {{ with .Resources.GetMatch "b.*" }}{{ .Content }}{{ end }}| ` b := TestRunning(t, files) - b.AssertFileContent("public/nn/p1/index.html", "Single: P1 nn|

P1 nn.

", "F1 nn|") + b.AssertFileContent("public/nn/p1/index.html", "All: P1 nn|page|

P1 nn.

", "F1 nn|") b.EditFileReplaceAll("content/p1/index.nn.md", "P1 nn.", "P1 nn edit.").Build() - b.AssertFileContent("public/nn/p1/index.html", "Single: P1 nn|

P1 nn edit.

\n|") + b.AssertFileContent("public/nn/p1/index.html", "All: P1 nn|page|

P1 nn edit.

") b.EditFileReplaceAll("content/p1/f1.nn.txt", "F1 nn", "F1 nn edit.").Build() - b.AssertFileContent("public/nn/p1/index.html", "Bundled File: F1 nn edit.") + b.AssertFileContent("public/nn/p1/index.html", "All: P1 nn|page|

P1 nn edit.

\n|Bundled File: F1 nn edit.|") b.EditFileReplaceAll("content/p1/b.nn.md", "B nn.", "B nn edit.").Build() b.AssertFileContent("public/nn/p1/index.html", "B nn edit.") } @@ -1590,7 +1688,6 @@ Single: {{ .Title }}|{{ .Content }}| ` b := TestRunning(t, files) - b.AssertFileContent("public/nn/p1nn/index.html", "Single: P1 nn|

P1 nn.

") b.EditFileReplaceAll("content/nn/p1nn/index.md", "P1 nn.", "P1 nn edit.").Build() b.AssertFileContent("public/nn/p1nn/index.html", "Single: P1 nn|

P1 nn edit.

\n|") @@ -1946,7 +2043,7 @@ func TestRebuildEditTagIssue13648(t *testing.T) { baseURL = "https://example.com" disableLiveReload = true -- layouts/all.html -- -All. {{ range .Pages }}{{ .Title }}|{{ end }} +All. {{ range .Pages }}{{ .Title }}|{{ end }}$ -- content/p1.md -- --- title: "P1" @@ -1959,10 +2056,7 @@ tags: ["tag1"] b.AssertFileContent("public/tags/index.html", "All. Tag1|") b.EditFileReplaceAll("content/p1.md", "tag1", "tag2").Build() - // Note that the below is still not correct, as this is effectively a rename, and - // Tag2 should be removed from the list. - // But that is a harder problem to tackle. - b.AssertFileContent("public/tags/index.html", "All. Tag1|Tag2|") + b.AssertFileContent("public/tags/index.html", "All. Tag1|$") } func TestRebuildEditNonReferencedResourceIssue13748(t *testing.T) { diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 00e4c0060..36e50ae3b 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -49,8 +49,10 @@ func TestResourceChainBasic(t *testing.T) { ts.Close() }) - b := newTestSitesBuilder(t) - b.WithTemplatesAdded("index.html", fmt.Sprintf(` + for i := range 2 { + + b := newTestSitesBuilder(t) + b.WithTemplatesAdded("index.html", fmt.Sprintf(` {{ $hello := "

Hello World!

" | resources.FromString "hello.html" | fingerprint "sha512" | minify | fingerprint }} {{ $cssFingerprinted1 := "body { background-color: lightblue; }" | resources.FromString "styles.css" | minify | fingerprint }} {{ $cssFingerprinted2 := "body { background-color: orange; }" | resources.FromString "styles2.css" | minify | fingerprint }} @@ -85,21 +87,20 @@ PRINT PROTOCOL ERROR DETAILS: {{ with $gopherprotocol }}{{ with .Err }}Err: {{ . FAILED REMOTE ERROR DETAILS CONTENT: {{ with $failedImg }}{{ with .Err }}{{ with .Cause }}{{ . }}|{{ with .Data }}Body: {{ .Body }}|StatusCode: {{ .StatusCode }}|ContentLength: {{ .ContentLength }}|ContentType: {{ .ContentType }}{{ end }}{{ end }}{{ end }}{{ end }}| `, ts.URL)) - fs := b.Fs.Source + fs := b.Fs.Source - imageDir := filepath.Join("assets", "images") - b.Assert(os.MkdirAll(imageDir, 0o777), qt.IsNil) - src, err := os.Open("testdata/sunset.jpg") - b.Assert(err, qt.IsNil) - out, err := fs.Create(filepath.Join(imageDir, "sunset.jpg")) - b.Assert(err, qt.IsNil) - _, err = io.Copy(out, src) - b.Assert(err, qt.IsNil) - out.Close() + imageDir := filepath.Join("assets", "images") + b.Assert(os.MkdirAll(imageDir, 0o777), qt.IsNil) + src, err := os.Open("testdata/sunset.jpg") + b.Assert(err, qt.IsNil) + out, err := fs.Create(filepath.Join(imageDir, "sunset.jpg")) + b.Assert(err, qt.IsNil) + _, err = io.Copy(out, src) + b.Assert(err, qt.IsNil) + out.Close() - b.Running() + b.Running() - for i := range 2 { b.Logf("Test run %d", i) b.Build(BuildCfg{}) diff --git a/hugolib/roles/roles.go b/hugolib/roles/roles.go new file mode 100644 index 000000000..f2cf3d633 --- /dev/null +++ b/hugolib/roles/roles.go @@ -0,0 +1,213 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package roles + +import ( + "errors" + "fmt" + "iter" + "sort" + + "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/common/predicate" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" + "github.com/mitchellh/mapstructure" +) + +var _ sitesmatrix.DimensionInfo = (*roleWrapper)(nil) + +type RoleConfig struct { + // The weight of the role. + // Used to determine the order of the roles. + // If zero, we use the role name. + Weight int +} + +type Role interface { + Name() string +} + +type Roles []Role + +var _ Role = (*roleWrapper)(nil) + +func NewRole(r RoleInternal) Role { + return roleWrapper{r: r} +} + +type roleWrapper struct { + r RoleInternal +} + +func (r roleWrapper) Name() string { + return r.r.Name +} + +func (r roleWrapper) IsDefault() bool { + return r.r.Default +} + +type RoleInternal struct { + // Name is the name of the role, extracted from the key in the config. + Name string + + // Whether this role is the default role. + // This will be rendered in the root. + // There is only be one default role. + Default bool + + RoleConfig +} + +type RolesInternal struct { + roleConfigs map[string]RoleConfig + Sorted []RoleInternal +} + +func (r RolesInternal) IndexDefault() int { + for i, role := range r.Sorted { + if role.Default { + return i + } + } + panic("no default role found") +} + +func (r RolesInternal) ResolveName(i int) string { + if i < 0 || i >= len(r.Sorted) { + panic(fmt.Sprintf("index %d out of range for roles", i)) + } + return r.Sorted[i].Name +} + +func (r RolesInternal) ResolveIndex(name string) int { + for i, role := range r.Sorted { + if role.Name == name { + return i + } + } + panic(fmt.Sprintf("no role found for name %q", name)) +} + +// IndexMatch returns an iterator for the roles that match the filter. +func (r RolesInternal) IndexMatch(match predicate.P[string]) (iter.Seq[int], error) { + return func(yield func(i int) bool) { + for i, role := range r.Sorted { + if match(role.Name) { + if !yield(i) { + return + } + } + } + }, nil +} + +// ForEachIndex returns an iterator for the indices of the roles. +func (r RolesInternal) ForEachIndex() iter.Seq[int] { + return func(yield func(i int) bool) { + for i := range r.Sorted { + if !yield(i) { + return + } + } + } +} + +const defaultContentRoleFallback = "guest" + +func (r *RolesInternal) init(defaultContentRole string) (string, error) { + if r.roleConfigs == nil { + r.roleConfigs = make(map[string]RoleConfig) + } + defaultContentRoleProvided := defaultContentRole != "" + if len(r.roleConfigs) == 0 { + // Add a default role. + if defaultContentRole == "" { + defaultContentRole = defaultContentRoleFallback + } + r.roleConfigs[defaultContentRole] = RoleConfig{} + } + + var defaultSeen bool + for k, v := range r.roleConfigs { + if k == "" { + return "", errors.New("role name cannot be empty") + } + + if err := paths.ValidateIdentifier(k); err != nil { + return "", fmt.Errorf("role name %q is invalid: %s", k, err) + } + + var isDefault bool + if k == defaultContentRole { + isDefault = true + defaultSeen = true + } + + r.Sorted = append(r.Sorted, RoleInternal{Name: k, Default: isDefault, RoleConfig: v}) + } + + // Sort by weight if set, then by name. + sort.SliceStable(r.Sorted, func(i, j int) bool { + ri, rj := r.Sorted[i], r.Sorted[j] + if ri.Weight == rj.Weight { + return ri.Name < rj.Name + } + if rj.Weight == 0 { + return true + } + if ri.Weight == 0 { + return false + } + return ri.Weight < rj.Weight + }) + + if !defaultSeen { + if defaultContentRoleProvided { + return "", fmt.Errorf("the configured defaultContentRole %q does not exist", defaultContentRole) + } + // If no default role is set, we set the first one. + first := r.Sorted[0] + first.Default = true + r.roleConfigs[first.Name] = first.RoleConfig + r.Sorted[0] = first + defaultContentRole = first.Name + } + + return defaultContentRole, nil +} + +func (r RolesInternal) Has(role string) bool { + _, found := r.roleConfigs[role] + return found +} + +func DecodeConfig(defaultContentRole string, m map[string]any) (*config.ConfigNamespace[map[string]RoleConfig, RolesInternal], string, error) { + v, err := config.DecodeNamespace[map[string]RoleConfig](m, func(in any) (RolesInternal, any, error) { + var roles RolesInternal + var conf map[string]RoleConfig + if err := mapstructure.Decode(m, &conf); err != nil { + return roles, nil, err + } + roles.roleConfigs = conf + var err error + if defaultContentRole, err = roles.init(defaultContentRole); err != nil { + return roles, nil, err + } + return roles, nil, nil + }) + + return v, defaultContentRole, err +} diff --git a/hugolib/roles/roles_integration_test.go b/hugolib/roles/roles_integration_test.go new file mode 100644 index 000000000..7924e592c --- /dev/null +++ b/hugolib/roles/roles_integration_test.go @@ -0,0 +1,124 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package roles_test + +import ( + "testing" + + qt "github.com/frankban/quicktest" + + "github.com/gohugoio/hugo/hugolib" +) + +func TestRolesAndVersions(t *testing.T) { + t.Parallel() + files := ` +-- hugo.toml -- +baseURL = "https://example.org/" +defaultContentVersion = "v4.0.0" +defaultContentVersionInSubdir = true +defaultContentRoleInSubdir = true +defaultContentRole = "guest" +defaultContentLanguage = "en" +defaultContentLanguageInSubdir = true +disableKinds = ["taxonomy", "term", "rss", "sitemap"] + +[cascade.sites.matrix] +versions = ["v2**"] + +[languages] +[languages.en] +weight = 1 +[languages.nn] +weight = 2 +[roles] +[roles.guest] +weight = 300 +[roles.member] +weight = 200 +[versions] +[versions."v2.0.0"] +[versions."v1.2.3"] +[versions."v2.1.0"] +[versions."v3.0.0"] +[versions."v4.0.0"] +-- content/memberonlypost.md -- +--- +title: "Member Only" +sites: + matrix: + languages: "**" + roles: member + versions: "v4.0.0" +--- +Member content. +-- content/publicpost.md -- +--- +title: "Public" +sites: + matrix: + versions: ["v1.2.3", "v2.**", "! v2.1.*"] + complements: + versions: "v3**" +--- +Users with guest role will see this. +-- content/v3publicpost.md -- +--- +title: "Public v3" +sites: + matrix: + versions: "v3**" +--- +Users with guest role will see this. +-- layouts/all.html -- +Role: {{ .Site.Role.Name }}|Version: {{ .Site.Version.Name }}|Lang: {{ .Site.Language.Lang }}| +RegularPages: {{ range .RegularPages }}{{ .RelPermalink }} r: {{ .Site.Language.Name }} v: {{ .Site.Version.Name }} l: {{ .Site.Role.Name }}|{{ end }}$ + +` + + for range 3 { + b := hugolib.Test(t, files) + + b.AssertPublishDir( + "guest/v1.2.3/en/publicpost", "guest/v2.0.0/en/publicpost", "! guest/v2.1.0/en/publicpost", + "member/v4.0.0/en/memberonlypost", "member/v4.0.0/nn/memberonlypost", + ) + + b.AssertFileContent("public/guest/v2.0.0/en/index.html", + "Role: guest|Version: v2.0.0|", + ) + + b.AssertFileContent("public/guest/v3.0.0/en/index.html", "egularPages: /guest/v2.0.0/en/publicpost/ r: en v: v2.0.0 l: guest|/guest/v3.0.0/en/v3publicpost/ r: en v: v3.0.0 l: guest|$") + + } +} + +func TestDefaultContentRoleDoesNotExist(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ["rss", "sitemap", "section", "taxonomy", "term"] +defaultContentLanguage = "en" +defaultContentLanguageInSubDir = true +defaultContentRole = "doesnotexist" +[roles] +[roles.guest] +weight = 300 +[roles.member] +weight = 200 +` + b, err := hugolib.TestE(t, files) + b.Assert(err, qt.ErrorMatches, `.*failed to decode "roles": the configured defaultContentRole "doesnotexist" does not exist`) +} diff --git a/hugolib/segments/segments.go b/hugolib/segments/segments.go index 941c4ea5c..273811a53 100644 --- a/hugolib/segments/segments.go +++ b/hugolib/segments/segments.go @@ -15,175 +15,162 @@ package segments import ( "fmt" + "strings" "github.com/gobwas/glob" + "github.com/gohugoio/hugo/common/hugo" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/predicate" "github.com/gohugoio/hugo/config" - hglob "github.com/gohugoio/hugo/hugofs/glob" + hglob "github.com/gohugoio/hugo/hugofs/hglob" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "github.com/mitchellh/mapstructure" ) // Segments is a collection of named segments. type Segments struct { - s map[string]excludeInclude -} - -type excludeInclude struct { - exclude predicate.P[SegmentMatcherFields] - include predicate.P[SegmentMatcherFields] -} - -// ShouldExcludeCoarse returns whether the given fields should be excluded. -// This is used for the coarser grained checks, e.g. language and output format. -// Note that ShouldExcludeCoarse(fields) == ShouldExcludeFine(fields) may -// not always be true, but ShouldExcludeCoarse(fields) == true == ShouldExcludeFine(fields) -// will always be truthful. -func (e excludeInclude) ShouldExcludeCoarse(fields SegmentMatcherFields) bool { - return e.exclude != nil && e.exclude(fields) -} + builder *segmentsBuilder -// ShouldExcludeFine returns whether the given fields should be excluded. -// This is used for the finer grained checks, e.g. on individual pages. -func (e excludeInclude) ShouldExcludeFine(fields SegmentMatcherFields) bool { - if e.exclude != nil && e.exclude(fields) { - return true - } - return e.include != nil && !e.include(fields) + // SegmentFilter is the compiled filter for all segments to render. + SegmentFilter SegmentFilter } type SegmentFilter interface { // ShouldExcludeCoarse returns whether the given fields should be excluded on a coarse level. - ShouldExcludeCoarse(SegmentMatcherFields) bool + ShouldExcludeCoarse(SegmentQuery) bool // ShouldExcludeFine returns whether the given fields should be excluded on a fine level. - ShouldExcludeFine(SegmentMatcherFields) bool + ShouldExcludeFine(SegmentQuery) bool } type segmentFilter struct { - coarse predicate.P[SegmentMatcherFields] - fine predicate.P[SegmentMatcherFields] + exclude predicate.PR[SegmentQuery] + include predicate.PR[SegmentQuery] } -func (f segmentFilter) ShouldExcludeCoarse(field SegmentMatcherFields) bool { - return f.coarse(field) +func (f segmentFilter) ShouldExcludeCoarse(q SegmentQuery) bool { + return f.exclude(q).OK() } -func (f segmentFilter) ShouldExcludeFine(fields SegmentMatcherFields) bool { - return f.fine(fields) +func (f segmentFilter) ShouldExcludeFine(q SegmentQuery) bool { + return f.exclude(q).OK() || !f.include(q).OK() } -var ( - matchAll = func(SegmentMatcherFields) bool { return true } - matchNothing = func(SegmentMatcherFields) bool { return false } -) - -// Get returns a SegmentFilter for the given segments. -func (sms Segments) Get(onNotFound func(s string), ss ...string) SegmentFilter { - if ss == nil { - return segmentFilter{coarse: matchNothing, fine: matchNothing} - } - var sf segmentFilter - for _, s := range ss { - if seg, ok := sms.s[s]; ok { - if sf.coarse == nil { - sf.coarse = seg.ShouldExcludeCoarse - } else { - sf.coarse = sf.coarse.Or(seg.ShouldExcludeCoarse) - } - if sf.fine == nil { - sf.fine = seg.ShouldExcludeFine - } else { - sf.fine = sf.fine.Or(seg.ShouldExcludeFine) - } - } else if onNotFound != nil { - onNotFound(s) - } - } - - if sf.coarse == nil { - sf.coarse = matchAll - } - if sf.fine == nil { - sf.fine = matchAll - } - - return sf -} - -type SegmentConfig struct { - Excludes []SegmentMatcherFields - Includes []SegmentMatcherFields -} - -// SegmentMatcherFields is a matcher for a segment include or exclude. -// All of these are Glob patterns. -type SegmentMatcherFields struct { - Kind string - Path string - Lang string - Output string +type segmentsBuilder struct { + logger loggers.Logger + isConfigInit bool + configuredDimensions *sitesmatrix.ConfiguredDimensions + segmentCfg map[string]SegmentConfig + segmentsToRender []string } -func getGlob(s string) (glob.Glob, error) { - if s == "" { - return nil, nil - } - g, err := hglob.GetGlob(s) +func (b *Segments) compile() error { + filter, err := b.builder.build() if err != nil { - return nil, fmt.Errorf("failed to compile Glob %q: %w", s, err) + return err } - return g, nil + b.SegmentFilter = filter + b.builder = nil + return nil } -func compileSegments(f []SegmentMatcherFields) (predicate.P[SegmentMatcherFields], error) { +func (s *segmentsBuilder) buildOne(f []SegmentMatcherFields) (predicate.PR[SegmentQuery], error) { if f == nil { - return func(SegmentMatcherFields) bool { return false }, nil + return matchNothing, nil } var ( - result predicate.P[SegmentMatcherFields] - section predicate.P[SegmentMatcherFields] + result predicate.PR[SegmentQuery] + section predicate.PR[SegmentQuery] ) - addToSection := func(matcherFields SegmentMatcherFields, f func(fields SegmentMatcherFields) string) error { - s1 := f(matcherFields) - g, err := getGlob(s1) - if err != nil { - return err - } - matcher := func(fields SegmentMatcherFields) bool { - s2 := f(fields) - if s2 == "" { - return false - } - return g.Match(s2) - } + addSectionMatcher := func(matcher predicate.PR[SegmentQuery]) { if section == nil { section = matcher } else { section = section.And(matcher) } + } + + addToSection := func(matcherFields SegmentMatcherFields, f1 func(fields SegmentMatcherFields) []string, f2 func(q SegmentQuery) string) error { + s1 := f1(matcherFields) + if s1 == nil { + // Nothing to match against. + return nil + } + var sliceMatcher predicate.PR[SegmentQuery] + + for _, s := range s1 { + negate := strings.HasPrefix(s, hglob.NegationPrefix) + if negate { + s = strings.TrimPrefix(s, hglob.NegationPrefix) + } + + g, err := getGlob(s) + if err != nil { + return err + } + + m := func(fields SegmentQuery) predicate.Match { + s2 := f2(fields) + if s2 == "" { + return predicate.False + } + return predicate.BoolMatch(g.Match(s2) != negate) + } + + if negate { + sliceMatcher = sliceMatcher.And(m) + } else { + sliceMatcher = sliceMatcher.Or(m) + } + } + + if sliceMatcher != nil { + addSectionMatcher(sliceMatcher) + } + return nil } for _, fields := range f { - if fields.Kind != "" { - if err := addToSection(fields, func(fields SegmentMatcherFields) string { return fields.Kind }); err != nil { + if len(fields.Kind) > 0 { + if err := addToSection(fields, + func(fields SegmentMatcherFields) []string { return fields.Kind }, + func(fields SegmentQuery) string { return fields.Kind }, + ); err != nil { return result, err } } - if fields.Path != "" { - if err := addToSection(fields, func(fields SegmentMatcherFields) string { return fields.Path }); err != nil { + if len(fields.Path) > 0 { + if err := addToSection(fields, + func(fields SegmentMatcherFields) []string { return fields.Path }, + func(fields SegmentQuery) string { return fields.Path }, + ); err != nil { return result, err } } if fields.Lang != "" { - if err := addToSection(fields, func(fields SegmentMatcherFields) string { return fields.Lang }); err != nil { - return result, err + hugo.DeprecateWithLogger("config segments.[...]lang ", "Use sites.matrix instead, see https://gohugo.io/configuration/segments/#segment-definition", "v0.153.0", s.logger.Logger()) + fields.Sites.Matrix.Languages = []string{fields.Lang} + } + if !fields.Sites.Matrix.IsZero() { + intSetsCfg := sitesmatrix.IntSetsConfig{ + Globs: fields.Sites.Matrix, } + matrix := sitesmatrix.NewIntSetsBuilder(s.configuredDimensions).WithConfig(intSetsCfg).WithAllIfNotSet().Build() + + addSectionMatcher( + func(fields SegmentQuery) predicate.Match { + return predicate.BoolMatch(matrix.HasVector(fields.Site)) + }, + ) } - if fields.Output != "" { - if err := addToSection(fields, func(fields SegmentMatcherFields) string { return fields.Output }); err != nil { + if len(fields.Output) > 0 { + if err := addToSection(fields, + func(fields SegmentMatcherFields) []string { return fields.Output }, + func(fields SegmentQuery) string { return fields.Output }, + ); err != nil { return result, err } } @@ -196,54 +183,119 @@ func compileSegments(f []SegmentMatcherFields) (predicate.P[SegmentMatcherFields section = nil } - return result, nil } -func DecodeSegments(in map[string]any) (*config.ConfigNamespace[map[string]SegmentConfig, Segments], error) { - buildConfig := func(in any) (Segments, any, error) { - sms := Segments{ - s: map[string]excludeInclude{}, +func (s *segmentsBuilder) build() (SegmentFilter, error) { + var sf segmentFilter + + for _, segID := range s.segmentsToRender { + segCfg, ok := s.segmentCfg[segID] + if !ok { + continue + } + + include, err := s.buildOne(segCfg.Includes) + if err != nil { + return nil, err + } + + exclude, err := s.buildOne(segCfg.Excludes) + if err != nil { + return nil, err + } + if sf.include == nil { + sf.include = include + } else { + sf.include = sf.include.Or(include) + } + if sf.exclude == nil { + sf.exclude = exclude + } else { + sf.exclude = sf.exclude.Or(exclude) } + } + + if sf.exclude == nil { + sf.exclude = matchNothing + } + if sf.include == nil { + sf.include = matchAll + } + + return sf, nil +} + +func (b *Segments) InitConfig(logger loggers.Logger, _ sitesmatrix.VectorStore, configuredDimensions *sitesmatrix.ConfiguredDimensions) error { + if b.builder == nil || b.builder.isConfigInit { + return nil + } + b.builder.isConfigInit = true + b.builder.configuredDimensions = configuredDimensions + return b.compile() +} + +var ( + matchAll = func(SegmentQuery) predicate.Match { return predicate.True } + matchNothing = func(SegmentQuery) predicate.Match { return predicate.False } +) + +type SegmentConfig struct { + Excludes []SegmentMatcherFields + Includes []SegmentMatcherFields +} + +type SegmentQuery struct { + Kind string + Path string + Output string + Site sitesmatrix.Vector +} + +// SegmentMatcherFields holds string slices of ordered filters for segment matching. +// The Glob patterns can be negated by prefixing with "! ". +// The first match wins (either include or exclude). +type SegmentMatcherFields struct { + Kind []string + Path []string + Output []string + Lang string // Deprecated: use Sites.Matrix instead. + Sites sitesmatrix.Sites // Note that we only use Sites.Matrix for now. +} + +func getGlob(s string) (glob.Glob, error) { + if s == "" { + return nil, nil + } + g, err := hglob.GetGlob(s) + if err != nil { + return nil, fmt.Errorf("failed to compile Glob %q: %w", s, err) + } + return g, nil +} + +func DecodeSegments(in map[string]any, segmentsToRender []string, logger loggers.Logger) (*config.ConfigNamespace[map[string]SegmentConfig, *Segments], error) { + buildConfig := func(in any) (*Segments, any, error) { m, err := maps.ToStringMapE(in) if err != nil { - return sms, nil, err + return nil, nil, err } if m == nil { m = map[string]any{} } m = maps.CleanConfigStringMap(m) - var scfgm map[string]SegmentConfig - if err := mapstructure.Decode(m, &scfgm); err != nil { - return sms, nil, err + var segmentCfg map[string]SegmentConfig + if err := mapstructure.WeakDecode(m, &segmentCfg); err != nil { + return nil, nil, err } - for k, v := range scfgm { - var ( - include predicate.P[SegmentMatcherFields] - exclude predicate.P[SegmentMatcherFields] - err error - ) - if v.Excludes != nil { - exclude, err = compileSegments(v.Excludes) - if err != nil { - return sms, nil, err - } - } - if v.Includes != nil { - include, err = compileSegments(v.Includes) - if err != nil { - return sms, nil, err - } - } - - ei := excludeInclude{ - exclude: exclude, - include: include, - } - sms.s[k] = ei - + sms := &Segments{ + builder: &segmentsBuilder{ + logger: logger, + segmentCfg: segmentCfg, + segmentsToRender: segmentsToRender, + }, } return sms, nil, nil diff --git a/hugolib/segments/segments_integration_test.go b/hugolib/segments/segments_integration_test.go index 465a7abe0..cfdc1a10d 100644 --- a/hugolib/segments/segments_integration_test.go +++ b/hugolib/segments/segments_integration_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Hugo Authors. All rights reserved. +// Copyright 2025 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -64,7 +64,8 @@ tags: ["tag1", "tag2"] --- ` - b := hugolib.Test(t, files) + b := hugolib.Test(t, files, hugolib.TestOptInfo()) + b.AssertLogContains("deprecated") // lang => sites.matrix in v0.152.0 b.Assert(b.H.Configs.Base.RootConfig.RenderSegments, qt.DeepEquals, []string{"docs"}) b.AssertFileContent("public/docs/section1/page1/index.html", "Docs Page 1") diff --git a/hugolib/segments/segments_test.go b/hugolib/segments/segments_test.go index 1a2dfb97b..d98df281c 100644 --- a/hugolib/segments/segments_test.go +++ b/hugolib/segments/segments_test.go @@ -4,112 +4,148 @@ import ( "testing" qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/common/predicate" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" ) +var ( + testDims = sitesmatrix.NewTestingDimensions([]string{"no", "en"}, []string{"v1.0.0", "v1.2.0", "v2.0.0"}, []string{"guest", "member"}) + no = sitesmatrix.Vector{0, 0, 0} + en = sitesmatrix.Vector{1, 0, 0} + sitesNoStar = sitesmatrix.Sites{ + Matrix: sitesmatrix.StringSlices{ + Languages: []string{"n*"}, + }, + } + sitesNo = sitesmatrix.Sites{ + Matrix: sitesmatrix.StringSlices{ + Languages: []string{"no"}, + }, + } + + sitesNotNo = sitesmatrix.Sites{ + Matrix: sitesmatrix.StringSlices{ + Languages: []string{"! no"}, + }, + } +) + +var compileSegments = func(c *qt.C, includes ...SegmentMatcherFields) predicate.P[SegmentQuery] { + segments := Segments{ + builder: &segmentsBuilder{ + logger: loggers.NewDefault(), + configuredDimensions: testDims, + segmentsToRender: []string{"docs"}, + segmentCfg: map[string]SegmentConfig{ + "docs": { + Includes: includes, + }, + }, + }, + } + c.Assert(segments.compile(), qt.IsNil) + return segments.SegmentFilter.ShouldExcludeFine +} + func TestCompileSegments(t *testing.T) { c := qt.New(t) - c.Run("excludes", func(c *qt.C) { + c.Run("variants1", func(c *qt.C) { fields := []SegmentMatcherFields{ { - Lang: "n*", - Output: "rss", + Sites: sitesNoStar, + Output: []string{"rss"}, }, } - match, err := compileSegments(fields) - c.Assert(err, qt.IsNil) + shouldExclude := compileSegments(c, fields...) check := func() { - c.Assert(match, qt.IsNotNil) - c.Assert(match(SegmentMatcherFields{Lang: "no"}), qt.Equals, false) - c.Assert(match(SegmentMatcherFields{Lang: "no", Kind: "page"}), qt.Equals, false) - c.Assert(match(SegmentMatcherFields{Lang: "no", Output: "rss"}), qt.Equals, true) - c.Assert(match(SegmentMatcherFields{Lang: "no", Output: "html"}), qt.Equals, false) - c.Assert(match(SegmentMatcherFields{Kind: "page"}), qt.Equals, false) - c.Assert(match(SegmentMatcherFields{Lang: "no", Output: "rss", Kind: "page"}), qt.Equals, true) + c.Assert(shouldExclude, qt.IsNotNil) + c.Assert(shouldExclude(SegmentQuery{Site: no}), qt.Equals, true) + c.Assert(shouldExclude(SegmentQuery{Site: no, Kind: "page"}), qt.Equals, true) + c.Assert(shouldExclude(SegmentQuery{Site: no, Output: "rss"}), qt.Equals, false) + c.Assert(shouldExclude(SegmentQuery{Site: no, Output: "html"}), qt.Equals, true) + c.Assert(shouldExclude(SegmentQuery{Kind: "page"}), qt.Equals, true) + c.Assert(shouldExclude(SegmentQuery{Site: no, Output: "rss", Kind: "page"}), qt.Equals, false) } check() fields = []SegmentMatcherFields{ { - Path: "/blog/**", + Path: []string{"/blog/**"}, }, { - Lang: "n*", - Output: "rss", + Sites: sitesNoStar, + Output: []string{"rss"}, }, } - match, err = compileSegments(fields) - c.Assert(err, qt.IsNil) + shouldExclude = compileSegments(c, fields...) check() - c.Assert(match(SegmentMatcherFields{Path: "/blog/foo"}), qt.Equals, true) + c.Assert(shouldExclude(SegmentQuery{Path: "/blog/foo"}), qt.Equals, false) }) - c.Run("includes", func(c *qt.C) { + c.Run("variants2", func(c *qt.C) { fields := []SegmentMatcherFields{ { - Path: "/docs/**", + Path: []string{"/docs/**"}, }, { - Lang: "no", - Output: "rss", + Sites: sitesNo, + Output: []string{"rss", "json"}, }, } - match, err := compileSegments(fields) - c.Assert(err, qt.IsNil) - c.Assert(match, qt.IsNotNil) - c.Assert(match(SegmentMatcherFields{Lang: "no"}), qt.Equals, false) - c.Assert(match(SegmentMatcherFields{Kind: "page"}), qt.Equals, false) - c.Assert(match(SegmentMatcherFields{Kind: "page", Path: "/blog/foo"}), qt.Equals, false) - c.Assert(match(SegmentMatcherFields{Lang: "en"}), qt.Equals, false) - c.Assert(match(SegmentMatcherFields{Lang: "no", Output: "rss"}), qt.Equals, true) - c.Assert(match(SegmentMatcherFields{Lang: "no", Output: "html"}), qt.Equals, false) - c.Assert(match(SegmentMatcherFields{Kind: "page", Path: "/docs/foo"}), qt.Equals, true) + shouldExclude := compileSegments(c, fields...) + c.Assert(shouldExclude, qt.IsNotNil) + c.Assert(shouldExclude(SegmentQuery{Site: no}), qt.Equals, true) + c.Assert(shouldExclude(SegmentQuery{Kind: "page"}), qt.Equals, true) + c.Assert(shouldExclude(SegmentQuery{Kind: "page", Path: "/blog/foo"}), qt.Equals, true) + c.Assert(shouldExclude(SegmentQuery{Site: en}), qt.Equals, true) + c.Assert(shouldExclude(SegmentQuery{Site: no, Output: "rss"}), qt.Equals, false) + c.Assert(shouldExclude(SegmentQuery{Site: no, Output: "json"}), qt.Equals, false) + c.Assert(shouldExclude(SegmentQuery{Site: no, Output: "html"}), qt.Equals, true) + c.Assert(shouldExclude(SegmentQuery{Kind: "page", Path: "/docs/foo"}), qt.Equals, false) }) +} - c.Run("includes variant1", func(c *qt.C) { - c.Skip() +func TestCompileSegmentsNegate(t *testing.T) { + c := qt.New(t) - fields := []SegmentMatcherFields{ - { - Kind: "home", - }, - { - Path: "{/docs,/docs/**}", - }, - } + fields := []SegmentMatcherFields{ + { + Sites: sitesNotNo, + Output: []string{"! r**", "rem", "html"}, + }, + } - match, err := compileSegments(fields) - c.Assert(err, qt.IsNil) - c.Assert(match, qt.IsNotNil) - c.Assert(match(SegmentMatcherFields{Path: "/blog/foo"}), qt.Equals, false) - c.Assert(match(SegmentMatcherFields{Kind: "page", Path: "/docs/foo"}), qt.Equals, true) - c.Assert(match(SegmentMatcherFields{Kind: "home", Path: "/"}), qt.Equals, true) - }) + shouldExclude := compileSegments(c, fields...) + c.Assert(shouldExclude, qt.IsNotNil) + c.Assert(shouldExclude(SegmentQuery{Site: no, Output: "html"}), qt.Equals, true) + c.Assert(shouldExclude(SegmentQuery{Site: en, Output: "html"}), qt.Equals, false) + c.Assert(shouldExclude(SegmentQuery{Site: en, Output: "rss"}), qt.Equals, true) + c.Assert(shouldExclude(SegmentQuery{Site: en, Output: "rem"}), qt.Equals, true) } func BenchmarkSegmentsMatch(b *testing.B) { + c := qt.New(b) fields := []SegmentMatcherFields{ { - Path: "/docs/**", + Path: []string{"/docs/**"}, }, { - Lang: "no", - Output: "rss", + Sites: sitesNo, + Output: []string{"rss"}, }, } - match, err := compileSegments(fields) - if err != nil { - b.Fatal(err) - } + match := compileSegments(c, fields...) b.ResetTimer() - for i := 0; i < b.N; i++ { - match(SegmentMatcherFields{Lang: "no", Output: "rss"}) + for range b.N { + match(SegmentQuery{Site: no, Output: "rss"}) } } diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go index 6a5316682..20fe2bc9d 100644 --- a/hugolib/shortcode.go +++ b/hugolib/shortcode.go @@ -30,6 +30,7 @@ import ( "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/hstore" "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/tpl/tplimpl" "github.com/gohugoio/hugo/parser/pageparser" @@ -276,26 +277,40 @@ func (sc shortcode) String() string { return fmt.Sprintf("%s(%q, %t){%s}", sc.name, params, sc.doMarkup, sc.inner) } -type shortcodeHandler struct { - filename string - s *Site +// shared between all pages from the same source file. +type shortcodeParseInfo struct { + filename string + firstTemplateStore *tplimpl.TemplateStore // This represents the first site and must not be used for execution. - // Ordered list of shortcodes for a page. + // Ordered list of shortcodes. shortcodes []*shortcode // All the shortcode names in this set. - nameSet map[string]bool nameSetMu sync.RWMutex + nameSet map[string]bool // Configuration enableInlineShortcodes bool } -func newShortcodeHandler(filename string, s *Site) *shortcodeHandler { - sh := &shortcodeHandler{ +func (s *shortcodeParseInfo) addName(name string) { + s.nameSetMu.Lock() + defer s.nameSetMu.Unlock() + s.nameSet[name] = true +} + +func (s *shortcodeParseInfo) hasName(name string) bool { + s.nameSetMu.RLock() + defer s.nameSetMu.RUnlock() + _, ok := s.nameSet[name] + return ok +} + +func newShortcodeHandler(filename string, d *deps.Deps) *shortcodeParseInfo { + sh := &shortcodeParseInfo{ filename: filename, - s: s, - enableInlineShortcodes: s.ExecHelper.Sec().EnableInlineShortcodes, + firstTemplateStore: d.TemplateStore, + enableInlineShortcodes: d.ExecHelper.Sec().EnableInlineShortcodes, shortcodes: make([]*shortcode, 0, 4), nameSet: make(map[string]bool), } @@ -310,9 +325,7 @@ const ( ) func prepareShortcode( - ctx context.Context, level int, - s *Site, sc *shortcode, parent *ShortcodeWithPage, po *pageOutput, @@ -326,12 +339,12 @@ func prepareShortcode( // Allow the caller to delay the rendering of the shortcode if needed. var fn shortcodeRenderFunc = func(ctx context.Context) ([]byte, bool, error) { - if p.m.pageConfig.ContentMediaType.IsMarkdown() && sc.doMarkup { + if p.m.pageConfigSource.ContentMediaType.IsMarkdown() && sc.doMarkup { // Signal downwards that the content rendered will be // parsed and rendered by Goldmark. ctx = tpl.Context.IsInGoldmark.Set(ctx, true) } - r, err := doRenderShortcode(ctx, level, s, sc, parent, po, isRenderString) + r, err := doRenderShortcode(ctx, level, po.p.s.TemplateStore, sc, parent, po, isRenderString) if err != nil { return nil, false, toParseErr(err) } @@ -349,7 +362,7 @@ func prepareShortcode( func doRenderShortcode( ctx context.Context, level int, - s *Site, + ts *tplimpl.TemplateStore, sc *shortcode, parent *ShortcodeWithPage, po *pageOutput, @@ -372,7 +385,7 @@ func doRenderShortcode( templStr := sc.innerString() var err error - tmpl, err = s.TemplateStore.TextParse(templatePath, templStr) + tmpl, err = ts.TextParse(templatePath, templStr) if err != nil { if isRenderString { return zeroShortcode, p.wrapError(err) @@ -386,7 +399,7 @@ func doRenderShortcode( } else { // Re-use of shortcode defined earlier in the same page. - tmpl = s.TemplateStore.TextLookup(templatePath) + tmpl = ts.TextLookup(templatePath) if tmpl == nil { return zeroShortcode, fmt.Errorf("no earlier definition of shortcode %q found", sc.name) } @@ -407,9 +420,10 @@ func doRenderShortcode( Name: sc.name, Category: tplimpl.CategoryShortcode, Desc: layoutDescriptor, + Sites: p.s.siteVector, Consider: include, } - v, err := s.TemplateStore.LookupShortcode(q) + v, err := ts.LookupShortcode(q) if v == nil { return zeroShortcode, err } @@ -438,7 +452,7 @@ func doRenderShortcode( case string: inner += innerData case *shortcode: - s, err := prepareShortcode(ctx, level+1, s, innerData, data, po, isRenderString) + s, err := prepareShortcode(level+1, innerData, data, po, isRenderString) if err != nil { return zeroShortcode, err } @@ -449,7 +463,7 @@ func doRenderShortcode( } inner += ss default: - s.Log.Errorf("Illegal state on shortcode rendering of %q in page %q. Illegal type in inner data: %s ", + po.p.s.Log.Errorf("Illegal state on shortcode rendering of %q in page %q. Illegal type in inner data: %s ", sc.name, p.File().Path(), reflect.TypeOf(innerData)) return zeroShortcode, nil } @@ -476,7 +490,7 @@ func doRenderShortcode( // unchanged. // 2 If inner does not have a newline, strip the wrapping

block and // the newline. - switch p.m.pageConfig.Content.Markup { + switch p.m.pageConfigSource.Content.Markup { case "", "markdown": if match, _ := regexp.MatchString(innerNewlineRegexp, inner); !match { cleaner, err := regexp.Compile(innerCleanupRegexp) @@ -495,7 +509,7 @@ func doRenderShortcode( } - result, err := renderShortcodeWithPage(ctx, s.GetTemplateStore(), tmpl, data) + result, err := renderShortcodeWithPage(ctx, ts, tmpl, data) if err != nil && sc.isInline { fe := herrors.NewFileErrorFromName(err, p.File().Filename()) @@ -524,37 +538,15 @@ func doRenderShortcode( return prerenderedShortcode{s: result, hasVariants: hasVariants}, err } -func (s *shortcodeHandler) addName(name string) { - s.nameSetMu.Lock() - defer s.nameSetMu.Unlock() - s.nameSet[name] = true -} - -func (s *shortcodeHandler) transferNames(in *shortcodeHandler) { - s.nameSetMu.Lock() - defer s.nameSetMu.Unlock() - for k := range in.nameSet { - s.nameSet[k] = true - } -} - -func (s *shortcodeHandler) hasName(name string) bool { - s.nameSetMu.RLock() - defer s.nameSetMu.RUnlock() - _, ok := s.nameSet[name] - return ok -} - -func (s *shortcodeHandler) prepareShortcodesForPage(ctx context.Context, po *pageOutput, isRenderString bool) (map[string]shortcodeRenderer, error) { +func (s *shortcodeParseInfo) prepareShortcodesForPage(po *pageOutput, isRenderString bool) (map[string]shortcodeRenderer, error) { rendered := make(map[string]shortcodeRenderer) for _, v := range s.shortcodes { - s, err := prepareShortcode(ctx, 0, s.s, v, nil, po, isRenderString) + s, err := prepareShortcode(0, v, nil, po, isRenderString) if err != nil { return nil, err } rendered[v.placeholder] = s - } return rendered, nil @@ -582,7 +574,7 @@ func posFromInput(filename string, input []byte, offset int) text.Position { // pageTokens state: // - before: positioned just before the shortcode start // - after: shortcode(s) consumed (plural when they are nested) -func (s *shortcodeHandler) extractShortcode(ordinal, level int, source []byte, pt *pageparser.Iterator) (*shortcode, error) { +func (s *shortcodeParseInfo) extractShortcode(ordinal, level int, source []byte, pt *pageparser.Iterator) (*shortcode, error) { if s == nil { panic("handler nil") } @@ -670,7 +662,6 @@ Loop: sc.isClosing = true pt.Consume(2) } - return sc, nil case currItem.IsText(): sc.inner = append(sc.inner, currItem.ValStr(source)) @@ -680,7 +671,7 @@ Loop: // Used to check if the template expects inner content, // so just pick one arbitrarily with the same name. - templ := s.s.TemplateStore.LookupShortcodeByName(sc.name) + templ := s.firstTemplateStore.LookupShortcodeByName(sc.name) if templ == nil { return nil, fmt.Errorf("%s: template for shortcode %q not found", errorPrefix, sc.name) } diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go index a1f12e77a..90fda6593 100644 --- a/hugolib/shortcode_test.go +++ b/hugolib/shortcode_test.go @@ -109,7 +109,7 @@ title: "Shortcodes Galore!" p, err := pageparser.ParseMain(strings.NewReader(test.input), pageparser.Config{}) c.Assert(err, qt.IsNil) - handler := newShortcodeHandler("", s) + handler := newShortcodeHandler("", s.Deps) iter := p.Iterator() short, err := handler.extractShortcode(0, 0, p.Input(), iter) diff --git a/hugolib/site.go b/hugolib/site.go index ff9d0310a..be75f5c7b 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -32,6 +32,7 @@ import ( "github.com/bep/logg" "github.com/gohugoio/hugo/cache/dynacache" "github.com/gohugoio/hugo/common/hstore" + "github.com/gohugoio/hugo/common/hsync" "github.com/gohugoio/hugo/common/htime" "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/common/hugo" @@ -44,6 +45,9 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugolib/doctree" "github.com/gohugoio/hugo/hugolib/pagesfromdata" + "github.com/gohugoio/hugo/hugolib/roles" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" + "github.com/gohugoio/hugo/hugolib/versions" "github.com/gohugoio/hugo/internal/js/esbuild" "github.com/gohugoio/hugo/internal/warpc" "github.com/gohugoio/hugo/langs/i18n" @@ -79,8 +83,6 @@ import ( "github.com/gohugoio/hugo/resources/page/siteidentities" "github.com/gohugoio/hugo/resources/resource" - "github.com/gohugoio/hugo/lazy" - "github.com/fsnotify/fsnotify" bp "github.com/gohugoio/hugo/bufferpool" "github.com/gohugoio/hugo/helpers" @@ -99,22 +101,55 @@ const ( ) type Site struct { - state siteState - conf *allconfig.Config - language *langs.Language - languagei int - pageMap *pageMap - store *hstore.Scratch + state siteState + conf *allconfig.Config + language *langs.Language + store *hstore.Scratch + siteWrapped page.Site // The owning container. h *HugoSites *deps.Deps + *siteLanguageVersionRole + + relatedDocsHandler *page.RelatedDocsHandler + + publisher publisher.Publisher + frontmatterHandler pagemeta.FrontMatterHandler +} + +func (s Site) cloneForVersionAndRole(version, role int) (*Site, error) { + s.siteLanguageVersionRole = s.siteLanguageVersionRole.cloneForVersionAndRole(version, role) + d, err := s.Deps.Clone(&s, s.Conf) + if err != nil { + return nil, err + } + s.Deps = d + ss := &s + ss.siteWrapped = page.WrapSite(ss) + return ss, nil +} + +// For debugging purposes only. +func (s *Site) resolveDimensionNames() types.Strings3 { + return s.Conf.ConfiguredDimensions().ResolveNames(s.siteVector) +} + +type siteLanguageVersionRole struct { + siteVector sitesmatrix.Vector + + roleInternal roles.RoleInternal + role roles.Role + + versionInternal versions.VersionInternal + version versions.Version + + pageMap *pageMap // Page navigation. *pageFinder - taxonomies page.TaxonomyList - menus navigation.Menus + siteRefLinker siteRefLinker // Shortcut to the home page. Note that this may be nil if // home page, for some odd reason, is disabled. @@ -123,19 +158,31 @@ type Site struct { // The last modification date of this site. lastmod time.Time - relatedDocsHandler *page.RelatedDocsHandler - siteRefLinker - publisher publisher.Publisher - frontmatterHandler pagemeta.FrontMatterHandler + // Lazily loaded site dependencies + init *siteInit // The output formats that we need to render this site in. This slice // will be fixed once set. // This will be the union of Site.Pages' outputFormats. // This slice will be sorted. renderFormats output.Formats +} - // Lazily loaded site dependencies - init *siteInit +func (s siteLanguageVersionRole) cloneForVersionAndRole(version, role int) *siteLanguageVersionRole { + s.siteVector[sitesmatrix.Version] = version + s.siteVector[sitesmatrix.Role] = role + s.home = nil + s.lastmod = time.Time{} + s.init = &siteInit{} + return &s +} + +func (s siteLanguageVersionRole) Role() roles.Role { + return s.role +} + +func (s siteLanguageVersionRole) Version() versions.Version { + return s.version } func (s *Site) Debug() { @@ -146,6 +193,8 @@ func (s *Site) Debug() { // NewHugoSites creates HugoSites from the given config. func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { conf := cfg.Configs.GetFirstLanguageConfig() + rolesSorted := cfg.Configs.Base.Roles.Config.Sorted + versionsSorted := cfg.Configs.Base.Versions.Config.Sorted var logger loggers.Logger if cfg.TestLogger != nil { @@ -229,13 +278,16 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { var sites []*Site ns := &contentNodeShifter{ - numLanguages: len(confm.Languages), + conf: conf, } - treeConfig := doctree.Config[contentNodeI]{ - Shifter: ns, + treeConfig := doctree.Config[contentNode]{ + Shifter: ns, + TransformerRaw: &contentNodeTransformerRaw{}, } + dimensionLengths := sitesmatrix.Vector{len(confm.Languages), len(versionsSorted), len(rolesSorted)} + pageTrees := &pageTrees{ treePages: doctree.New( treeConfig, @@ -243,16 +295,16 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { treeResources: doctree.New( treeConfig, ), - treeTaxonomyEntries: doctree.NewTreeShiftTree[*weightedContentNode](doctree.DimensionLanguage.Index(), len(confm.Languages)), - treePagesFromTemplateAdapters: doctree.NewTreeShiftTree[*pagesfromdata.PagesFromTemplate](doctree.DimensionLanguage.Index(), len(confm.Languages)), + treeTaxonomyEntries: doctree.NewTreeShiftTree[*weightedContentNode](dimensionLengths), + treePagesFromTemplateAdapters: doctree.NewTreeShiftTree[*pagesfromdata.PagesFromTemplate](dimensionLengths), } pageTrees.createMutableTrees() for i, confp := range confm.ConfigLangs() { - language := confp.Language() + language := confp.Language().(*langs.Language) if language.Disabled { - continue + panic("cannot create site for disabled language: " + language.Lang) } k := language.Lang conf := confm.LanguageConfigMap[k] @@ -266,11 +318,15 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { s := &Site{ conf: conf, language: language, - languagei: i, frontmatterHandler: frontmatterHandler, store: hstore.NewScratch(), + siteLanguageVersionRole: &siteLanguageVersionRole{ + siteVector: sitesmatrix.Vector{i, 0, 0}, + }, } + s.siteWrapped = page.WrapSite(s) + if i == 0 { firstSiteDeps.Site = s s.Deps = firstSiteDeps @@ -282,13 +338,6 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { s.Deps = d } - s.pageMap = newPageMap(i, s, memCache, pageTrees) - - s.pageFinder = newPageFinder(s.pageMap) - s.siteRefLinker, err = newSiteRefLinker(s) - if err != nil { - return nil, err - } // Set up the main publishing chain. pub, err := publisher.NewDestinationPublisher( firstSiteDeps.ResourceSpec, @@ -303,34 +352,54 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { s.relatedDocsHandler = page.NewRelatedDocsHandler(s.conf.Related) // Site deps end. - s.prepareInits() sites = append(sites, s) + } if len(sites) == 0 { return nil, errors.New("no sites to build") } - // Pull the default content language to the top, then sort the sites by language weight (if set) or lang. - defaultContentLanguage := confm.Base.DefaultContentLanguage - sort.Slice(sites, func(i, j int) bool { - li := sites[i].language - lj := sites[j].language - if li.Lang == defaultContentLanguage { - return true - } - - if lj.Lang == defaultContentLanguage { - return false - } - - if li.Weight != lj.Weight { - return li.Weight < lj.Weight + sitesVersionsRoles := make([][][]*Site, len(sites)) + for i := 0; i < len(sites); i++ { + sitesVersionsRoles[i] = make([][]*Site, len(versionsSorted)) + for j := 0; j < len(versionsSorted); j++ { + sitesVersionsRoles[i][j] = make([]*Site, len(rolesSorted)) + } + } + + siteVersionRoles := map[types.Ints2][]roles.Role{} + siteRoleVersions := map[types.Ints2][]versions.Version{} + // i = site, j = version, k = role + for i, v1 := range sitesVersionsRoles { + for j, v2 := range v1 { + for k := range v2 { + var vrs *Site + if j == 0 && k == 0 { + vrs = sites[i] + } else { + prototype := sites[i] + vrs, err = prototype.cloneForVersionAndRole(j, k) + if err != nil { + return nil, err + } + } + vrs.roleInternal = rolesSorted[k] + vrs.role = roles.NewRole(rolesSorted[k]) + vrs.versionInternal = versionsSorted[j] + vrs.version = versions.NewVersion(versionsSorted[j]) + siteRoleVersions[types.Ints2{i, k}] = append(siteRoleVersions[types.Ints2{i, k}], vrs.version) + siteVersionRoles[types.Ints2{i, j}] = append(siteVersionRoles[types.Ints2{i, j}], vrs.role) + vrs.pageMap = newPageMap(vrs, memCache, pageTrees) + vrs.pageFinder = newPageFinder(vrs.pageMap) + vrs.siteRefLinker = newSiteRefLinker(vrs) + vrs.prepareInits() + v2[k] = vrs + } } - return li.Lang < lj.Lang - }) + } - h, err = newHugoSites(cfg, firstSiteDeps, pageTrees, sites) + h, err = newHugoSites(cfg, firstSiteDeps, pageTrees, sitesVersionsRoles) if err == nil && h == nil { panic("hugo: newHugoSitesNew returned nil error and nil HugoSites") } @@ -338,32 +407,56 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { return h, err } -func newHugoSites(cfg deps.DepsCfg, d *deps.Deps, pageTrees *pageTrees, sites []*Site) (*HugoSites, error) { +func newHugoSites( + cfg deps.DepsCfg, + d *deps.Deps, + pageTrees *pageTrees, + sitesVersionsRoles [][][]*Site, +) (*HugoSites, error) { + first := make([]*Site, len(sitesVersionsRoles)) + for i, v := range sitesVersionsRoles { + first[i] = v[0][0] + } + + sitesVersionsRolesMap := map[sitesmatrix.Vector]*Site{} + for _, v1 := range sitesVersionsRoles { + for _, v2 := range v1 { + for _, s := range v2 { + sitesVersionsRolesMap[s.siteVector] = s + } + } + } + numWorkers := config.GetNumWorkerMultiplier() - numWorkersSite := min(numWorkers, len(sites)) + numWorkersSite := min(numWorkers, len(sitesVersionsRolesMap)) workersSite := para.New(numWorkersSite) + var sitesLanguages []*Site + for _, v := range sitesVersionsRoles { + sitesLanguages = append(sitesLanguages, v[0][0]) + } + h := &HugoSites{ - Sites: sites, - Deps: sites[0].Deps, - Configs: cfg.Configs, - workersSite: workersSite, - numWorkersSites: numWorkers, - numWorkers: numWorkers, - pageTrees: pageTrees, + Sites: first, + sitesVersionsRoles: sitesVersionsRoles, + sitesVersionsRolesMap: sitesVersionsRolesMap, + sitesLanguages: sitesLanguages, + Deps: first[0].Deps, + Configs: cfg.Configs, + workersSite: workersSite, + numWorkersSites: numWorkers, + numWorkers: numWorkers, + pageTrees: pageTrees, cachePages: dynacache.GetOrCreatePartition[string, page.Pages](d.MemCache, "/pags/all", dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}, ), cacheContentSource: dynacache.GetOrCreatePartition[string, *resources.StaleValue[[]byte]](d.MemCache, "/cont/src", dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}), translationKeyPages: maps.NewSliceCache[page.Page](), - currentSite: sites[0], + currentSite: first[0], skipRebuildForFilenames: make(map[string]bool), - init: &hugoSitesInit{ - data: lazy.New(), - gitInfo: lazy.New(), - }, - progressReporter: &progressReporter{}, + init: &hugoSitesInit{}, + progressReporter: &progressReporter{}, } // Assemble dependencies to be used in hugo.Deps. @@ -394,43 +487,50 @@ func newHugoSites(cfg deps.DepsCfg, d *deps.Deps, pageTrees *pageTrees, sites [] h.hugoInfo = hugo.NewInfo(h.Configs.GetFirstLanguageConfig(), dependencies) var prototype *deps.Deps - for i, s := range sites { - s.h = h - // The template store needs to be initialized after the h container is set on s. - if i == 0 { - templateStore, err := tplimpl.NewStore( - tplimpl.StoreOptions{ - Fs: s.BaseFs.Layouts.Fs, - Log: s.Log, - DefaultContentLanguage: s.Conf.DefaultContentLanguage(), - Watching: s.Conf.Watching(), - PathParser: s.Conf.PathParser(), - Metrics: d.Metrics, - OutputFormats: s.conf.OutputFormats.Config, - MediaTypes: s.conf.MediaTypes.Config, - DefaultOutputFormat: s.conf.DefaultOutputFormat, - TaxonomySingularPlural: s.conf.Taxonomies, - RenderHooks: s.conf.Markup.Goldmark.RenderHooks, - }, tplimpl.SiteOptions{ - Site: s, - TemplateFuncs: tplimplinit.CreateFuncMap(s.Deps), - }) - if err != nil { - return nil, err + + var i int + for _, v := range sitesVersionsRoles { + for _, r := range v { + for _, s := range r { + s.h = h + // The template store needs to be initialized after the h container is set on s. + if i == 0 { + templateStore, err := tplimpl.NewStore( + tplimpl.StoreOptions{ + Fs: s.BaseFs.Layouts.Fs, + Log: s.Log, + Watching: s.Conf.Watching(), + PathParser: s.Conf.PathParser(), + Metrics: d.Metrics, + OutputFormats: s.conf.OutputFormats.Config, + MediaTypes: s.conf.MediaTypes.Config, + DefaultOutputFormat: s.conf.DefaultOutputFormat, + TaxonomySingularPlural: s.conf.Taxonomies, + RenderHooks: s.conf.Markup.Goldmark.RenderHooks, + }, tplimpl.SiteOptions{ + Site: s, + TemplateFuncs: tplimplinit.CreateFuncMap(s.Deps), + }) + if err != nil { + return nil, err + } + s.Deps.TemplateStore = templateStore + } else { + s.Deps.TemplateStore = prototype.TemplateStore.WithSiteOpts( + tplimpl.SiteOptions{ + Site: s, + TemplateFuncs: tplimplinit.CreateFuncMap(s.Deps), + }) + } + if err := s.Deps.Compile(prototype); err != nil { + return nil, err + } + if i == 0 { + prototype = s.Deps + } + + i++ } - s.Deps.TemplateStore = templateStore - } else { - s.Deps.TemplateStore = prototype.TemplateStore.WithSiteOpts( - tplimpl.SiteOptions{ - Site: s, - TemplateFuncs: tplimplinit.CreateFuncMap(s.Deps), - }) - } - if err := s.Deps.Compile(prototype); err != nil { - return nil, err - } - if i == 0 { - prototype = s.Deps } } @@ -439,25 +539,32 @@ func newHugoSites(cfg deps.DepsCfg, d *deps.Deps, pageTrees *pageTrees, sites [] donec: make(chan bool), } - h.init.data.Add(func(context.Context) (any, error) { - err := h.loadData() - if err != nil { - return nil, fmt.Errorf("failed to load data: %w", err) - } - return nil, nil + h.init.data = hsync.OnceMoreFunc(func(ctx context.Context) error { + return h.loadData() }) - h.init.gitInfo.Add(func(context.Context) (any, error) { - err := h.loadGitInfo() - if err != nil { - return nil, fmt.Errorf("failed to load Git info: %w", err) - } - return nil, nil + h.init.gitInfo = hsync.OnceMoreFunc(func(ctx context.Context) error { + return h.loadGitInfo() }) return h, nil } +// GetSiteDimension returns the value of the specified site dimension (such as language, version, or role) +// based on the provided dimension name string. If the dimension name is unknown, it panics. +func (s *Site) Dimension(d string) page.SiteDimension { + switch d { + case sitesmatrix.DimensionName(sitesmatrix.Language): + return s.language + case sitesmatrix.DimensionName(sitesmatrix.Version): + return s.version + case sitesmatrix.DimensionName(sitesmatrix.Role): + return s.role + default: + panic(fmt.Sprintf("unknown dimension %q", d)) + } +} + // Returns the server port. func (s *Site) ServerPort() int { return s.conf.C.BaseURL.Port() @@ -472,6 +579,10 @@ func (s *Site) Copyright() string { return s.conf.Copyright } +func (s *Site) Lang() string { + return s.language.Lang +} + func (s *Site) Config() page.SiteConfig { return page.SiteConfig{ Privacy: s.conf.Privacy, @@ -562,7 +673,7 @@ func (s *Site) Param(key any) (any, error) { // Returns a map of all the data inside /data. func (s *Site) Data() map[string]any { - return s.s.h.Data() + return s.h.Data() } func (s *Site) BuildDrafts() bool { @@ -604,7 +715,7 @@ func (s *Site) Pages() page.Pages { pageMapQueryPagesBelowPath: pageMapQueryPagesBelowPath{ Path: "", KeyPart: "global", - Include: pagePredicates.ShouldListGlobal, + Include: pagePredicates.ShouldListGlobal.BoolFunc(), }, Recursive: true, IncludeSelf: true, @@ -621,7 +732,7 @@ func (s *Site) RegularPages() page.Pages { pageMapQueryPagesBelowPath: pageMapQueryPagesBelowPath{ Path: "", KeyPart: "global", - Include: pagePredicates.ShouldListGlobal.And(pagePredicates.KindPage), + Include: pagePredicates.ShouldListGlobal.And(pagePredicates.KindPage).BoolFunc(), }, Recursive: true, }, @@ -652,8 +763,7 @@ func (s *Site) CheckReady() { func (s *Site) Taxonomies() page.TaxonomyList { s.CheckReady() - s.init.taxonomies.Do(context.Background()) - return s.taxonomies + return s.init.taxonomies.Value(context.Background()) } type ( @@ -686,10 +796,11 @@ func (t taxonomiesConfig) Values() taxonomiesConfigValues { // Lazily loaded site dependencies. type siteInit struct { - prevNext *lazy.Init - prevNextInSection *lazy.Init - menus *lazy.Init - taxonomies *lazy.Init + prevNext hsync.FuncResetter + prevNextInSection hsync.FuncResetter + + menus hsync.ValueResetter[navigation.Menus] + taxonomies hsync.ValueResetter[page.TaxonomyList] } func (init *siteInit) Reset() { @@ -702,9 +813,7 @@ func (init *siteInit) Reset() { func (s *Site) prepareInits() { s.init = &siteInit{} - var init lazy.Init - - s.init.prevNext = init.Branch(func(context.Context) (any, error) { + s.init.prevNext = hsync.OnceMoreFunc(func(ctx context.Context) error { regularPages := s.RegularPages() if s.conf.Page.NextPrevSortOrder == "asc" { regularPages = regularPages.Reverse() @@ -731,10 +840,10 @@ func (s *Site) prepareInits() { pos.prevPage = regularPages[i+1] } } - return nil, nil + return nil }) - s.init.prevNextInSection = init.Branch(func(context.Context) (any, error) { + s.init.prevNextInSection = hsync.OnceMoreFunc(func(ctx context.Context) error { setNextPrev := func(pas page.Pages) { for i, p := range pas { np, ok := p.(nextPrevInSectionProvider) @@ -765,7 +874,7 @@ func (s *Site) prepareInits() { pageMapQueryPagesBelowPath: pageMapQueryPagesBelowPath{ Path: "", KeyPart: "sectionorhome", - Include: pagePredicates.KindSection.Or(pagePredicates.KindHome), + Include: pagePredicates.KindSection.Or(pagePredicates.KindHome).BoolFunc(), }, IncludeSelf: true, Recursive: true, @@ -780,35 +889,38 @@ func (s *Site) prepareInits() { setNextPrev(ps) } - return nil, nil + return nil }) - s.init.menus = init.Branch(func(context.Context) (any, error) { - err := s.assembleMenus() - return nil, err + s.init.menus = hsync.OnceMoreValue(func(ctx context.Context) navigation.Menus { + m, err := s.assembleMenus() + if err != nil { + panic(err) + } + return m }) - s.init.taxonomies = init.Branch(func(ctx context.Context) (any, error) { - if err := s.pageMap.CreateSiteTaxonomies(ctx); err != nil { - return nil, err + s.init.taxonomies = hsync.OnceMoreValue(func(ctx context.Context) page.TaxonomyList { + taxonomies, err := s.pageMap.CreateSiteTaxonomies(ctx) + if err != nil { + panic(err) } - return s.taxonomies, nil + return taxonomies }) } func (s *Site) Menus() navigation.Menus { s.CheckReady() - s.init.menus.Do(context.Background()) - return s.menus + return s.init.menus.Value(context.Background()) } func (s *Site) initRenderFormats() { formatSet := make(map[string]bool) formats := output.Formats{} - w := &doctree.NodeShiftTreeWalker[contentNodeI]{ + w := &doctree.NodeShiftTreeWalker[contentNode]{ Tree: s.pageMap.treePages, - Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { + Handle: func(key string, n contentNode) (bool, error) { if p, ok := n.(*pageState); ok { for _, f := range p.m.pageConfig.ConfiguredOutputFormats { if !formatSet[f.Name] { @@ -860,7 +972,7 @@ type siteRefLinker struct { notFoundURL string } -func newSiteRefLinker(s *Site) (siteRefLinker, error) { +func newSiteRefLinker(s *Site) siteRefLinker { logger := s.Log.Error() notFoundURL := s.conf.RefLinksNotFoundURL @@ -868,7 +980,7 @@ func newSiteRefLinker(s *Site) (siteRefLinker, error) { if strings.EqualFold(errLevel, "warning") { logger = s.Log.Warn() } - return siteRefLinker{s: s, errorLogger: logger, notFoundURL: notFoundURL}, nil + return siteRefLinker{s: s, errorLogger: logger, notFoundURL: notFoundURL} } func (s siteRefLinker) logNotFound(ref, what string, p page.Page, position text.Position) { @@ -1226,7 +1338,7 @@ func (h *HugoSites) fileEventsContentPaths(p []pathChange) []pathChange { // SitemapAbsURL is a convenience method giving the absolute URL to the sitemap. func (s *Site) SitemapAbsURL() string { base := "" - if len(s.conf.Languages) > 1 || s.Conf.DefaultContentLanguageInSubdir() { + if s.Conf.IsMultilingual() || s.Conf.DefaultContentLanguageInSubdir() { base = s.Language().Lang } p := s.AbsURL(base, false) @@ -1243,15 +1355,15 @@ func (s *Site) createNodeMenuEntryURL(in string) string { } // make it match the nodes menuEntryURL := in - menuEntryURL = s.s.PathSpec.URLize(menuEntryURL) + menuEntryURL = s.PathSpec.URLize(menuEntryURL) if !s.conf.CanonifyURLs { - menuEntryURL = paths.AddContextRoot(s.s.PathSpec.Cfg.BaseURL().String(), menuEntryURL) + menuEntryURL = paths.AddContextRoot(s.PathSpec.Cfg.BaseURL().String(), menuEntryURL) } return menuEntryURL } -func (s *Site) assembleMenus() error { - s.menus = make(navigation.Menus) +func (s *Site) assembleMenus() (navigation.Menus, error) { + menus := make(navigation.Menus) type twoD struct { MenuName, EntryName string @@ -1308,7 +1420,7 @@ func (s *Site) assembleMenus() error { flat[twoD{sectionPagesMenu, me.KeyName()}] = &me return false, nil }); err != nil { - return err + return nil, err } } @@ -1324,7 +1436,7 @@ func (s *Site) assembleMenus() error { } return false, nil }); err != nil { - return err + return nil, err } // Create Children Menus First @@ -1351,15 +1463,15 @@ func (s *Site) assembleMenus() error { // Assembling Top Level of Tree for menu, e := range flat { if e.Parent == "" { - _, ok := s.menus[menu.MenuName] + _, ok := menus[menu.MenuName] if !ok { - s.menus[menu.MenuName] = navigation.Menu{} + menus[menu.MenuName] = navigation.Menu{} } - s.menus[menu.MenuName] = s.menus[menu.MenuName].Add(e) + menus[menu.MenuName] = menus[menu.MenuName].Add(e) } } - return nil + return menus, nil } // get any language code to prefix the target file path with. @@ -1384,6 +1496,28 @@ func (s *Site) getLanguagePermalinkLang(alwaysInSubDir bool) string { return s.GetLanguagePrefix() } +func (s *Site) getPrefixRole() string { + role := s.roleInternal + if role.Default { + if s.conf.DefaultContentRoleInSubdir { + return role.Name + } + return "" + } + return role.Name +} + +func (s *Site) getPrefixVersion() string { + version := s.versionInternal + if version.Default { + if s.conf.DefaultContentVersionInSubdir { + return version.Name + } + return "" + } + return version.Name +} + // Prepare site for a new full build. func (s *Site) resetBuildState(sourceChanged bool) { s.relatedDocsHandler = s.relatedDocsHandler.Clone() @@ -1411,7 +1545,7 @@ func (s *Site) errorCollator(results <-chan error, errs chan<- error) { // i.e. 2 arguments, so we test for that. func (s *Site) GetPage(ref ...string) (page.Page, error) { s.CheckReady() - p, err := s.s.getPageForRefs(ref...) + p, err := s.getPageForRefs(ref...) if p == nil { // The nil struct has meaning in some situations, mostly to avoid breaking @@ -1443,7 +1577,7 @@ const ( pageDependencyScopeGlobal ) -func (s *Site) renderAndWritePage(statCounter *uint64, name string, targetPath string, p *pageState, d any, templ *tplimpl.TemplInfo) error { +func (s *Site) renderAndWritePage(statCounter *uint64, targetPath string, p *pageState, d any, templ *tplimpl.TemplInfo) error { s.h.onPageRender() renderBuffer := bp.GetBuffer() defer bp.PutBuffer(renderBuffer) @@ -1617,3 +1751,11 @@ func (s *Site) render(ctx *siteRenderContext) (err error) { return } + +func (s *Site) String() string { + if s == nil { + return "Site (nil)" + } + + return fmt.Sprintf("Site %v: %s", s.siteVector, s.resolveDimensionNames()) +} diff --git a/hugolib/site_render.go b/hugolib/site_render.go index 6dbb19827..719430966 100644 --- a/hugolib/site_render.go +++ b/hugolib/site_render.go @@ -85,9 +85,9 @@ func (s *Site) renderPages(ctx *siteRenderContext) error { cfg := ctx.cfg - w := &doctree.NodeShiftTreeWalker[contentNodeI]{ + w := &doctree.NodeShiftTreeWalker[contentNode]{ Tree: s.pageMap.treePages, - Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { + Handle: func(key string, n contentNode) (bool, error) { if p, ok := n.(*pageState); ok { if cfg.shouldRender(ctx.infol, p) { select { @@ -114,7 +114,7 @@ func (s *Site) renderPages(ctx *siteRenderContext) error { err := <-errs if err != nil { - return fmt.Errorf("failed to render pages: %w", herrors.ImproveRenderErr(err)) + return fmt.Errorf("%v failed to render pages: %w", s.resolveDimensionNames(), herrors.ImproveRenderErr(err)) } return nil } @@ -129,6 +129,7 @@ func pageRenderer( defer wg.Done() for p := range pages { + if p.m.isStandalone() && !ctx.shouldRenderStandalonePage(p.Kind()) { continue } @@ -178,7 +179,7 @@ func pageRenderer( d = s.h.Sites } - if err := s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, "page "+p.Title(), targetPath, p, d, templ); err != nil { + if err := s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, targetPath, p, d, templ); err != nil { results <- err } @@ -256,7 +257,6 @@ func (s *Site) renderPaginator(p *pageState, templ *tplimpl.TemplInfo) error { if err := s.renderAndWritePage( &s.PathSpec.ProcessingStats.PaginatorPages, - p.Title(), targetPaths.TargetFilename, p, p, templ); err != nil { return err } @@ -268,9 +268,9 @@ func (s *Site) renderPaginator(p *pageState, templ *tplimpl.TemplInfo) error { // renderAliases renders shell pages that simply have a redirect in the header. func (s *Site) renderAliases() error { - w := &doctree.NodeShiftTreeWalker[contentNodeI]{ + w := &doctree.NodeShiftTreeWalker[contentNode]{ Tree: s.pageMap.treePages, - Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { + Handle: func(key string, n contentNode) (bool, error) { p := n.(*pageState) // We cannot alias a page that's not rendered. diff --git a/hugolib/site_sections.go b/hugolib/site_sections.go index 385f3f291..f238a885b 100644 --- a/hugolib/site_sections.go +++ b/hugolib/site_sections.go @@ -26,5 +26,5 @@ func (s *Site) Sections() page.Pages { // Home is a shortcut to the home page, equivalent to .Site.GetPage "home". func (s *Site) Home() page.Page { s.CheckReady() - return s.s.home + return s.home } diff --git a/hugolib/site_sections_test.go b/hugolib/site_sections_test.go index 0bf166092..357d3e77d 100644 --- a/hugolib/site_sections_test.go +++ b/hugolib/site_sections_test.go @@ -421,3 +421,23 @@ baseURL = "https://example.com/" b.AssertFileContent("public/docs/logs/sdk/index.html", "/docs/logs/sdk/|/docs/logs/|") b.AssertFileContent("public/docs/logs/sdk_exporters/stdout/index.html", "/docs/logs/sdk_exporters/stdout/|/docs/logs/|") } + +func TestNestedSectionsEmpty(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +baseURL = "https://example.com/" +-- content/a/b/c/_index.md -- +--- +title: "C" +--- +-- layouts/all.html -- +All: {{ .Title }}|{{ .Kind }}| +` + b := Test(t, files) + + b.AssertFileContent("public/a/index.html", "All: As|section|") + b.AssertFileExists("public/a/b/index.html", false) + b.AssertFileContent("public/a/b/c/index.html", "All: C|section|") +} diff --git a/hugolib/site_test.go b/hugolib/site_test.go index 7e3452b1c..877b8ac19 100644 --- a/hugolib/site_test.go +++ b/hugolib/site_test.go @@ -1019,7 +1019,7 @@ Content: {{ .Content }}| func checkLinkCase(site *Site, link string, currentPage page.Page, relative bool, outputFormat string, expected string, t *testing.T, i int) { t.Helper() - if out, err := site.refLink(link, currentPage, relative, outputFormat); err != nil || out != expected { + if out, err := site.siteRefLinker.refLink(link, currentPage, relative, outputFormat); err != nil || out != expected { t.Fatalf("[%d] Expected %q from %q to resolve to %q, got %q - error: %s", i, link, currentPage.Path(), expected, out, err) } } diff --git a/hugolib/sitesmatrix/dimensions.go b/hugolib/sitesmatrix/dimensions.go new file mode 100644 index 000000000..6bae609c9 --- /dev/null +++ b/hugolib/sitesmatrix/dimensions.go @@ -0,0 +1,282 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sitesmatrix + +import ( + "fmt" +) + +const ( + // Dimensions in the Hugo build matrix. + // These can be used as indices into the Vector type. + Language int = iota + Version + Role +) + +// Vector represents a site vector in the Hugo sites matrix from the three dimensions: +// Language, Version and Role. +// This is a fixed-size array for performance reasons (for one, it can be used as map key). +type Vector [3]int + +// Compare returns -1 if v1 is less than v2, 0 if they are equal, and 1 if v1 is greater than v2. +// This adds a implicit weighting to the dimensions, where the first dimension is the most important, +// but this is just used for sorting to get stable output. +func (v1 Vector) Compare(v2 Vector) int { + // note that a and b will never be equal. + minusOneOrOne := func(a, b int) int { + if a < b { + return -1 + } + return 1 + } + if v1[0] != v2[0] { + return minusOneOrOne(v1[0], v2[0]) + } + if v1[1] != v2[1] { + return minusOneOrOne(v1[1], v2[1]) + } + if v1[2] != v2[2] { + return minusOneOrOne(v1[2], v2[2]) + } + // They are equal. + return 0 +} + +// Distance returns the distance between v1 and v2 +// using the first dimension that is different. +func (v1 Vector) Distance(v2 Vector) int { + if v1[0] != v2[0] { + return v1[0] - v2[0] + } + if v1[1] != v2[1] { + return v1[1] - v2[1] + } + if v1[2] != v2[2] { + return v1[2] - v2[2] + } + return 0 +} + +// EuclideanDistanceSquared returns the Euclidean distance between two vectors as the sum of the squared differences. + +func (v1 Vector) HasVector(v2 Vector) bool { + return v1 == v2 +} + +func (v1 Vector) HasAnyVector(vp VectorProvider) bool { + if vp.LenVectors() == 0 { + return false + } + + return !vp.ForEachVector(func(v2 Vector) bool { + if v1 == v2 { + return false // stop iteration + } + return true // continue iteration + }) +} + +func (v1 Vector) LenVectors() int { + return 1 +} + +func (v1 Vector) VectorSample() Vector { + return v1 +} + +func (v1 Vector) EqualsVector(other VectorProvider) bool { + if other.LenVectors() != 1 { + return false + } + return other.VectorSample() == v1 +} + +func (v1 Vector) ForEachVector(yield func(v Vector) bool) bool { + return yield(v1) +} + +// Language returns the language dimension. +func (v1 Vector) Language() int { + return v1[Language] +} + +// Version returns the version dimension. +func (v1 Vector) Version() int { + return v1[Version] +} + +// IsFirst returns true if this is the first vector in the matrix, i.e. all dimensions are 0. +func (v1 Vector) IsFirst() bool { + return v1[Language] == 0 && v1[Version] == 0 && v1[Role] == 0 +} + +// Role returns the role dimension. +func (v1 Vector) Role() int { + return v1[Role] +} + +func (v1 Vector) Weight() int { + return 0 +} + +var _ ToVectorStoreProvider = Vectors{} + +type Vectors map[Vector]struct{} + +func (vs Vectors) ForEachVector(yield func(v Vector) bool) bool { + for v := range vs { + if !yield(v) { + return false + } + } + return true +} + +func (vs Vectors) ToVectorStore() VectorStore { + return newVectorStoreMapFromVectors(vs) +} + +// Sample returns one of the vectors in the set. +func (vs Vectors) Sample() Vector { + for v := range vs { + return v + } + panic("no vectors") +} + +type ( + VectorIterator interface { + // ForEachVector iterates over all vectors in the provider. + // It returns false if the iteration was stopped early. + ForEachVector(func(v Vector) bool) bool + } +) + +type VectorIteratorFunc func(func(v Vector) bool) bool + +func (f VectorIteratorFunc) ForEachVector(yield func(v Vector) bool) bool { + return f(yield) +} + +// Bools holds boolean values for each dimension in the Hugo build matrix. +type Bools [3]bool + +func (d Bools) Language() bool { + return d[Language] +} + +func (d Bools) Version() bool { + return d[Version] +} + +func (d Bools) Role() bool { + return d[Role] +} + +func (d Bools) IsZero() bool { + return !d[0] && !d[1] && !d[2] +} + +type VectorProvider interface { + VectorIterator + // HasVector returns true if the given vector is contained in the provider. + // Used for membership testing of files, resources and pages. + HasVector(HasAnyVectorv Vector) bool + + // HasAnyVector returns true if any of the vectors in the provider matches any of the vectors in v. + HasAnyVector(v VectorProvider) bool + + // LenVectors returns the number of vectors in the provider. + LenVectors() int + + // VectorSample returns one of the vectors in the provider, usually the first or the only one. + // This will panic if the provider is empty. + VectorSample() Vector + + // Equals returns true if this provider is equal to the other provider. + EqualsVector(other VectorProvider) bool +} + +type VectorStore interface { + VectorProvider + Complement(...VectorProvider) VectorStore + WithLanguageIndices(i int) VectorStore + HasLanguage(lang int) bool + HasVersion(version int) bool + HasRole(role int) bool + MustHash() uint64 + + // Used in tests. + KeysSorted() ([]int, []int, []int) + Vectors() []Vector +} + +type ToVectorStoreProvider interface { + ToVectorStore() VectorStore +} + +type weightedVectorStore struct { + VectorStore + weight int +} + +func (w weightedVectorStore) Weight() int { + return w.weight +} + +func NewWeightedVectorStore(vs VectorStore, weight int) VectorStore { + if vs == nil { + return nil + } + return weightedVectorStore{VectorStore: vs, weight: weight} +} + +// Dimension is a dimension in the Hugo build matrix. +type Dimension int8 + +func ParseDimension(s string) (int, error) { + switch s { + case "language": + return Language, nil + case "version": + return Version, nil + case "role": + return Role, nil + default: + return 0, fmt.Errorf("unknown dimension %q", s) + } +} + +func DimensionName(d int) string { + switch d { + case Language: + return "language" + case Version: + return "version" + case Role: + return "role" + default: + panic("unknown dimension") + } +} + +// Common information provided by all of language, version and role. +type DimensionInfo interface { + // The name. This corresponds to the key in the config, e.g. "en", "v1.2.3", "guest". + Name() string + + // Whether this is the default value for this dimension. + IsDefault() bool +} diff --git a/hugolib/sitesmatrix/dimensions_test.go b/hugolib/sitesmatrix/dimensions_test.go new file mode 100644 index 000000000..6d138a9d4 --- /dev/null +++ b/hugolib/sitesmatrix/dimensions_test.go @@ -0,0 +1,72 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sitesmatrix + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestDimensionsCompare(t *testing.T) { + c := qt.New(t) + + c.Assert(Vector{1, 2, 3}.Compare(Vector{1, 2, 8}), qt.Equals, -1) + c.Assert(Vector{1, 2, 3}.Compare(Vector{1, 2, 3}), qt.Equals, 0) + c.Assert(Vector{1, 2, 3}.Compare(Vector{1, 2, 0}), qt.Equals, 1) + c.Assert(Vector{1, 2, 3}.Compare(Vector{1, 0, 3}), qt.Equals, 1) + c.Assert(Vector{1, 2, 3}.Compare(Vector{0, 3, 2}), qt.Equals, 1) + c.Assert(Vector{1, 2, 3}.Compare(Vector{0, 0, 0}), qt.Equals, 1) + c.Assert(Vector{0, 0, 0}.Compare(Vector{1, 2, 3}), qt.Equals, -1) + c.Assert(Vector{0, 0, 0}.Compare(Vector{0, 0, 0}), qt.Equals, 0) + c.Assert(Vector{0, 0, 0}.Compare(Vector{1, 0, 0}), qt.Equals, -1) + c.Assert(Vector{0, 0, 0}.Compare(Vector{0, 1, 0}), qt.Equals, -1) + c.Assert(Vector{0, 0, 0}.Compare(Vector{0, 0, 1}), qt.Equals, -1) +} + +func TestDimensionsDistance(t *testing.T) { + c := qt.New(t) + + c.Assert(Vector{1, 2, 3}.Distance(Vector{1, 2, 8}), qt.Equals, -5) + c.Assert(Vector{1, 2, 3}.Distance(Vector{1, 2, 3}), qt.Equals, 0) + c.Assert(Vector{1, 2, 3}.Distance(Vector{1, 2, 0}), qt.Equals, 3) + c.Assert(Vector{1, 2, 3}.Distance(Vector{1, 0, 3}), qt.Equals, 2) + c.Assert(Vector{1, 2, 3}.Distance(Vector{0, 3, 2}), qt.Equals, 1) +} + +func BenchmarkCompare(b *testing.B) { + b.Run("Equal", func(b *testing.B) { + v1 := Vector{1, 2, 3} + v2 := Vector{1, 2, 3} + for range b.N { + v1.Compare(v2) + } + }) + + b.Run("First different", func(b *testing.B) { + v1 := Vector{1, 2, 3} + v2 := Vector{2, 2, 3} + for range b.N { + v1.Compare(v2) + } + }) + + b.Run("Last different", func(b *testing.B) { + v1 := Vector{1, 2, 3} + v2 := Vector{1, 2, 4} + for range b.N { + v1.Compare(v2) + } + }) +} diff --git a/hugolib/sitesmatrix/sitematrix_integration_test.go b/hugolib/sitesmatrix/sitematrix_integration_test.go new file mode 100644 index 000000000..9408a701d --- /dev/null +++ b/hugolib/sitesmatrix/sitematrix_integration_test.go @@ -0,0 +1,1138 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sitesmatrix_test + +import ( + "fmt" + "strings" + "testing" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/hugolib" +) + +func TestPageRotate(t *testing.T) { + t.Parallel() + files := ` +-- hugo.toml -- +baseURL = "https://example.org/" +defaultContentVersion = "v4.0.0" +defaultContentVersionInSubdir = true +defaultContentRoleInSubdir = true +defaultContentRole = "guest" +defaultContentLanguage = "en" +defaultContentLanguageInSubdir = true +disableKinds = ["taxonomy", "term", "rss", "sitemap"] + +[cascade] +versions = ["v2**"] + +[languages] +[languages.en] +weight = 1 +[languages.nn] +weight = 2 +[roles] +[roles.guest] +weight = 300 +[roles.member] +weight = 200 +[versions] +[versions."v2.0.0"] +[versions."v1.2.3"] +[versions."v2.1.0"] +[versions."v3.0.0"] +[versions."v4.0.0"] +-- content/_index.en.md -- +--- +title: "Home" +roles: ["**"] +versions: ["**"] +--- +-- content/_index.nn.md -- +--- +title: "Heim" +roles: ["**"] +versions: ["**"] +--- +-- content/memberonlypost.md -- +--- +title: "Member Only" +roles: ["member"] +languages: ["**"] +--- +Member content. +-- content/publicpost.md -- +--- +title: "Public" +versions: ["v1.2.3", "v2.**", "! v2.1.*"] +versionDelegees: ["v3**"] +--- +Users with guest role will see this. +-- content/v3publicpost.md -- +--- +title: "Public v3" +versions: ["v3**"] +languages: ["**"] +--- +Users with guest role will see this. +-- layouts/all.html -- +Rotate(language): {{ with .Rotate "language" }}{{ range . }}{{ template "printp" . }}|{{ end }}{{ end }}$ +Rotate(version): {{ with .Rotate "version" }}{{ range . }}{{ template "printp" . }}|{{ end }}{{ end }}$ +Rotate(role): {{ with .Rotate "role" }}{{ range . }}{{ template "printp" . }}|{{ end }}{{ end }}$ +.Site.Dimension.language {{ (.Site.Dimension "language").Name }}| +.Site.Dimension.version: {{ (.Site.Dimension "version").Name }}| +.Site.Dimension.role: {{ (.Site.Dimension "role").Name }}| +{{ define "printp" }}{{ .RelPermalink }}:{{ with .Site }}{{ template "prints" . }}{{ end }}{{ end }} +{{ define "prints" }}/l:{{ .Language.Name }}/v:{{ .Version.Name }}/r:{{ .Role.Name }}{{ end }} + + +` + + for range 3 { + b := hugolib.Test(t, files) + + b.AssertFileContent("public/guest/v3.0.0/en/index.html", + "Rotate(language): /guest/v3.0.0/en/:/l:en/v:v3.0.0/r:guest|/guest/v3.0.0/nn/:/l:nn/v:v3.0.0/r:guest|$", + "Rotate(version): /guest/v4.0.0/en/:/l:en/v:v4.0.0/r:guest|/guest/v3.0.0/en/:/l:en/v:v3.0.0/r:guest|/guest/v2.1.0/en/:/l:en/v:v2.1.0/r:guest|/guest/v2.0.0/en/:/l:en/v:v2.0.0/r:guest|/guest/v1.2.3/en/:/l:en/v:v1.2.3/r:guest", + "Rotate(role): /member/v3.0.0/en/:/l:en/v:v3.0.0/r:member|/guest/v3.0.0/en/:/l:en/v:v3.0.0/r:guest|$", + ".Site.Dimension.language en|", + ".Site.Dimension.version: v3.0.0|", + ".Site.Dimension.role: guest|", + ) + + } +} + +func TestFileMountSitesMatrix(t *testing.T) { + filesTemplate := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "rss", "sitemap", "section"] +defaultContentLanguage = "en" +defaultContentLanguageInSubDir = true +defaultCOntentVersionInSubDir = true +[versions] +[versions."v1.2.3"] +[versions."v2.0.0"] +[languages] +[languages.en] +weight = 1 +[languages.nn] +weight = 2 +[module] +[[module.mounts]] +source = 'content/en' +target = 'content' +DIMSEN +[[module.mounts]] +source = 'content/nn' +target = 'content' +DIMSNN +[[module.mounts]] +source = 'content/all' +target = 'content' +[module.mounts.sites.matrix] +languages = ["**"] +versions = ["**"] +-- content/en/p1/index.md -- +--- +title: "Title English" +--- +-- content/en/p1/mytext.txt -- +Text English +-- content/nn/p1/index.md -- +--- +title: "Tittel Nynorsk" +--- +-- content/all/p2/index.md -- +--- +title: "p2 all" +--- +-- content/nn/p1/mytext.txt -- +Tekst Nynorsk +-- layouts/all.html -- +{{ $mytext := .Resources.Get "mytext.txt" }} +{{ .Title }}|{{ with $mytext }}{{ .Content | safeHTML }}{{ end }}| +site.GetPage p2: {{ with .Site.GetPage "p2" }}{{ .Title }}|{{ end }}$ +site.GetPage p1: {{ with .Site.GetPage "p1" }}{{ .Title }}|{{ end }}$ + +` + + testOne := func(t *testing.T, name, files string) { + t.Run(name, func(t *testing.T) { + t.Parallel() + + b := hugolib.Test(t, files) + + nn20 := b.SiteHelper("nn", "v2.0.0", "") + b.Assert(nn20.PageHelper("/p2").MatrixFromFile()["matrix"], + qt.DeepEquals, + map[string][]string{"languages": {"en", "nn"}, "roles": {"guest"}, "versions": {"v2.0.0", "v1.2.3"}}) + + b.AssertFileContent("public/v2.0.0/nn/p1/index.html", "Tittel Nynorsk", "Tekst Nynorsk", "site.GetPage p1: Tittel Nynorsk|") + b.AssertFileContent("public/v2.0.0/nn/p2/index.html", "p2 all||", "site.GetPage p2: p2 all", "site.GetPage p1: Tittel Nynorsk|") + b.AssertFileContent("public/v2.0.0/nn/p2/index.html", "p2 all||", "site.GetPage p1: Tittel Nynorsk|$") + b.AssertFileContent("public/v1.2.3/en/p2/index.html", "p2 all||", "site.GetPage p2: p2 all") + }) + } + + // Format from v0.148.0: + dims := `[module.mounts.sites.matrix] +languages = ["en"] +versions = ["v1**"] +` + files := strings.Replace(filesTemplate, "DIMSEN", dims, 1) + dims = strings.Replace(dims, `["en"]`, `["nn"]`, 1) + dims = strings.Replace(dims, `["v1**"]`, `["v2**"]`, 1) + files = strings.Replace(files, "DIMSNN", dims, 1) + testOne(t, "new", files) + + // Old format: + files = strings.Replace(filesTemplate, "DIMSEN", `lang = "en"`, 1) + files = strings.Replace(files, "DIMSNN", `lang = "nn"`, 1) + testOne(t, "old", files) +} + +func TestSpecificMountShouldAlwaysWin(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ["rss", "sitemap", "section"] +defaultContentLanguage = "en" +defaultContentLanguageInSubDir = true +defaultCOntentVersionInSubDir = true +defaultContentVersion = "v2.0.0" +[taxonomies] +tag = "tags" +[languages] +[languages.en] +weight = 1 +[languages.nn] +weight = 2 +[versions] +[versions."v1.2.3"] +[versions."v2.0.0"] +[module] +[[module.mounts]] +source = 'content/nn' +target = 'content' +[module.mounts.sites.matrix] +languages = ["nn"] +versions = ["v1.**"] +[[module.mounts]] +source = 'content/en' +target = 'content' +[module.mounts.sites.matrix] +languages = ["**"] +versions = ["**"] +-- content/en/_index.md -- +--- +title: "English Home" +tags: ["tag1"] +--- +-- content/en/p1.md -- +--- +title: "English p1" +--- +-- content/nn/_index.md -- +--- +title: "Nynorsk Heim" +tags: ["tag2"] +--- +-- layouts/all.html -- +title: {{ .Title }}| +tags: {{ range $term, $taxonomy := .Site.Taxonomies.tags }}{{ $term }}: {{ range $taxonomy.Pages }}{{ .Title }}: {{ .RelPermalink}}|{{ end }}{{ end }}$ +` + + for range 2 { + b := hugolib.Test(t, files) + + // b.AssertPublishDir("asdf") + b.AssertFileContent("public/v1.2.3/nn/index.html", "title: Nynorsk Heim|", "tags: tag2: Nynorsk Heim: /v1.2.3/nn/|$") + b.AssertFileContent("public/v2.0.0/en/index.html", "title: English Home|", "tags: tag1: English Home: /v2.0.0/en/|$") + b.AssertFileContent("public/v2.0.0/nn/index.html", "title: English Home|", "tags: tag1: English Home: /v2.0.0/nn/|$") // v2.0.0 is only in English. + b.AssertFileContent("public/v1.2.3/en/index.html", "title: English Home|") + } +} + +const filesVariationsSitesMatrixBase = ` +-- hugo.toml -- +baseURL = "https://example.org/" +disableKinds = ["rss", "sitemap", "section"] +defaultContentLanguage = "en" +defaultContentLanguageInSubDir = true +defaultCOntentVersionInSubDir = true +defaultContentVersion = "v2.0.0" +defaultContentRole = "guest" +defaultContentRoleInSubDir = true +[taxonomies] +tag = "tags" +[languages] +[languages.en] +weight = 1 +[languages.nn] +weight = 2 +[versions] +[versions."v1.2.3"] +[versions."v1.4.0"] +[versions."v2.0.0"] +[roles] +[roles.guest] +weight = 300 +[roles.member] +weight = 200 +[module] +[[module.mounts]] +source = 'content/nn' +target = 'content' +[module.mounts.sites.matrix] +languages = ["nn"] +versions = ["v1.2.*"] +[[module.mounts]] +source = 'content/en' +target = 'content' +[module.mounts.sites.matrix] +languages = ["**"] +versions = ["**"] +[[module.mounts]] +source = 'content/other' +target = 'content' +-- content/en/_index.md -- +--- +title: "English Home" +tags: ["tag1"] +--- + +Ref home: {{< ref "/" >}}| +-- content/en/p1.md -- +--- +title: "English p1" +--- +-- content/nn/_index.md -- +--- +title: "Nynorsk Heim" +tags: ["tag2"] +--- + +Ref home: {{< ref "/" >}}| +` + +const filesVariationsSitesMatrix = filesVariationsSitesMatrixBase + ` +-- layouts/all.html -- +title: {{ .Title }}| +tags: {{ range $term, $taxonomy := .Site.Taxonomies.tags }}{{ $term }}: {{ range $taxonomy.Pages }}{{ .Title }}: {{ .RelPermalink}}|{{ end }}{{ end }}$ +.Language.IsDefault: {{ with .Rotate "language" }}{{ range . }}{{ .RelPermalink }}: {{ with .Site.Language }}{{ .Name}}: {{ .IsDefault }}|{{ end }}{{ end }}{{ end }}$ +.Version.IsDefault: {{ with .Rotate "version" }}{{ range . }}{{ .RelPermalink }}: {{ with .Site.Version }}{{ .Name}}: {{ .IsDefault }}|{{ end }}{{ end }}{{ end }}$ +.Role.IsDefault: {{ with .Rotate "role" }}{{ range . }}{{ .RelPermalink }}: {{ with .Site.Role }}{{ .Name}}: {{ .IsDefault }}|{{ end }}{{ end }}{{ end }}$ +` + +func TestFrontMatterSitesMatrix(t *testing.T) { + t.Parallel() + + files := filesVariationsSitesMatrix + + files += ` +-- content/other/p2.md -- +--- +title: "NN p2" +sites: + matrix: + languages: + - nn + versions: + - v1.2.3 +--- +` + b := hugolib.Test(t, files) + b.AssertFileContent("public/guest/v1.2.3/nn/p2/index.html", "title: NN p2|") +} + +func TestFrontMatterSitesMatrixShouldWin(t *testing.T) { + t.Parallel() + + files := filesVariationsSitesMatrix + + // nn mount config is nn, v1.2.*. + files += ` +-- content/nn/p2.md -- +--- +title: "EN p2" +sites: + matrix: + languages: + - en + versions: + - v1.4.* +--- +` + b := hugolib.Test(t, files) + b.AssertFileContent("public/guest/v1.4.0/en/p2/index.html", "title: EN p2|") +} + +func TestGetPageAndRef(t *testing.T) { + t.Parallel() + + files := filesVariationsSitesMatrixBase + ` +-- layouts/all.html -- +Title: {{ .Title }}| +Home: {{ with .Site.GetPage "/" }}{{ with .Site }}Language: {{ .Language.Name }}|Version: {{ .Version.Name }}|Role: {{ .Role.Name }}{{ end }}|{{ end }}$ +Content: {{ .Content }}$ +` + + b := hugolib.Test(t, files) + b.AssertFileContent( + "public/guest/v2.0.0/en/index.html", "Home: Language: en|Version: v2.0.0|Role: guest|$", + "Ref home: https://example.org/guest/v2.0.0/en/|", + ) + b.AssertFileContent( + "public/member/v1.4.0/nn/index.html", "Home: Language: nn|Version: v1.4.0|Role: member|$", + ) + + b.AssertFileContent( + "public/guest/v1.2.3/nn/index.html", "Home: Language: nn|Version: v1.2.3|Role: guest|$", + "Ref home: https://example.org/guest/v1.2.3/nn/|", + ) +} + +func TestFrontMatterSitesMatrixShouldBeMergedWithMount(t *testing.T) { + t.Parallel() + + files := filesVariationsSitesMatrix + + // nn mount config is nn, v1.2.*. + // This changes only the language, not the version. + files += ` +-- content/nn/p2.md -- +--- +title: "EN p2" +sites: + matrix: + languages: + - en +--- +` + b := hugolib.Test(t, files) + b.AssertFileContent("public/guest/v1.2.3/en/p2/index.html", "title: EN p2|") +} + +func TestCascadeMatrixInFrontMatter(t *testing.T) { + t.Parallel() + files := ` +-- hugo.toml -- +disableKinds = ["rss", "sitemap", "taxonomy", "term"] +defaultContentLanguage = "en" +defaultContentLanguageInSubDir = true +[languages] +[languages.en] +weight = 1 +[languages.nn] +weight = 2 +[languages.sv] +weight = 3 +-- content/_index.md -- ++++ +title = "English home" +[cascade] +[cascade.params] +p1 = "p1cascade" +[cascade.target.sites.matrix] +languages = ["en"] ++++ +-- content/_index.nn.md -- ++++ +title = "Scandinavian home" +[sites.matrix] +languages = "{nn,sv}" +[cascade] +[cascade.params] +p1 = "p1cascadescandinavian" +[cascade.sites.matrix] +languages = "{nn,sv}" +[cascade.target] +path = "**scandinavian**" ++++ +-- content/mysection/_index.md -- ++++ +title = "My section" +[cascade.target.sites.matrix] +languages = "**" ++++ +-- content/mysection/p1.md -- ++++ +title = "English p1" ++++ +-- content/mysection/scandinavianpages/p1.md -- ++++ +title = "Scandinavian p1" ++++ +-- layouts/all.html -- +{{ .Title }}|{{ .Site.Language.Name }}|{{ .Site.Version.Name }}|p1: {{ .Params.p1 }}| +` + b := hugolib.Test(t, files) + b.AssertFileContent("public/en/mysection/p1/index.html", "English p1|en|v1|p1: p1cascade|") + b.AssertFileContent("public/sv/mysection/scandinavianpages/p1/index.html", "Scandinavian p1|sv|v1|p1: p1cascadescandinavian|") +} + +func TestCascadeMatrixConfigPerLanguage(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ["rss", "sitemap", "taxonomy", "term"] +defaultContentLanguage = "en" +defaultContentLanguageInSubDir = true +[cascade.params] +p1 = "p1cascadeall" +p2 = "p2cascadeall" +[languages] +[languages.en] +weight = 1 +[languages.en.cascade.params] +p1 = "p1cascadeen" +[languages.nn] +weight = 2 +[languages.nn.cascade.params] +p1 = "p1cascadenn" +-- content/_index.md -- +--- +title: "English home" +--- +-- content/mysection/p1.md -- +--- +title: "English p1" +--- +-- content/mysection/p1.nn.md -- +--- +title: "Nynorsk p1" +--- +-- layouts/all.html -- +{{ .Title }}|{{ .Site.Language.Name }}|{{ .Site.Version.Name }}|p1: {{ .Params.p1 }}|p2: {{ .Params.p2 }}| + +` + b := hugolib.Test(t, files) + b.AssertFileContent("public/en/mysection/p1/index.html", "English p1|en|v1|p1: p1cascadeen|p2: p2cascadeall|") + b.AssertFileContent("public/nn/mysection/p1/index.html", "Nynorsk p1|nn|v1|p1: p1cascadenn|p2: p2cascadeall|") +} + +func TestCascadeMatrixNoHomeContent(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ["rss", "sitemap", "taxonomy", "term"] +defaultContentLanguage = "en" +defaultContentLanguageInSubDir = true +[cascade.params] +p1 = "p1cascadeall" +p2 = "p2cascadeall" +-- content/mysection/p1.md -- +-- layouts/all.html -- +{{ .Title }}|{{ .Kind }}|{{ .Site.Language.Name }}|{{ .Site.Version.Name }}|p1: {{ .Params.p1 }}|p2: {{ .Params.p2 }}| +` + + b := hugolib.Test(t, files) + b.AssertFileContent("public/en/index.html", "|home|en|v1|p1: p1cascadeall|p2: p2cascadeall|") + b.AssertFileContent("public/en/mysection/index.html", "Mysections|section|en|v1|p1: p1cascadeall|p2: p2cascadeall|") + b.AssertFileContent("public/en/mysection/p1/index.html", "|page|en|v1|p1: p1cascadeall|p2: p2cascadeall|") +} + +func TestMountCascadeFrontMatterSitesMatrixAndComplementsShouldBeMerged(t *testing.T) { + t.Parallel() + + // Pick language from mount, role from cascade and version from front matter. + files := ` +-- hugo.toml -- +disableKinds = ["rss", "sitemap", "section", "taxonomy", "term"] +defaultContentLanguage = "en" +defaultContentLanguageInSubDir = true +defaultCOntentVersionInSubDir = true +defaultContentVersion = "v1.2.3" +defaultContentRole = "guest" +defaultContentRoleInSubDir = true + +[languages] +[languages.en] +weight = 1 +[languages.nn] +weight = 2 + +[versions] +[versions."v1.2.3"] +[versions."v1.4.0"] +[versions."v2.0.0"] + +[roles] +[roles.guest] +weight = 300 +[roles.member] +weight = 200 + +[[module.mounts]] +source = 'content/other' +target = 'content' +[module.mounts.sites.matrix] +languages = ["nn"] # used. +versions = ["v1.2.*"] # not used. +roles = ["guest"] # not used. +[module.mounts.sites.complements] +languages = ["en"] # used. +versions = ["v1.4.*"] # not used. +roles = ["member"] # not used. + +[cascade.sites.matrix] +roles = ["member"] # used +versions = ["v1.2.*"] # not used. +[cascade.sites.complements] +roles = ["guest"] # used +versions = ["v2**"] # not used. + +-- content/other/p2.md -- ++++ +title = "NN p2" +[sites.matrix] +versions = ["v1.2.*","v1.4.*"] +[sites.complements] +versions = ["v2.*.*"] ++++ +-- content/other/p3.md -- ++++ +title = "NN p3" +[sites.matrix] +versions = "v1.4.*" ++++ +-- layouts/all.html -- +All. {{ site.Language.Name }}|{{ site.Version.Name }}|{{ site.Role.Name }} + +` + + b := hugolib.Test(t, files) + b.AssertFileContent("public/member/v1.4.0/nn/p2/index.html", "All.") + s := b.SiteHelper("nn", "v1.4.0", "member") + p2 := s.PageHelper("/p2") + s.Assert(p2.MatrixFromPageConfig(), qt.DeepEquals, + map[string]map[string][]string{ + "complements": { + "languages": {"en"}, + "roles": {"guest"}, + "versions": {"v2.0.0"}, + }, + "matrix": { + "languages": {"nn"}, + "roles": {"member"}, + "versions": {"v1.4.0", "v1.2.3"}, + }, + }, + ) + + p3 := s.PageHelper("/p3") + s.Assert(p3.MatrixFromPageConfig(), qt.DeepEquals, map[string]map[string][]string{ + "complements": { + "languages": {"en"}, + "roles": {"guest"}, + "versions": {"v2.0.0"}, + }, + "matrix": { + "languages": {"nn"}, + "roles": {"member"}, + "versions": {"v1.4.0"}, + }, + }) +} + +func TestLanguageVersionRoleIsDefault(t *testing.T) { + files := filesVariationsSitesMatrix + + b := hugolib.Test(t, files) + b.AssertFileContent("public/guest/v2.0.0/en/index.html", + ".Language.IsDefault: /guest/v2.0.0/en/: en: true|/guest/v2.0.0/nn/: nn: false|$", + ".Version.IsDefault: /guest/v2.0.0/en/: v2.0.0: true|/guest/v1.4.0/en/: v1.4.0: false|/guest/v1.2.3/en/: v1.2.3: false|$", + ".Role.IsDefault: /member/v2.0.0/en/: member: false|/guest/v2.0.0/en/: guest: true|$", + ) +} + +func TestModuleMountsLanguageOverlap(t *testing.T) { + files := ` +-- hugo.toml -- +disableKinds = ["rss", "sitemap", "section", "taxonomy", "term"] +theme = "mytheme" +defaultContentLanguage = "en" +defaultContentLanguageInSubDir = true +[languages] +[languages.en] +weight = 1 +[languages.nn] +weight = 2 +[[module.mounts]] +source = 'content' +target = 'content' +[module.mounts.sites.matrix] +languages = "en" +-- content/p1.md -- +--- +title: "Project p1" +--- +-- themes/mytheme/hugo.toml -- + +[[module.mounts]] +source = 'content' +target = 'content' +[module.mounts.sites.matrix] +languages = "**" +-- themes/mytheme/content/p1.md -- +--- +title: "Theme p1" +--- +-- themes/mytheme/content/p2.md -- +--- +title: "Theme p1" +--- +-- layouts/all.html -- +{{ .Title }}| +` + + b := hugolib.Test(t, files) + + sEn := b.SiteHelper("en", "", "") + sNN := b.SiteHelper("nn", "", "guest") + + b.Assert(sEn.PageHelper("/p1").MatrixFromPageConfig()["matrix"], qt.DeepEquals, map[string][]string{"languages": {"en"}, "roles": {"guest"}, "versions": {"v1"}}) + b.Assert(sEn.PageHelper("/p2").MatrixFromPageConfig()["matrix"], qt.DeepEquals, map[string][]string{ + "languages": {"en", "nn"}, + "roles": {"guest"}, + "versions": {"v1"}, + }) + + p1NN := sNN.PageHelper("/p1") + p2NN := sNN.PageHelper("/p2") + + b.Assert(p2NN.MatrixFromPageConfig()["matrix"], qt.DeepEquals, map[string][]string{ + "languages": {"en", "nn"}, + "roles": {"guest"}, + "versions": {"v1"}, + }) + + b.Assert(p1NN.MatrixFromFile()["matrix"], qt.DeepEquals, map[string][]string{ + "languages": {"nn"}, // It's defined as ** in the mount. + "roles": {"guest"}, + "versions": {"v1"}, + }) +} + +func TestMountLanguageComplements(t *testing.T) { + files := ` +-- hugo.toml -- +disableKinds = ["rss", "sitemap", "section", "taxonomy", "term"] +defaultContentLanguage = "en" +defaultContentLanguageInSubDir = true +[languages] +[languages.en] +weight = 1 +[languages.sv] +weight = 2 +[languages.da] +weight = 3 +[[module.mounts]] +source = 'content/en' +target = 'content' +[module.mounts.sites.matrix] +languages = "en" +[module.mounts.sites.complements] +languages = "{sv,da}" +[[module.mounts]] +source = 'content/sv' +target = 'content' +[module.mounts.sites.matrix] +languages = "sv" +-- content/en/p1.md -- +--- +title: "English p1" +--- +-- content/en/p2.md -- +--- +title: "English p2" +--- +-- content/sv/p1.md -- +--- +title: "Swedish p1" +--- +-- layouts/home.html -- +RegularPagesRecursive: {{ range .RegularPagesRecursive }}{{ .Title }}|{{ .RelPermalink }}{{ end }}$ +-- layouts/all.html -- +{{ .Title }}|{{ .Site.Language.Name }}| +` + + b := hugolib.Test(t, files) + b.AssertFileContent("public/en/index.html", "RegularPagesRecursive: English p1|/en/p1/English p2|/en/p2/$") + b.AssertFileContent("public/sv/index.html", "RegularPagesRecursive: English p2|/en/p2/Swedish p1|/sv/p1/$") + b.AssertFileContent("public/da/index.html", "RegularPagesRecursive: English p1|/en/p1/English p2|/en/p2/$") +} + +func TestContentAdapterSitesMatrixResources(t *testing.T) { + t.Parallel() + files := ` +-- hugo.toml -- +disableKinds = ["rss", "sitemap", "section", "taxonomy", "term"] +defaultContentLanguage = "en" +defaultContentLanguageInSubDir = true +[languages] +[languages.en] +weight = 1 +[languages.nn] +weight = 2 +[languages.sv] +weight = 3 +[languages.da] +weight = 4 +-- content/_content.gotmpl -- +{{ $scandinavia := dict "languages" "{nn,sv}" }} +{{ $en := dict "languages" "en" }} +{{ $da := dict "languages" "da" }} +{{ $contentMarkdown := dict "value" "**Hello World**" "mediaType" "text/markdown" }} +{{ $contentTextEnglish := dict "value" "Hello World" "mediaType" "text/plain" }} +{{ $contentTextNorsk := dict "value" "Hallo verd" "mediaType" "text/plain" }} +{{ .AddPage (dict "path" "p1" "title" "P1 en" "content" $contentMarkdown "sites" (dict "matrix" $en )) }} +{{ .AddPage (dict "path" "p1" "title" "P1 scandinavia" "content" $contentMarkdown "sites" (dict "matrix" $scandinavia )) }} +{{ .AddPage (dict "path" "p1" "title" "P1 da" "content" $contentMarkdown "sites" (dict "matrix" $da )) }} +{{ .AddResource (dict "path" "p1/hello.txt" "title" "Hello en" "content" $contentTextEnglish "sites" (dict "matrix" $en )) }} +{{ .AddResource (dict "path" "p1/hello.txt" "title" "Hello scandinavia" "content" $contentTextNorsk "sites" (dict "matrix" $scandinavia "complements" $da )) }} +-- layouts/all.html -- +len .Resources: {{ len .Resources}}| +{{ $hello := .Resources.Get "hello.txt" }} +All. {{ .Title }}|Hello: {{ with $hello }}{{ .RelPermalink }}|{{ .Content | safeHTML }}{{ end }}| +` + + b := hugolib.Test(t, files) + b.AssertFileContent("public/en/p1/index.html", "Hello: /en/p1/hello.txt|Hello World|") + b.AssertFileContent("public/nn/p1/index.html", "P1 scandinavia|", "/nn/p1/hello.txt|Hallo verd|") + b.AssertFileContent("public/sv/p1/index.html", "P1 scandinavia|", "/sv/p1/hello.txt|Hallo verd|") + b.AssertFileContent("public/da/p1/index.html", "P1 da|", "/sv/p1/hello.txt|Hallo verd|") // Because it's closest of the Scandinavian complements. +} + +func TestContentAdapterSitesMatrixContentPageSwitchLanguage(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ["rss", "sitemap", "section", "taxonomy", "term"] +defaultContentLanguage = "en" +defaultContentLanguageInSubDir = true +disableLiveReload = true +[languages] +[languages.en] +weight = 1 +[languages.sv] +weight = 2 +[languages.da] +weight = 3 +[languages.no] +weight = 4 +-- content/_content.gotmpl -- +{{ $contentMarkdown := dict "value" "**Hello World**" "mediaType" "text/markdown" }} +{{ $en := dict "languages" "en" }} +{{ $sv := dict "languages" "sv" }} +{{ $da := dict "languages" "da" }} +{{ $no := dict "languages" "no" }} +{{ .AddPage (dict "path" "p1" "title" "P1-1" "content" $contentMarkdown "sites" (dict "matrix" $en )) }} +{{ .AddPage (dict "path" "p1" "title" "P1-2" "content" $contentMarkdown "sites" (dict "matrix" $sv )) }} +{{ .AddPage (dict "path" "p1" "title" "P1-3" "content" $contentMarkdown "sites" (dict "matrix" $da )) }} +-- layouts/all.html -- +All. {{ .Title }}|{{ .Site.Language.Name }}| +` + + b := hugolib.TestRunning(t, files) + + b.AssertFileExists("public/no/p1/index.html", false) + b.AssertFileExists("public/sv/p1/index.html", true) + + b.RemovePublishDir() + b.AssertFileExists("public/sv/p1/index.html", false) + + b.EditFileReplaceAll("content/_content.gotmpl", `"sv"`, `"no"`).Build() + + b.AssertFileExists("public/no/p1/index.html", true) + b.AssertFileExists("public/sv/p1/index.html", false) +} + +func TestContentAdapterSitesMatrixContentResourceSwitchLanguage(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ["rss", "sitemap", "section", "taxonomy", "term"] +disableLiveReload = true +defaultContentLanguage = "en" +defaultContentLanguageInSubDir = true +[languages] +[languages.en] +weight = 1 +[languages.sv] +weight = 2 +[languages.da] +weight = 3 +[languages.no] +weight = 4 +-- content/_content.gotmpl -- +{{ $contentTextEnglish := dict "value" "Hello World" "mediaType" "text/plain" }} +{{ $contentTextSwedish := dict "value" "Hei världen" "mediaType" "text/plain" }} +{{ $contentTextDanish := dict "value" "Hej verden" "mediaType" "text/plain" }} + +{{ $en := dict "languages" "en" }} +{{ $sv := dict "languages" "sv" }} +{{ $da := dict "languages" "da" }} +{{ .AddResource (dict "path" "hello.txt" "title" "Hello" "content" $contentTextEnglish "sites" (dict "matrix" $en )) }} +{{ .AddResource (dict "path" "hello.txt" "title" "Hello" "content" $contentTextSwedish "sites" (dict "matrix" $sv )) }} +{{ .AddResource (dict "path" "hello.txt" "title" "Hello" "content" $contentTextDanish "sites" (dict "matrix" $da )) }} + +-- layouts/all.html -- +{{ $hello := .Resources.Get "hello.txt" }} +All. {{ .Title }}|{{ .Site.Language.Name }}|hello: {{ with $hello }}{{ .RelPermalink }}: {{ .Content }}|{{ end }}| +` + + b := hugolib.TestRunning(t, files) + + b.AssertFileContent("public/en/index.html", "en|hello: /en/hello.txt: Hello World|") + b.AssertFileContent("public/sv/index.html", "sv|hello: /sv/hello.txt: Hei världen|") + b.AssertFileContent("public/no/index.html", "no|hello: |") + + b.EditFileReplaceAll("content/_content.gotmpl", `"sv"`, `"no"`).Build() + + b.AssertFileContent("public/no/index.html", "no|hello: /no/hello.txt: Hei världen|") + b.AssertFileContent("public/en/index.html", "en|hello: /en/hello.txt: Hello World|") + b.AssertFileContent("public/sv/index.html", "|sv|hello: |") +} + +const filesContentAdapterSitesMatrixFromConfig = ` +-- hugo.toml -- +disableKinds = ["rss", "sitemap", "section", "taxonomy", "term"] +defaultContentLanguage = "en" +defaultContentLanguageInSubDir = true +defaultContentRole = "enrole" +defaultContentRoleInSubDir = true +[languages] +[languages.en] +weight = 1 +[languages.en.cascade.sites.matrix] +roles = ["enrole"] +[languages.nn] +weight = 2 +[languages.nn.cascade.sites.matrix] +roles = ["nnrole"] +[roles] +[roles.enrole] +weight = 200 +[roles.nnrole] +weight = 100 +-- layouts/all.html -- +All. {{ .Title }}|{{ .Site.Language.Name }}|{{ .Site.Role.Name }}| +Resources: {{ range .Resources }}{{ .Title }}|{{ end }}$ + +` + +func TestContentAdapterSitesMatrixSitesMatrixFromConfig(t *testing.T) { + t.Parallel() + + files := filesContentAdapterSitesMatrixFromConfig + ` +-- content/_content.gotmpl -- +{{ $title := printf "P1 %s:%s" .Site.Language.Name .Site.Role.Name }} +{{ .AddPage (dict "path" "p1" "title" $title ) }} +` + + b := hugolib.Test(t, files) + + b.AssertFileContent("public/enrole/en/p1/index.html", "P1 en:enrole|en|enrole|") + b.AssertPublishDir("! nn/p1") +} + +func TestContentAdapterSitesMatrixSitesMatrixFromConfigEnableAllLanguages(t *testing.T) { + t.Parallel() + + files := filesContentAdapterSitesMatrixFromConfig + ` +-- content/_content.gotmpl -- +{{ .EnableAllLanguages }} +{{ $title := printf "P1 %s:%s" .Site.Language.Name .Site.Role.Name }} +{{ .AddPage (dict "path" "p1" "title" $title ) }} +{{ $.AddResource (dict "path" "p1/mytext.txt" "title" $title "content" (dict "value" $title) )}} +` + + b := hugolib.Test(t, files) + + b.AssertFileContent("public/enrole/en/p1/index.html", "P1 en:enrole|en|enrole|", "Resources: P1 en:enrole|$") + b.AssertFileContent("public/nnrole/nn/p1/index.html", "P1 nn:nnrole|nn|nnrole|", "Resources: P1 nn:nnrole|$") +} + +func TestSitesMatrixCustomContentFilenameIdentifier(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ["rss", "sitemap", "section", "taxonomy", "term"] +defaultContentLanguage = "en" +defaultContentLanguageInSubDir = true +[languages] +[languages.en] +weight = 1 +[languages.nn] +weight = 2 +[languages.sv] +weight = 3 +[languages.da] +weight = 4 +[languages.de] +weight = 5 +-- content/p1.en.md -- +--- +title: "P1 en" +--- +-- content/p1._scandinavian_.md -- +--- +title: "P1 scandinavian" +sites: + matrix: + languages: "{nn,sv,da}" +--- +-- content/p1.de.md -- +--- +title: "P1 de" +--- +-- layouts/all.html -- +All.{{ .Title }}| +` + + b := hugolib.Test(t, files) + + b.AssertFileContent("public/en/p1/index.html", "All.P1 en|") + b.AssertFileContent("public/nn/p1/index.html", "All.P1 scandinavian|") + b.AssertFileContent("public/sv/p1/index.html", "All.P1 scandinavian|") + b.AssertFileContent("public/da/p1/index.html", "All.P1 scandinavian|") + b.AssertFileContent("public/de/p1/index.html", "All.P1 de|") +} + +func newSitesMatrixContentBenchmarkBuilder(t testing.TB, numPages int, skipRender, multipleDimensions bool) *hugolib.IntegrationTestBuilder { + files := ` +-- hugo.toml -- +disableKinds = ["rss", "sitemap", "section", "taxonomy", "term"] +defaultContentLanguage = "en" +defaultContentLanguageInSubDir = true +defaultCOntentVersionInSubDir = true +defaultContentVersion = "v1.0.0" +defaultContentRole = "guest" +defaultContentRoleInSubDir = true +title = "Benchmark" + +[languages] +[languages.en] +weight = 1 + + +` + + if multipleDimensions { + files += ` +[languages.nn] +weight = 2 +[languages.sv] +weight = 3 + +[versions] +[versions."v1.0.0"] +[versions."v2.0.0"] + +[roles] +[roles.guest] +weight = 300 +[roles.member] +weight = 200 +` + } + + files += ` + +[[module.mounts]] +source = 'content' +target = 'content' +[module.mounts.sites.matrix] +languages = ["**"] +versions = ["**"] +roles = ["**"] +-- layouts/all.html -- +All. {{ .Title }}| +` + if multipleDimensions { + files += ` +Rotate(language): {{ with .Rotate "language" }}{{ range . }}{{ template "printp" . }}|{{ end }}{{ end }}$ +Rotate(version): {{ with .Rotate "version" }}{{ range . }}{{ template "printp" . }}|{{ end }}{{ end }}$ +Rotate(role): {{ with .Rotate "role" }}{{ range . }}{{ template "printp" . }}|{{ end }}{{ end }}$ +{{ define "printp" }}{{ .RelPermalink }}:{{ with .Site }}{{ template "prints" . }}{{ end }}{{ end }} +{{ define "prints" }}/l:{{ .Language.Name }}/v:{{ .Version.Name }}/r:{{ .Role.Name }}{{ end }} + +` + } + + for i := range numPages { + files += fmt.Sprintf(` +-- content/p%d/index.md -- +--- +title: "P%d" +--- +`, i+1, i+1) + } + + b := hugolib.NewIntegrationTestBuilder( + hugolib.IntegrationTestConfig{ + T: t, + TxtarString: files, + BuildCfg: hugolib.BuildCfg{ + SkipRender: skipRender, + }, + }) + return b +} + +func TestSitesMatrixContentBenchmark(t *testing.T) { + const numPages = 3 + b := newSitesMatrixContentBenchmarkBuilder(t, numPages, false, true) + + b.Build() + + for _, lang := range []string{"en", "nn", "sv"} { + for _, ver := range []string{"v1.0.0", "v2.0.0"} { + for _, role := range []string{"guest", "member"} { + base := "public/" + role + "/" + ver + "/" + lang + b.AssertFileContent(base+"/index.html", "All. Benchmark|") + for i := range numPages { + b.AssertFileContent(fmt.Sprintf("%s/p%d/index.html", base, i+1), fmt.Sprintf("All. P%d|", i+1)) + } + + } + } + } +} + +func BenchmarkSitesMatrixContent(b *testing.B) { + for _, numPages := range []int{10, 100} { + for _, multipleDimensions := range []bool{false, true} { + b.Run(fmt.Sprintf("n%d/md%t", numPages, multipleDimensions), func(b *testing.B) { + builders := make([]*hugolib.IntegrationTestBuilder, b.N) + for i := 0; i < b.N; i++ { + builders[i] = newSitesMatrixContentBenchmarkBuilder(b, numPages, true, true) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + builders[i].Build() + } + }) + } + } +} diff --git a/hugolib/sitesmatrix/vectorstores.go b/hugolib/sitesmatrix/vectorstores.go new file mode 100644 index 000000000..6f3858574 --- /dev/null +++ b/hugolib/sitesmatrix/vectorstores.go @@ -0,0 +1,941 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sitesmatrix + +import ( + "cmp" + "fmt" + "iter" + xmaps "maps" + "slices" + "sort" + "sync" + + "github.com/gohugoio/hashstructure" + "github.com/gohugoio/hugo/common/hashing" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/predicate" + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/hugofs/hglob" +) + +var ( + _ VectorStore = &IntSets{} + _ VectorStore = &vectorStoreMap{} + _ hashstructure.Hashable = &IntSets{} + _ hashstructure.Hashable = &vectorStoreMap{} +) + +func newVectorStoreMap(cap int) *vectorStoreMap { + return &vectorStoreMap{ + sets: make(Vectors, cap), + h: &hashOnce{}, + } +} + +func newVectorStoreMapFromVectors(v Vectors) *vectorStoreMap { + return &vectorStoreMap{ + sets: v, + h: &hashOnce{}, + } +} + +// A vector store backed by a map. +type vectorStoreMap struct { + sets Vectors + h *hashOnce +} + +func (s *vectorStoreMap) initHash() { + s.h.once.Do(func() { + var err error + s.h.hash, err = hashing.Hash(s.sets) + if err != nil { + panic(fmt.Errorf("failed to calculate hash for MapVectorStore: %w", err)) + } + }) +} + +func (s *vectorStoreMap) setVector(vec Vector) { + s.sets[vec] = struct{}{} +} + +func (s *vectorStoreMap) Ordinal() int { + return 0 +} + +func (s *vectorStoreMap) KeysSorted() ([]int, []int, []int) { + var k0, k1, k2 []int + for v := range s.sets { + k0 = append(k0, v.Language()) + k1 = append(k1, v.Version()) + k2 = append(k2, v.Role()) + } + sort.Ints(k0) + sort.Ints(k1) + sort.Ints(k2) + k0 = slices.Compact(k0) + k1 = slices.Compact(k1) + k2 = slices.Compact(k2) + + return k0, k1, k2 +} + +func (s *vectorStoreMap) Hash() (uint64, error) { + s.initHash() + return s.h.hash, nil +} + +func (s *vectorStoreMap) MustHash() uint64 { + i, _ := s.Hash() + return i +} + +func (s *vectorStoreMap) HasVector(v Vector) bool { + if _, ok := s.sets[v]; ok { + return true + } + return false +} + +func (s *vectorStoreMap) HasAnyVector(v VectorProvider) bool { + if v == nil || s.LenVectors() == 0 || v.LenVectors() == 0 { + return false + } + + return !v.ForEachVector(func(vec Vector) bool { + if s.HasVector(vec) { + return false // stop iteration + } + return true // continue iteration + }) +} + +func (s *vectorStoreMap) LenVectors() int { + return len(s.sets) +} + +// Complement returns a new VectorStore that contains all vectors in s that are not in any of ss. +func (s *vectorStoreMap) Complement(ss ...VectorProvider) VectorStore { + if len(ss) == 0 || (len(ss) == 1 && ss[0] == s) { + return nil + } + + for _, v := range ss { + if v == s { + var s *vectorStoreMap + return s + } + } + + result := newVectorStoreMap(36) + + s.ForEachVector(func(vec Vector) bool { + var found bool + for _, v := range ss { + if v.HasVector(vec) { + found = true + break + } + } + + if !found { + result.setVector(vec) + } + + return true + }) + + return result +} + +func (s *vectorStoreMap) EqualsVector(other VectorProvider) bool { + if other == nil { + return false + } + if s == other { + return true + } + if s.LenVectors() != other.LenVectors() { + return false + } + return other.ForEachVector(func(v Vector) bool { + _, ok := s.sets[v] + return ok + }) +} + +func (s *vectorStoreMap) VectorSample() Vector { + if len(s.sets) == 0 { + panic("no vectors available") + } + for v := range s.sets { + return v + } + panic("unreachable") +} + +func (s *vectorStoreMap) ForEachVector(yield func(v Vector) bool) bool { + if len(s.sets) == 0 { + return true + } + for v := range s.sets { + if !yield(v) { + return false + } + } + return true +} + +func (s *vectorStoreMap) Vectors() []Vector { + if s == nil || len(s.sets) == 0 { + return nil + } + var vectors []Vector + for v := range s.sets { + vectors = append(vectors, v) + } + sort.Slice(vectors, func(i, j int) bool { + v1, v2 := vectors[i], vectors[j] + return v1.Compare(v2) < 0 + }) + return vectors +} + +func (s *vectorStoreMap) WithLanguageIndices(i int) VectorStore { + c := s.clone() + + for v := range c.sets { + v[Language] = i + c.sets[v] = struct{}{} + } + + return c +} + +func (s *vectorStoreMap) HasLanguage(i int) bool { + for v := range s.sets { + if v.Language() == i { + return true + } + } + return false +} + +func (s *vectorStoreMap) HasVersion(i int) bool { + for v := range s.sets { + if v.Version() == i { + return true + } + } + return false +} + +func (s *vectorStoreMap) HasRole(i int) bool { + for v := range s.sets { + if v.Role() == i { + return true + } + } + return false +} + +func (s *vectorStoreMap) clone() *vectorStoreMap { + c := *s + c.h = &hashOnce{} + c.sets = xmaps.Clone(s.sets) + return &c +} + +func NewIntSetsBuilder(cfg *ConfiguredDimensions) *IntSetsBuilder { + if cfg == nil { + panic("cfg is required") + } + return &IntSetsBuilder{cfg: cfg, s: &IntSets{h: &hashOnce{}}} +} + +type ConfiguredDimension interface { + predicate.IndexMatcher + IndexDefault() int + ResolveIndex(string) int + ResolveName(int) string + ForEachIndex() iter.Seq[int] +} + +type ConfiguredDimensions struct { + ConfiguredLanguages ConfiguredDimension + ConfiguredVersions ConfiguredDimension + ConfiguredRoles ConfiguredDimension +} + +func (c *ConfiguredDimensions) ResolveNames(v Vector) types.Strings3 { + return types.Strings3{ + c.ConfiguredLanguages.ResolveName(v.Language()), + c.ConfiguredVersions.ResolveName(v.Version()), + c.ConfiguredRoles.ResolveName(v.Role()), + } +} + +func (c *ConfiguredDimensions) ResolveVector(names types.Strings3) Vector { + var vec Vector + if s := names[0]; s != "" { + vec[0] = c.ConfiguredLanguages.ResolveIndex(s) + } else { + vec[0] = c.ConfiguredLanguages.IndexDefault() + } + if s := names[1]; s != "" { + vec[1] = c.ConfiguredVersions.ResolveIndex(s) + } else { + vec[1] = c.ConfiguredVersions.IndexDefault() + } + if s := names[2]; s != "" { + vec[2] = c.ConfiguredRoles.ResolveIndex(s) + } else { + vec[2] = c.ConfiguredRoles.IndexDefault() + } + return vec +} + +// IntSets holds the ordered sets of integers for the dimensions, +// which is used for fast membership testing of files, resources and pages. +type IntSets struct { + languages *maps.OrderedIntSet `mapstructure:"-" json:"-"` + versions *maps.OrderedIntSet `mapstructure:"-" json:"-"` + roles *maps.OrderedIntSet `mapstructure:"-" json:"-"` + + h *hashOnce +} + +type hashOnce struct { + once sync.Once + hash uint64 +} + +func (s *IntSets) ToVectorStoreMap() *vectorStoreMap { + if s == nil { + return nil + } + + result := newVectorStoreMap(36) + + s.ForEachVector(func(vec Vector) bool { + result.setVector(vec) + return true + }) + + return result +} + +func (s *IntSets) IsSuperSet(other *IntSets) bool { + return s.languages.IsSuperSet(other.languages) && + s.versions.IsSuperSet(other.versions) && + s.roles.IsSuperSet(other.roles) +} + +func (s *IntSets) DifferenceCardinality(other *IntSets) Vector { + return Vector{ + int(s.languages.Values().DifferenceCardinality(other.languages.Values())), + int(s.versions.Values().DifferenceCardinality(other.versions.Values())), + int(s.roles.Values().DifferenceCardinality(other.roles.Values())), + } +} + +func (s *IntSets) Intersects(other *IntSets) bool { + if s == nil || other == nil { + return false + } + return s.languages.Values().IntersectionCardinality(other.languages.Values()) > 0 && + s.versions.Values().IntersectionCardinality(other.versions.Values()) > 0 && + s.roles.Values().IntersectionCardinality(other.roles.Values()) > 0 +} + +// Complement returns a new VectorStore that contains all vectors in s that are not in any of ss. +func (s *IntSets) Complement(ss ...VectorProvider) VectorStore { + if len(ss) == 0 || (len(ss) == 1 && ss[0] == s) { + return nil + } + + for _, v := range ss { + vv, ok := v.(*IntSets) + if !ok { + continue + } + if vv.IsSuperSet(s) { + var s *IntSets + return s + } + } + + result := newVectorStoreMap(36) + + s.ForEachVector(func(vec Vector) bool { + var found bool + for _, v := range ss { + if v.HasVector(vec) { + found = true + break + } + } + + if !found { + result.setVector(vec) + } + + return true + }) + + return result +} + +func (s *IntSets) EqualsVector(other VectorProvider) bool { + if s == nil && other == nil { + return true + } + if s == nil || other == nil { + return false + } + if s == other { + return true + } + if s.LenVectors() != other.LenVectors() { + return false + } + + return other.ForEachVector(func(v Vector) bool { + return s.HasVector(v) + }) +} + +func (s *IntSets) VectorSample() Vector { + if s.LenVectors() == 0 { + panic("no vectors available") + } + + return Vector{ + s.languages.Next(0), + s.versions.Next(0), + s.roles.Next(0), + } +} + +// The reason we don't use iter.Seq is https://github.com/golang/go/issues/69015 +// This is 60% faster and allocation free. +// The yield function should return false to stop iteration. +func (s *IntSets) ForEachVector(yield func(v Vector) bool) bool { + if s.LenVectors() == 0 { + return true + } + + b := s.languages.ForEachKey(func(lang int) bool { + return s.versions.ForEachKey(func(ver int) bool { + return s.roles.ForEachKey(func(role int) bool { + return yield(Vector{lang, ver, role}) + }) + }) + }) + + return b +} + +func (s *IntSets) Vectors() []Vector { + if s.LenVectors() == 0 { + return nil + } + + var vectors []Vector + s.ForEachVector(func(v Vector) bool { + vectors = append(vectors, v) + return true + }) + + sort.Slice(vectors, func(i, j int) bool { + v1, v2 := vectors[i], vectors[j] + return v1.Compare(v2) < 0 + }) + + return vectors +} + +func (s *IntSets) KeysSorted() ([]int, []int, []int) { + if s == nil { + return nil, nil, nil + } + languages := s.languages.KeysSorted() + versions := s.versions.KeysSorted() + roles := s.roles.KeysSorted() + return languages, versions, roles +} + +func (s *IntSets) HasLanguage(lang int) bool { + if s == nil { + return false + } + return s.languages.Has(lang) +} + +func (s *IntSets) LenVectors() int { + if s == nil { + return 0 + } + return s.languages.Len() * s.versions.Len() * s.roles.Len() +} + +func (s *IntSets) HasRole(role int) bool { + if s == nil { + return false + } + return s.roles.Has(role) +} + +func (s *IntSets) String() string { + return fmt.Sprintf("Languages: %v, Versions: %v, Roles: %v", s.languages, s.versions, s.roles) +} + +// HasVector checks if the given vector is contained in the sets. +func (s *IntSets) HasVector(v Vector) bool { + if s == nil { + return false + } + if !s.languages.Has(v.Language()) { + return false + } + if !s.versions.Has(v.Version()) { + return false + } + if !s.roles.Has(v.Role()) { + return false + } + return true +} + +func (s *IntSets) HasAnyVector(v VectorProvider) bool { + if s == nil || v == nil { + return false + } + if s.LenVectors() == 0 || v.LenVectors() == 0 { + return false + } + + if vs, ok := v.(*IntSets); ok { + // Fast path. + return s.Intersects(vs) + } + + return !v.ForEachVector(func(vec Vector) bool { + if s.HasVector(vec) { + return false // stop iteration + } + return true // continue iteration + }) +} + +func (s *IntSets) HasVersion(ver int) bool { + if s == nil { + return false + } + return s.versions.Has(ver) +} + +func (s IntSets) shallowClone() *IntSets { + s.h = &hashOnce{} + return &s +} + +// WithLanguageIndices replaces the current language set with a single language index. +func (s *IntSets) WithLanguageIndices(i int) VectorStore { + c := s.shallowClone() + c.languages = maps.NewOrderedIntSet(i) + return c.init() +} + +func (s *IntSets) Hash() (uint64, error) { + s.initHash() + return s.h.hash, nil +} + +func (s *IntSets) MustHash() uint64 { + if s == nil { + return 0 + } + hash, err := s.Hash() + if err != nil { + panic(fmt.Errorf("failed to calculate hash for IntSets: %w", err)) + } + return hash +} + +// setDefaultsIfNotSet applies default values to the IntSets if they are not already set. +func (s *IntSets) setDefaultsIfNotSet(cfg *ConfiguredDimensions) { + if s.languages == nil { + s.languages = maps.NewOrderedIntSet() + s.languages.Set(cfg.ConfiguredLanguages.IndexDefault()) + } + if s.versions == nil { + s.versions = maps.NewOrderedIntSet() + s.versions.Set(cfg.ConfiguredVersions.IndexDefault()) + } + if s.roles == nil { + s.roles = maps.NewOrderedIntSet() + s.roles.Set(cfg.ConfiguredRoles.IndexDefault()) + } +} + +func (s *IntSets) setDefaultsAndAllLAnguagesIfNotSet(cfg *ConfiguredDimensions) { + if s.languages == nil { + s.languages = maps.NewOrderedIntSet() + for i := range cfg.ConfiguredLanguages.ForEachIndex() { + s.languages.Set(i) + } + } + s.setDefaultsIfNotSet(cfg) +} + +func (s *IntSets) setAllIfNotSet(cfg *ConfiguredDimensions) { + if s.languages == nil { + s.languages = maps.NewOrderedIntSet() + for i := range cfg.ConfiguredLanguages.ForEachIndex() { + s.languages.Set(i) + } + } + if s.versions == nil { + s.versions = maps.NewOrderedIntSet() + for i := range cfg.ConfiguredVersions.ForEachIndex() { + s.versions.Set(i) + } + } + if s.roles == nil { + s.roles = maps.NewOrderedIntSet() + for i := range cfg.ConfiguredRoles.ForEachIndex() { + s.roles.Set(i) + } + } +} + +func (s *IntSets) initHash() { + s.h.once.Do(func() { + var err error + s.h.hash, err = hashing.Hash(s.languages.Words(), s.versions.Words(), s.roles.Words()) + if err != nil { + panic(fmt.Errorf("failed to calculate hash for IntSets: %w", err)) + } + }) +} + +func (s *IntSets) init() *IntSets { + return s +} + +func (s *IntSets) setDimensionsFromOtherIfNotSet(other VectorIterator) { + if other == nil { + return + } + setLang := s.languages == nil + setVer := s.versions == nil + setRole := s.roles == nil + + if !(setLang || setVer || setRole) { + return + } + + other.ForEachVector(func(v Vector) bool { + if !s.HasVector(v) { + s.setValuesInNilSets(v, setLang, setVer, setRole) + } + return true + }) +} + +func (s *IntSets) setValuesInNilSets(vec Vector, setLang, setVer, setRole bool) { + if setLang { + if s.languages == nil { + s.languages = maps.NewOrderedIntSet() + } + s.languages.Set(vec.Language()) + } + if setVer { + if s.versions == nil { + s.versions = maps.NewOrderedIntSet() + } + s.versions.Set(vec.Version()) + } + + if setRole { + if s.roles == nil { + s.roles = maps.NewOrderedIntSet() + } + s.roles.Set(vec.Role()) + } +} + +type IntSetsBuilder struct { + cfg *ConfiguredDimensions + s *IntSets + + // Set when a Glob (e.g. "en") filter is provided but no matches are found. + GlobFilterMisses Bools +} + +func (b *IntSetsBuilder) Build() *IntSets { + b.s.init() + return b.s +} + +func (b *IntSetsBuilder) WithConfig(cfg IntSetsConfig) *IntSetsBuilder { + applyFilter := func(what string, values []string, matcher ConfiguredDimension) (*maps.OrderedIntSet, error) { + var result *maps.OrderedIntSet + if len(values) == 0 { + + if cfg.ApplyDefaults > 0 { + result = maps.NewOrderedIntSet() + } + switch cfg.ApplyDefaults { + case IntSetsConfigApplyDefaultsIfNotSet: + result.Set(matcher.IndexDefault()) + case IntSetsConfigApplyDefaultsAndAllLanguagesIfNotSet: + if what == "languages" { + for i := range matcher.ForEachIndex() { + result.Set(i) + } + } else { + result.Set(matcher.IndexDefault()) + } + } + + return result, nil + } + + // Dot separated globs. + filter, err := predicate.NewStringPredicateFromGlobs(values, hglob.GetGlobDot) + if err != nil { + return nil, fmt.Errorf("failed to create filter for %s: %w", what, err) + } + for _, pattern := range values { + iter, err := matcher.IndexMatch(filter) + if err != nil { + return nil, fmt.Errorf("failed to match %s %q: %w", what, pattern, err) + } + for i := range iter { + if result == nil { + result = maps.NewOrderedIntSet() + } + result.Set(i) + } + } + + return result, nil + } + + l, err1 := applyFilter("languages", cfg.Globs.Languages, b.cfg.ConfiguredLanguages) + v, err2 := applyFilter("versions", cfg.Globs.Versions, b.cfg.ConfiguredVersions) + r, err3 := applyFilter("roles", cfg.Globs.Roles, b.cfg.ConfiguredRoles) + + if err := cmp.Or(err1, err2, err3); err != nil { + panic(fmt.Errorf("failed to apply filters: %w", err)) + } + + b.GlobFilterMisses = Bools{ + len(cfg.Globs.Languages) > 0 && l == nil, + len(cfg.Globs.Versions) > 0 && v == nil, + len(cfg.Globs.Roles) > 0 && r == nil, + } + + b.s.languages = l + b.s.versions = v + b.s.roles = r + + return b +} + +func (s *IntSetsBuilder) WithLanguageIndices(idxs ...int) *IntSetsBuilder { + if len(idxs) == 0 { + return s + } + if s.s.languages == nil { + s.s.languages = maps.NewOrderedIntSet() + } + for _, i := range idxs { + s.s.languages.Set(i) + } + return s +} + +func (s *IntSetsBuilder) WithDefaultsAndAllLanguagesIfNotSet() *IntSetsBuilder { + s.s.setDefaultsAndAllLAnguagesIfNotSet(s.cfg) + return s +} + +func (s *IntSetsBuilder) WithAllIfNotSet() *IntSetsBuilder { + s.s.setAllIfNotSet(s.cfg) + return s +} + +func (s *IntSetsBuilder) WithDefaultsIfNotSet() *IntSetsBuilder { + s.s.setDefaultsIfNotSet(s.cfg) + return s +} + +func (s *IntSetsBuilder) WithDimensionsFromOtherIfNotSet(other VectorIterator) *IntSetsBuilder { + s.s.setDimensionsFromOtherIfNotSet(other) + return s +} + +func (b *IntSetsBuilder) WithSets(languages, versions, roles *maps.OrderedIntSet) *IntSetsBuilder { + b.s.languages = languages + b.s.versions = versions + b.s.roles = roles + return b +} + +type IntSetsConfigApplyDefaults int + +const ( + IntSetsConfigApplyDefaultsNone IntSetsConfigApplyDefaults = iota + IntSetsConfigApplyDefaultsIfNotSet + IntSetsConfigApplyDefaultsAndAllLanguagesIfNotSet +) + +type IntSetsConfig struct { + ApplyDefaults IntSetsConfigApplyDefaults + Globs StringSlices +} + +// Sites holds configuration about which sites a file/content/page/resource belongs to. +type Sites struct { + // Matrix defines what sites to build this content for. + Matrix StringSlices `mapstructure:"matrix" json:"matrix,omitzero"` + // Complements defines what sites to complement with this content. + Complements StringSlices `mapstructure:"complements" json:"complements,omitzero"` +} + +func (s *Sites) Equal(other Sites) bool { + return s.Matrix.Equal(other.Matrix) && s.Complements.Equal(other.Complements) +} + +func (s *Sites) SetFromParamsIfNotSet(params maps.Params) { + const ( + matrixKey = "matrix" + complementsKey = "complements" + ) + + if m, ok := params[matrixKey]; ok { + s.Matrix.SetFromParamsIfNotSet(m.(maps.Params)) + } + if f, ok := params[complementsKey]; ok { + s.Complements.SetFromParamsIfNotSet(f.(maps.Params)) + } +} + +// IsZero returns true if all slices are empty. +func (s Sites) IsZero() bool { + return s.Matrix.IsZero() && s.Complements.IsZero() +} + +// StringSlices holds slices of Glob patterns for languages, versions, and roles. +type StringSlices struct { + Languages []string `mapstructure:"languages" json:"languages"` + Versions []string `mapstructure:"versions" json:"versions"` + Roles []string `mapstructure:"roles" json:"roles"` +} + +func (d StringSlices) Equal(other StringSlices) bool { + return slices.Equal(d.Languages, other.Languages) && + slices.Equal(d.Versions, other.Versions) && + slices.Equal(d.Roles, other.Roles) +} + +func (d *StringSlices) SetFromParamsIfNotSet(params maps.Params) { + const ( + languagesKey = "languages" + versionsKey = "versions" + rolesKey = "roles" + ) + + if len(d.Languages) == 0 { + if v, ok := params[languagesKey]; ok { + d.Languages = types.ToStringSlicePreserveString(v) + } + } + + if len(d.Versions) == 0 { + if v, ok := params[versionsKey]; ok { + d.Versions = types.ToStringSlicePreserveString(v) + } + } + + if len(d.Roles) == 0 { + if v, ok := params[rolesKey]; ok { + d.Roles = types.ToStringSlicePreserveString(v) + } + } +} + +func (d StringSlices) IsZero() bool { + return len(d.Languages) == 0 && len(d.Versions) == 0 && len(d.Roles) == 0 +} + +// Used in tests. +type testDimension struct { + names []string +} + +func (m testDimension) IndexDefault() int { + return 0 +} + +func (m *testDimension) ResolveIndex(name string) int { + for i, n := range m.names { + if n == name { + return i + } + } + return -1 +} + +func (m *testDimension) ResolveName(i int) string { + if i < 0 || i >= len(m.names) { + return "" + } + return m.names[i] +} + +func (m *testDimension) ForEachIndex() iter.Seq[int] { + return func(yield func(i int) bool) { + for i := range m.names { + if !yield(i) { + return + } + } + } +} + +func (m *testDimension) IndexMatch(match predicate.P[string]) (iter.Seq[int], error) { + return func(yield func(i int) bool) { + for i, n := range m.names { + if match(n) { + if !yield(i) { + return + } + } + } + }, nil +} + +// NewTestingDimensions creates a new ConfiguredDimensions for testing. +func NewTestingDimensions(languages, versions, roles []string) *ConfiguredDimensions { + return &ConfiguredDimensions{ + ConfiguredLanguages: &testDimension{names: languages}, + ConfiguredVersions: &testDimension{names: versions}, + ConfiguredRoles: &testDimension{names: roles}, + } +} diff --git a/hugolib/sitesmatrix/vectorstores_test.go b/hugolib/sitesmatrix/vectorstores_test.go new file mode 100644 index 000000000..2f41a3bc9 --- /dev/null +++ b/hugolib/sitesmatrix/vectorstores_test.go @@ -0,0 +1,511 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sitesmatrix_test + +import ( + "fmt" + "testing" + + "github.com/bits-and-blooms/bitset" + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/common/hashing" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" +) + +var testDims = sitesmatrix.NewTestingDimensions([]string{"en", "no"}, []string{"v1", "v2", "v3"}, []string{"admin", "editor", "viewer", "guest"}) + +func TestIntSets(t *testing.T) { + c := qt.New(t) + + sets := sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(1, 2), + maps.NewOrderedIntSet(1, 2, 3), + maps.NewOrderedIntSet(1, 2, 3), + ).Build() + + sets2 := sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(1, 2), + maps.NewOrderedIntSet(1, 2, 3), + maps.NewOrderedIntSet(1, 2, 4), + ).Build() + + sets3 := sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(2, 1), + maps.NewOrderedIntSet(1, 2, 3), + maps.NewOrderedIntSet(2, 1, 4), + ).Build() + + sets4 := sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(3, 4, 5), + maps.NewOrderedIntSet(3, 4, 5), + maps.NewOrderedIntSet(3, 4, 5), + ).Build() + + c.Assert(hashing.HashStringHex(sets), qt.Equals, "790f6004619934ff") + c.Assert(hashing.HashStringHex(sets2), qt.Equals, "99abddc51cd22f24") + c.Assert(hashing.HashStringHex(sets3), qt.Equals, "99abddc51cd22f24") + + c.Assert(sets.HasVector(sitesmatrix.Vector{1, 2, 3}), qt.Equals, true) + c.Assert(sets.HasVector(sitesmatrix.Vector{3, 2, 3}), qt.Equals, false) + c.Assert(sets.HasAnyVector(sets2), qt.Equals, true) + c.Assert(sets.HasAnyVector(sets3), qt.Equals, true) + c.Assert(sets.HasAnyVector(sets4), qt.Equals, false) + + c.Assert(sets.VectorSample(), qt.Equals, sitesmatrix.Vector{1, 1, 1}) + c.Assert(sets.EqualsVector(sets), qt.Equals, true) + c.Assert(sets.EqualsVector( + sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(1, 2), + maps.NewOrderedIntSet(1, 2, 3), + maps.NewOrderedIntSet(1, 2, 3), + ).Build(), + ), qt.Equals, true) + c.Assert(sets.EqualsVector( + sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(1, 2, 3), + maps.NewOrderedIntSet(1, 2, 3, 4), + maps.NewOrderedIntSet(1, 2, 3, 4), + ).Build(), + ), qt.Equals, false) + + c.Assert(sets.EqualsVector( + sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(1, 2), + maps.NewOrderedIntSet(1, 2, 3), + maps.NewOrderedIntSet(2, 3, 4), + ).Build(), + ), qt.Equals, false) + + allCount := 0 + seen := make(map[sitesmatrix.Vector]bool) + ok := sets.ForEachVector(func(v sitesmatrix.Vector) bool { + c.Assert(seen[v], qt.IsFalse) + seen[v] = true + allCount++ + return true + }) + + c.Assert(ok, qt.IsTrue) + + // 2 languages * 3 versions * 3 roles = 18 combinations. + c.Assert(allCount, qt.Equals, 18) +} + +func TestBitSetExperiments(t *testing.T) { + c := qt.New(t) + + a, b, d := bitset.New(10), bitset.New(10), bitset.New(10) + + ints := func(v *bitset.BitSet) []uint { + var ints []uint + for i := range v.EachSet() { + ints = append(ints, i) + } + return ints + } + + a.Set(3).Set(4) + b.Set(4).Set(5).Set(6).Set(7) + d.Set(3).Set(4) + + c.Assert(b.Test(5), qt.Equals, true) + + fmt.Println("a:", ints(a), "==>", ints(b), "=> Intersection:", ints(a.Intersection(b)), "=> Symdiff:", ints(a.SymmetricDifference(b))) + fmt.Println("===>", ints(a.Difference(b))) + fmt.Println("===>", ints(b.Difference(a))) + fmt.Println("===> d", d.Difference(a).Count()) +} + +func TestIntSetsComplement(t *testing.T) { + c := qt.New(t) + + type values struct { + v0 []int + v1 []int + v2 []int + } + + type test struct { + name string + left values + input []values + expected []sitesmatrix.Vector + } + + runOne := func(c *qt.C, test test) { + c.Helper() + + self := sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(test.left.v0...), + maps.NewOrderedIntSet(test.left.v1...), + maps.NewOrderedIntSet(test.left.v2...), + ).Build() + + var input []sitesmatrix.VectorProvider + for _, v := range test.input { + input = append(input, sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(v.v0...), + maps.NewOrderedIntSet(v.v1...), + maps.NewOrderedIntSet(v.v2...), + ).Build()) + } + + result := self.Complement(input...) + + vectors := result.Vectors() + c.Assert(vectors, qt.DeepEquals, test.expected) + + c.Assert(hashing.HashStringHex(result), qt.Not(qt.Equals), hashing.HashStringHex(self)) + } + + for _, test := range []test{ + { + name: "Test 3", + left: values{ + v0: []int{1, 3, 5}, + v1: []int{1}, + v2: []int{1}, + }, + input: []values{ + {v0: []int{2}, v1: []int{1}, v2: []int{1}}, + {v0: []int{2, 3}, v1: []int{1}, v2: []int{1}}, + }, + expected: []sitesmatrix.Vector{ + {1, 1, 1}, + {5, 1, 1}, + }, + }, + { + name: "Test 1", + left: values{ + v0: []int{1}, + v1: []int{1}, + v2: []int{1}, + }, + input: []values{ + {v0: []int{2}, v1: []int{1}, v2: []int{1}}, + }, + expected: []sitesmatrix.Vector{ + {1, 1, 1}, + }, + }, + { + name: "Same values", + left: values{ + v0: []int{1}, + v1: []int{1}, + v2: []int{1}, + }, + input: []values{ + {v0: []int{1}, v1: []int{1}, v2: []int{1}}, + }, + expected: nil, + }, + { + name: "Test 2", + left: values{ + v0: []int{1, 3, 5}, + v1: []int{1}, + v2: []int{1}, + }, + input: []values{ + {v0: []int{2}, v1: []int{1}, v2: []int{1}}, + }, + expected: []sitesmatrix.Vector{ + {1, 1, 1}, + {3, 1, 1}, + {5, 1, 1}, + }, + }, + { + name: "Many", + left: values{ + v0: []int{1, 3, 5, 6, 7}, + v1: []int{1, 3, 5, 6, 7}, + v2: []int{1, 3, 5, 6, 7}, + }, + input: []values{ + { + v0: []int{1, 3, 5, 6, 7}, + v1: []int{1, 3, 5, 6}, + v2: []int{1, 3, 5, 6, 7}, + }, + }, + expected: []sitesmatrix.Vector{ + {1, 7, 1}, + {1, 7, 3}, + {1, 7, 5}, + {1, 7, 6}, + {1, 7, 7}, + {3, 7, 1}, + {3, 7, 3}, + {3, 7, 5}, + {3, 7, 6}, + {3, 7, 7}, + {5, 7, 1}, + {5, 7, 3}, + {5, 7, 5}, + {5, 7, 6}, + {5, 7, 7}, + {6, 7, 1}, + {6, 7, 3}, + {6, 7, 5}, + {6, 7, 6}, + {6, 7, 7}, + {7, 7, 1}, + {7, 7, 3}, + {7, 7, 5}, + {7, 7, 6}, + {7, 7, 7}, + }, + }, + } { + c.Run(test.name, func(c *qt.C) { + runOne(c, test) + }) + } +} + +func TestIntSetsComplementOfComplement(t *testing.T) { + c := qt.New(t) + + sets1 := sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(1), + maps.NewOrderedIntSet(1), + maps.NewOrderedIntSet(1), + ).Build() + + sets2 := sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(1, 2), + maps.NewOrderedIntSet(1), + maps.NewOrderedIntSet(1, 3), + ).Build() + + c1 := sets2.Complement(sets1) + + vectors := c1.Vectors() + // c.Assert(len(vectors), qt.Equals, 3) + c.Assert(vectors, qt.DeepEquals, []sitesmatrix.Vector{ + {1, 1, 3}, + {2, 1, 1}, + {2, 1, 3}, + }) + + c.Assert(hashing.HashStringHex(c1), qt.Not(qt.Equals), hashing.HashStringHex(sets1)) + c.Assert(hashing.HashStringHex(c1), qt.Not(qt.Equals), hashing.HashStringHex(sets2)) + + c2 := sets1.Complement(c1) + c.Assert(c2.Vectors(), qt.DeepEquals, []sitesmatrix.Vector{ + {1, 1, 1}, + }) +} + +func BenchmarkIntSetsComplement(b *testing.B) { + sets1 := sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(1, 2, 3), + maps.NewOrderedIntSet(1, 2, 3), + maps.NewOrderedIntSet(1, 2, 3), + ).Build() + + sets1Copy := sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(1, 2, 3), + maps.NewOrderedIntSet(1, 2, 3), + maps.NewOrderedIntSet(1, 2, 3), + ).Build() + + sets2 := sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(1, 2, 3, 4), + maps.NewOrderedIntSet(1, 2, 3, 4), + maps.NewOrderedIntSet(1, 2, 3, 4, 6), + ).Build() + + setsLanguage1 := sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(1), + maps.NewOrderedIntSet(1), + maps.NewOrderedIntSet(1), + ).Build() + + b.ResetTimer() + b.Run("sub set", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = sets2.Complement(sets1) + } + }) + + b.Run("super set", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = sets1.Complement(sets2) + } + }) + + b.Run("self", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = sets1.Complement(sets1) + } + }) + + b.Run("self multiple", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = sets1.Complement(sets1, sets1, sets1, sets1, sets1, sets1, sets1) + } + }) + + b.Run("same", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = sets1.Complement(sets1Copy) + } + }) + + b.Run("one overlapping language", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = setsLanguage1.Complement(sets1) + } + }) +} + +func BenchmarkSets(b *testing.B) { + sets1 := sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(1, 2), + maps.NewOrderedIntSet(1, 2, 3), + maps.NewOrderedIntSet(1, 2, 3), + ).Build() + + sets1Copy := sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(1, 2), + maps.NewOrderedIntSet(1, 2, 3), + maps.NewOrderedIntSet(1, 2, 3), + ).Build() + + sets2 := sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(1, 2, 3), + maps.NewOrderedIntSet(1, 2, 3, 4), + maps.NewOrderedIntSet(1, 2, 3, 4), + ).Build() + + v1 := sitesmatrix.Vector{1, 2, 3} + v2 := sitesmatrix.Vector{3, 2, 3} + + b.ResetTimer() + b.Run("Build", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(1, 2), + maps.NewOrderedIntSet(1, 2, 3), + maps.NewOrderedIntSet(1, 2, 3), + ).Build() + } + }) + b.Run("HasVector", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = sets1.HasVector(v1) + _ = sets1.HasVector(v2) + } + }) + + b.Run("HasAnyVector", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = sets1.HasAnyVector(sets2) + } + }) + + b.Run("FirstVector", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = sets1.VectorSample() + } + }) + + b.Run("LenVectors", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = sets1.LenVectors() + } + }) + + b.Run("ForEachVector", func(b *testing.B) { + for i := 0; i < b.N; i++ { + allCount := 0 + ok := sets1.ForEachVector(func(v sitesmatrix.Vector) bool { + allCount++ + _ = v + return true + }) + + if !ok { + b.Fatal("Expected ForEachVector to return true") + } + + if allCount != 18 { + b.Fatalf("Expected 18 combinations, got %d", allCount) + } + } + }) + + b.Run("EqualsVector pointer equal", func(b *testing.B) { + for i := 0; i < b.N; i++ { + if !sets1.EqualsVector(sets1) { + b.Fatal("Expected sets1 to equal itself") + } + } + }) + + b.Run("EqualsVector equal copy", func(b *testing.B) { + for i := 0; i < b.N; i++ { + if !sets1.EqualsVector(sets1Copy) { + b.Fatal("Expected sets1 to equal its copy") + } + } + }) + + b.Run("EqualsVector different sets", func(b *testing.B) { + for i := 0; i < b.N; i++ { + if sets1.EqualsVector(sets2) { + b.Fatal("Expected sets1 to not equal sets2") + } + } + }) + + b.Run("Distance", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = sets1.VectorSample().Distance(v1) + _ = sets1.VectorSample().Distance(v2) + } + }) +} + +func TestVectorStoreMap(t *testing.T) { + c := qt.New(t) + + c.Run("Complement", func(c *qt.C) { + v1 := sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(1, 3), + maps.NewOrderedIntSet(1), + maps.NewOrderedIntSet(1, 3), + ).Build() + + m := v1.ToVectorStoreMap() + + v2 := sitesmatrix.NewIntSetsBuilder(testDims).WithSets( + maps.NewOrderedIntSet(1, 2), + maps.NewOrderedIntSet(1), + maps.NewOrderedIntSet(1, 3), + ).Build() + + complement := m.Complement(v2) + + c.Assert(complement.Vectors(), qt.DeepEquals, []sitesmatrix.Vector{ + {3, 1, 1}, + {3, 1, 3}, + }) + }) +} diff --git a/hugolib/template_test.go b/hugolib/template_test.go index a08f83cb8..31b18bdc4 100644 --- a/hugolib/template_test.go +++ b/hugolib/template_test.go @@ -437,10 +437,11 @@ func TestPartialWithReturn(t *testing.T) { } c.Run("Return", func(c *qt.C) { - b := newBuilder(c) + for range 2 { + b := newBuilder(c) - b.WithTemplatesAdded( - "index.html", ` + b.WithTemplatesAdded( + "index.html", ` Test Partials With Return Values: add42: 50: {{ partial "add42.tpl" 8 }} @@ -449,18 +450,19 @@ dollarContext: 60: {{ partial "dollarContext.tpl" 18 }} adder: 70: {{ partial "dict.tpl" (dict "adder" 28) }} complex: 80: {{ partial "complex.tpl" 38 }} `, - ) + ) - b.CreateSites().Build(BuildCfg{}) + b.CreateSites().Build(BuildCfg{}) - b.AssertFileContent("public/index.html", ` + b.AssertFileContent("public/index.html", ` add42: 50: 50 hello world: hello world dollarContext: 60: 60 adder: 70: 70 complex: 80: 80 `, - ) + ) + } }) } diff --git a/hugolib/versions/versions.go b/hugolib/versions/versions.go new file mode 100644 index 000000000..142087da9 --- /dev/null +++ b/hugolib/versions/versions.go @@ -0,0 +1,211 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package versions + +import ( + "errors" + "fmt" + "iter" + "sort" + + "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/common/predicate" + "github.com/gohugoio/hugo/common/version" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" + "github.com/mitchellh/mapstructure" +) + +var _ sitesmatrix.DimensionInfo = (*versionWrapper)(nil) + +type VersionConfig struct { + // The weight of the version. + // Used to determine the order of the versions. + // If zero, we use the version name to sort. + Weight int +} + +type Version interface { + Name() string +} + +type Versions []Version + +type versionWrapper struct { + v VersionInternal +} + +func (v versionWrapper) Name() string { + return v.v.Name +} + +func (v versionWrapper) IsDefault() bool { + return v.v.Default +} + +func NewVersion(v VersionInternal) Version { + return versionWrapper{v: v} +} + +var _ Version = (*versionWrapper)(nil) + +type VersionInternal struct { + // Name of the version. + // This is the key from the config. + Name string + // Whether this version is the default version. + // This will be by default rendered in the root. + // There can only be one default version. + Default bool + + VersionConfig +} + +type VersionsInternal struct { + versionConfigs map[string]VersionConfig + + Sorted []VersionInternal +} + +func (r VersionsInternal) IndexDefault() int { + for i, version := range r.Sorted { + if version.Default { + return i + } + } + panic("no default version found") +} + +func (r VersionsInternal) ResolveName(i int) string { + if i < 0 || i >= len(r.Sorted) { + panic(fmt.Sprintf("index %d out of range for versions", i)) + } + return r.Sorted[i].Name +} + +func (r VersionsInternal) ResolveIndex(name string) int { + for i, version := range r.Sorted { + if version.Name == name { + return i + } + } + panic(fmt.Sprintf("no version found for name %q", name)) +} + +// IndexMatch returns an iterator for the versions that match the filter. +func (r VersionsInternal) IndexMatch(match predicate.P[string]) (iter.Seq[int], error) { + return func(yield func(i int) bool) { + for i, version := range r.Sorted { + if match(version.Name) { + if !yield(i) { + return + } + } + } + }, nil +} + +// ForEachIndex returns an iterator for the indices of the versions. +func (r VersionsInternal) ForEachIndex() iter.Seq[int] { + return func(yield func(i int) bool) { + for i := range r.Sorted { + if !yield(i) { + return + } + } + } +} + +const defaultContentVersionFallback = "v1" + +func (r *VersionsInternal) init(defaultContentVersion string) error { + if r.versionConfigs == nil { + r.versionConfigs = make(map[string]VersionConfig) + } + defaultContentVersionProvided := defaultContentVersion != "" + if len(r.versionConfigs) == 0 { + if defaultContentVersion == "" { + defaultContentVersion = defaultContentVersionFallback + } + // Add a default version. + r.versionConfigs[defaultContentVersion] = VersionConfig{} + } + + var defaultSeen bool + for k, v := range r.versionConfigs { + if k == "" { + return errors.New("version name cannot be empty") + } + if err := paths.ValidateIdentifier(k); err != nil { + return fmt.Errorf("version name %q is invalid: %s", k, err) + } + + var isDefault bool + if k == defaultContentVersion { + isDefault = true + defaultSeen = true + } + + r.Sorted = append(r.Sorted, VersionInternal{Name: k, Default: isDefault, VersionConfig: v}) + } + + // Sort by weight if set, then semver descending. + sort.SliceStable(r.Sorted, func(i, j int) bool { + ri, rj := r.Sorted[i], r.Sorted[j] + if ri.Weight == rj.Weight { + v1, v2 := version.MustParseVersion(ri.Name), version.MustParseVersion(rj.Name) + return v1.Compare(v2) < 0 + } + if rj.Weight == 0 { + return true + } + if ri.Weight == 0 { + return false + } + return ri.Weight < rj.Weight + }) + + if !defaultSeen { + if defaultContentVersionProvided { + return fmt.Errorf("the configured defaultContentVersion %q does not exist", defaultContentVersion) + } + // If no default version is set, we set the first one. + first := r.Sorted[0] + first.Default = true + r.versionConfigs[first.Name] = first.VersionConfig + r.Sorted[0] = first + } + + return nil +} + +func (r VersionsInternal) Has(version string) bool { + _, found := r.versionConfigs[version] + return found +} + +func DecodeConfig(defaultContentVersion string, m map[string]any) (*config.ConfigNamespace[map[string]VersionConfig, VersionsInternal], error) { + return config.DecodeNamespace[map[string]VersionConfig](m, func(in any) (VersionsInternal, any, error) { + var versions VersionsInternal + var conf map[string]VersionConfig + if err := mapstructure.Decode(m, &conf); err != nil { + return versions, nil, err + } + versions.versionConfigs = conf + if err := versions.init(defaultContentVersion); err != nil { + return versions, nil, err + } + return versions, nil, nil + }) +} diff --git a/hugolib/versions/versions_integration_test.go b/hugolib/versions/versions_integration_test.go new file mode 100644 index 000000000..cff84e060 --- /dev/null +++ b/hugolib/versions/versions_integration_test.go @@ -0,0 +1,37 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package versions_test + +import ( + "testing" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/hugolib" +) + +func TestDefaultContentVersionDoesNotExist(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ["rss", "sitemap", "section", "taxonomy", "term"] +defaultContentLanguage = "en" +defaultContentLanguageInSubDir = true +defaultContentVersion = "doesnotexist" +[versions] +[versions."v1.0.0"] +` + b, err := hugolib.TestE(t, files) + b.Assert(err, qt.ErrorMatches, `.*failed to decode "versions": the configured defaultContentVersion "doesnotexist" does not exist`) +} diff --git a/identity/finder_test.go b/identity/finder_test.go index abfab9d75..949a43713 100644 --- a/identity/finder_test.go +++ b/identity/finder_test.go @@ -21,9 +21,9 @@ import ( ) func BenchmarkFinder(b *testing.B) { - m1 := identity.NewManager("") - m2 := identity.NewManager("") - m3 := identity.NewManager("") + m1 := identity.NewManager() + m2 := identity.NewManager() + m3 := identity.NewManager() m1.AddIdentity( testIdentity{"base", "id1", "", "pe1"}, testIdentity{"base2", "id2", "eq1", ""}, diff --git a/identity/identity.go b/identity/identity.go index c78ed0fdd..7cc303ffa 100644 --- a/identity/identity.go +++ b/identity/identity.go @@ -41,10 +41,9 @@ const ( var NopManager = new(nopManager) // NewIdentityManager creates a new Manager. -func NewManager(name string, opts ...ManagerOption) Manager { +func NewManager(opts ...ManagerOption) Manager { idm := &identityManager{ Identity: Anonymous, - name: name, ids: Identities{}, } @@ -94,9 +93,6 @@ func FirstIdentity(v any) Identity { func PrintIdentityInfo(v any) { WalkIdentitiesDeep(v, func(level int, id Identity) bool { var s string - if idm, ok := id.(*identityManager); ok { - s = " " + idm.name - } fmt.Printf("%s%s (%T)%s\n", strings.Repeat(" ", level), id.IdentifierBase(), id, s) return false }) @@ -154,8 +150,8 @@ type DependencyManagerScopedProvider interface { // ForEeachIdentityProvider provides a way iterate over identities. type ForEeachIdentityProvider interface { // ForEeachIdentityProvider calls cb for each Identity. - // If cb returns true, the iteration is terminated. - // The return value is whether the iteration was terminated. + // If cb returns false, the iteration is terminated. + // It returns false if the iteration was terminated early. ForEeachIdentity(cb func(id Identity) bool) bool } @@ -296,9 +292,6 @@ func (s StringIdentity) IdentifierBase() string { type identityManager struct { Identity - // Only used for debugging. - name string - // mu protects _changes_ to this manager, // reads currently assumes no concurrent writes. mu sync.RWMutex @@ -363,10 +356,6 @@ func (im *identityManager) GetDependencyManagerForScopesAll() []Manager { return []Manager{im} } -func (im *identityManager) String() string { - return fmt.Sprintf("IdentityManager(%s)", im.name) -} - func (im *identityManager) forEeachIdentity(fn func(id Identity) bool) bool { // The absence of a lock here is deliberate. This is currently only used on server reloads // in a single-threaded context. @@ -376,11 +365,11 @@ func (im *identityManager) forEeachIdentity(fn func(id Identity) bool) bool { } } for _, fe := range im.forEachIds { - if fe.ForEeachIdentity(fn) { - return true + if !fe.ForEeachIdentity(fn) { + return false } } - return false + return true } type nopManager int diff --git a/identity/identity_test.go b/identity/identity_test.go index f9b04aa14..dac92d86b 100644 --- a/identity/identity_test.go +++ b/identity/identity_test.go @@ -34,7 +34,7 @@ func BenchmarkIdentityManager(b *testing.B) { b.Run("identity.NewManager", func(b *testing.B) { for i := 0; i < b.N; i++ { - m := identity.NewManager("") + m := identity.NewManager() if m == nil { b.Fatal("manager is nil") } @@ -43,7 +43,7 @@ func BenchmarkIdentityManager(b *testing.B) { b.Run("Add unique", func(b *testing.B) { ids := createIds(b.N) - im := identity.NewManager("") + im := identity.NewManager() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -55,7 +55,7 @@ func BenchmarkIdentityManager(b *testing.B) { b.Run("Add duplicates", func(b *testing.B) { id := &testIdentity{base: "a", name: "b"} - im := identity.NewManager("") + im := identity.NewManager() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -107,9 +107,9 @@ func BenchmarkIsNotDependent(b *testing.B) { } newNestedManager := func(depth, count int) identity.Manager { - m1 := identity.NewManager("") + m1 := identity.NewManager() for range depth { - m2 := identity.NewManager("") + m2 := identity.NewManager() m1.AddIdentity(m2) for j := range count { id := fmt.Sprintf("id%d", j) @@ -139,9 +139,9 @@ func TestIdentityManager(t *testing.T) { c := qt.New(t) newNestedManager := func() identity.Manager { - m1 := identity.NewManager("") - m2 := identity.NewManager("") - m3 := identity.NewManager("") + m1 := identity.NewManager() + m2 := identity.NewManager() + m3 := identity.NewManager() m1.AddIdentity( testIdentity{"base", "id1", "", "pe1"}, testIdentity{"base2", "id2", "eq1", ""}, diff --git a/identity/predicate_identity.go b/identity/predicate_identity.go index bad247867..347c90460 100644 --- a/identity/predicate_identity.go +++ b/identity/predicate_identity.go @@ -17,26 +17,8 @@ package identity import ( "fmt" "sync/atomic" - - hglob "github.com/gohugoio/hugo/hugofs/glob" ) -// NewGlobIdentity creates a new Identity that -// is probably dependent on any other Identity -// that matches the given pattern. -func NewGlobIdentity(pattern string) Identity { - glob, err := hglob.GetGlob(pattern) - if err != nil { - panic(err) - } - - predicate := func(other Identity) bool { - return glob.Match(other.IdentifierBase()) - } - - return NewPredicateIdentity(predicate, nil) -} - var predicateIdentityCounter = &atomic.Uint32{} type predicateIdentity struct { diff --git a/identity/predicate_identity_test.go b/identity/predicate_identity_test.go index 3a54dee75..17a21fc71 100644 --- a/identity/predicate_identity_test.go +++ b/identity/predicate_identity_test.go @@ -20,23 +20,6 @@ import ( qt "github.com/frankban/quicktest" ) -func TestGlobIdentity(t *testing.T) { - c := qt.New(t) - - gid := NewGlobIdentity("/a/b/*") - - c.Assert(isNotDependent(gid, StringIdentity("/a/b/c")), qt.IsFalse) - c.Assert(isNotDependent(gid, StringIdentity("/a/c/d")), qt.IsTrue) - c.Assert(isNotDependent(StringIdentity("/a/b/c"), gid), qt.IsTrue) - c.Assert(isNotDependent(StringIdentity("/a/c/d"), gid), qt.IsTrue) -} - -func isNotDependent(a, b Identity) bool { - f := NewFinder(FinderConfig{}) - r := f.Contains(a, b, -1) - return r == 0 -} - func TestPredicateIdentity(t *testing.T) { c := qt.New(t) diff --git a/internal/js/esbuild/batch.go b/internal/js/esbuild/batch.go index aa50cf2c1..7f5d39855 100644 --- a/internal/js/esbuild/batch.go +++ b/internal/js/esbuild/batch.go @@ -31,6 +31,7 @@ import ( "github.com/evanw/esbuild/pkg/api" "github.com/gohugoio/hugo/cache/dynacache" + "github.com/gohugoio/hugo/common/hsync" "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/paths" @@ -38,7 +39,6 @@ import ( "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/internal/js" - "github.com/gohugoio/hugo/lazy" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" @@ -123,8 +123,7 @@ func (o *opts[K, C]) Key() K { } func (o *opts[K, C]) Reset() { - mu := o.once.ResetWithLock() - defer mu.Unlock() + o.once.Reset() o.h.resetCounter++ } @@ -220,7 +219,7 @@ func (c *BatcherClient) New(id string) (js.Batcher, error) { return nil, initErr } - dependencyManager := c.d.Conf.NewIdentityManager("jsbatch_" + id) + dependencyManager := c.d.Conf.NewIdentityManager() configID := "config_" + id b := &batcher{ @@ -362,7 +361,7 @@ func (b *batcher) Group(ctx context.Context, id string) js.BatcherGroup { group, found := b.scriptGroups[id] if !found { - idm := b.client.d.Conf.NewIdentityManager("jsbatch_" + id) + idm := b.client.d.Conf.NewIdentityManager() b.dependencyManager.AddIdentity(idm) group = &scriptGroup{ @@ -864,7 +863,7 @@ type optionsMap[K key, C any] map[K]optionsGetSetter[K, C] type opts[K any, C optionsCompiler[C]] struct { key K h *optsHolder[C] - once lazy.OnceMore + once hsync.OnceMore } type optsHolder[C optionsCompiler[C]] struct { diff --git a/langs/config.go b/langs/config.go index 7cca0f5e7..db708c3ac 100644 --- a/langs/config.go +++ b/langs/config.go @@ -15,8 +15,14 @@ package langs import ( "errors" + "fmt" + "iter" + "slices" + "sort" - "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/common/predicate" + "github.com/gohugoio/hugo/config" "github.com/mitchellh/mapstructure" ) @@ -44,15 +50,174 @@ type LanguageConfig struct { Disabled bool } -func DecodeConfig(m map[string]any) (map[string]LanguageConfig, error) { - m = maps.CleanConfigStringMap(m) - var langs map[string]LanguageConfig +type LanguageInternal struct { + // Name is the name of the role, extracted from the key in the config. + Name string - if err := mapstructure.WeakDecode(m, &langs); err != nil { - return nil, err + // Whether this role is the default role. + // This will be rendered in the root. + // There is only be one default role. + Default bool + + LanguageConfig +} + +type LanguagesInternal struct { + LanguageConfigs map[string]LanguageConfig + Sorted []LanguageInternal +} + +func (ls LanguagesInternal) IndexDefault() int { + for i, role := range ls.Sorted { + if role.Default { + return i + } + } + panic("no default role found") +} + +func (ls LanguagesInternal) ResolveName(i int) string { + if i < 0 || i >= len(ls.Sorted) { + panic(fmt.Sprintf("index %d out of range for languages", i)) + } + return ls.Sorted[i].Name +} + +func (ls LanguagesInternal) ResolveIndex(name string) int { + for i, role := range ls.Sorted { + if role.Name == name { + return i + } + } + panic(fmt.Sprintf("no language found for name %q", name)) +} + +// IndexMatch returns an iterator for the roles that match the filter. +func (ls LanguagesInternal) IndexMatch(match predicate.P[string]) (iter.Seq[int], error) { + return func(yield func(i int) bool) { + for i, l := range ls.Sorted { + if match(l.Name) { + if !yield(i) { + return + } + } + } + }, nil +} + +// ForEachIndex returns an iterator for the indices of the languages. +func (ls LanguagesInternal) ForEachIndex() iter.Seq[int] { + return func(yield func(i int) bool) { + for i := range ls.Sorted { + if !yield(i) { + return + } + } + } +} + +func (ls *LanguagesInternal) init(defaultContentLanguage string, disabledLanguages []string) (string, error) { + const en = "en" + + if len(ls.LanguageConfigs) == 0 { + // Add a default language. + if defaultContentLanguage == "" { + defaultContentLanguage = en + } + ls.LanguageConfigs[defaultContentLanguage] = LanguageConfig{} + } + + var ( + defaultSeen bool + enIdx int = -1 + ) + for k, v := range ls.LanguageConfigs { + if !v.Disabled && slices.Contains(disabledLanguages, k) { + // This language is disabled. + v.Disabled = true + ls.LanguageConfigs[k] = v + } + + if k == "" { + return "", errors.New("language name cannot be empty") + } + + if err := paths.ValidateIdentifier(k); err != nil { + return "", fmt.Errorf("language name %q is invalid: %s", k, err) + } + + var isDefault bool + if k == defaultContentLanguage { + isDefault = true + defaultSeen = true + } + + if isDefault && v.Disabled { + return "", fmt.Errorf("default language %q is disabled", k) + } + + if !v.Disabled { + ls.Sorted = append(ls.Sorted, LanguageInternal{Name: k, Default: isDefault, LanguageConfig: v}) + } } - if len(langs) == 0 { - return nil, errors.New("no languages configured") + + // Sort by weight if set, then by name. + sort.SliceStable(ls.Sorted, func(i, j int) bool { + ri, rj := ls.Sorted[i], ls.Sorted[j] + if ri.Weight == rj.Weight { + return ri.Name < rj.Name + } + if rj.Weight == 0 { + return true + } + if ri.Weight == 0 { + return false + } + return ri.Weight < rj.Weight + }) + + for i, l := range ls.Sorted { + if l.Name == en { + enIdx = i + break + } + } + + if !defaultSeen { + if defaultContentLanguage != "" { + // Set by the user, but not found in the config. + return "", fmt.Errorf("defaultContentLanguage %q not found in languages configuration", defaultContentLanguage) + } + // Not set by the user, so we use the first language in the config. + defaultIdx := 0 + if enIdx != -1 { + defaultIdx = enIdx + } + d := ls.Sorted[defaultIdx] + d.Default = true + ls.LanguageConfigs[d.Name] = d.LanguageConfig + ls.Sorted[defaultIdx] = d + defaultContentLanguage = d.Name + } - return langs, nil + + return defaultContentLanguage, nil +} + +func DecodeConfig(defaultContentLanguage string, disabledLanguages []string, m map[string]any) (*config.ConfigNamespace[map[string]LanguageConfig, LanguagesInternal], string, error) { + v, err := config.DecodeNamespace[map[string]LanguageConfig](m, func(in any) (LanguagesInternal, any, error) { + var languages LanguagesInternal + var conf map[string]LanguageConfig + if err := mapstructure.Decode(m, &conf); err != nil { + return languages, nil, err + } + languages.LanguageConfigs = conf + var err error + if defaultContentLanguage, err = languages.init(defaultContentLanguage, disabledLanguages); err != nil { + return languages, nil, err + } + return languages, nil, nil + }) + + return v, defaultContentLanguage, err } diff --git a/langs/i18n/translationProvider.go b/langs/i18n/translationProvider.go index 5932fe1f0..227492794 100644 --- a/langs/i18n/translationProvider.go +++ b/langs/i18n/translationProvider.go @@ -14,11 +14,13 @@ package i18n import ( + "context" "encoding/json" "fmt" "strings" "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/parser/metadecoders" "github.com/gohugoio/hugo/common/herrors" @@ -62,7 +64,7 @@ func (tp *TranslationProvider) NewResource(dst *deps.Deps) error { Fs: dst.BaseFs.I18n.Fs, IgnoreFile: dst.SourceSpec.IgnoreFile, PathParser: dst.SourceSpec.Cfg.PathParser(), - WalkFn: func(path string, info hugofs.FileMetaInfo) error { + WalkFn: func(ctx context.Context, path string, info hugofs.FileMetaInfo) error { if info.IsDir() { return nil } @@ -76,7 +78,7 @@ func (tp *TranslationProvider) NewResource(dst *deps.Deps) error { tp.t = NewTranslator(bundle, dst.Conf, dst.Log) - dst.Translate = tp.t.Func(dst.Conf.Language().Lang) + dst.Translate = tp.t.Func(dst.Conf.Language().(*langs.Language).Lang) return nil } @@ -126,7 +128,7 @@ func addTranslationFile(bundle *i18n.Bundle, r *source.File) error { // CloneResource sets the language func for the new language. func (tp *TranslationProvider) CloneResource(dst, src *deps.Deps) error { - dst.Translate = tp.t.Func(dst.Conf.Language().Lang) + dst.Translate = tp.t.Func(dst.Conf.Language().(*langs.Language).Lang) return nil } diff --git a/langs/language.go b/langs/language.go index d34ea1cc7..41f1898b2 100644 --- a/langs/language.go +++ b/langs/language.go @@ -24,13 +24,17 @@ import ( "github.com/gohugoio/hugo/common/htime" "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/hugolib/sitesmatrix" "github.com/gohugoio/locales" translators "github.com/gohugoio/localescompressed" ) +var _ sitesmatrix.DimensionInfo = (*Language)(nil) + type Language struct { // The language code, e.g. "en" or "no". - // This is currently only settable as the key in the language map in the config. + // This is the key used in the languages map in the configuration, + // and currently only settable as the key in the language map in the config. Lang string // Fields from the language config. @@ -41,16 +45,27 @@ type Language struct { translator locales.Translator timeFormatter htime.TimeFormatter tag language.Tag + // collator1 and collator2 are the same, we have 2 to prevent deadlocks. collator1 *Collator collator2 *Collator - location *time.Location + location *time.Location + isDefault bool // This is just an alias of Site.Params. params maps.Params } +// Name is an alias for Lang. +func (l *Language) Name() string { + return l.Lang +} + +func (l *Language) IsDefault() bool { + return l.isDefault +} + // NewLanguage creates a new language. func NewLanguage(lang, defaultContentLanguage, timeZone string, languageConfig LanguageConfig) (*Language, error) { translator := translators.GetTranslator(lang) @@ -87,6 +102,7 @@ func NewLanguage(lang, defaultContentLanguage, timeZone string, languageConfig L tag: tag, collator1: coll1, collator2: coll2, + isDefault: lang == defaultContentLanguage, } return l, l.loadLocation(timeZone) @@ -187,3 +203,13 @@ type Collator struct { func (c *Collator) CompareStrings(a, b string) int { return c.c.CompareString(a, b) } + +// IndexDefault returns the index of the default language. +func IndexDefault(languages Languages) int { + for i, l := range languages { + if l.isDefault { + return i + } + } + panic("no default lang found") +} diff --git a/langs/languages_integration_test.go b/langs/languages_integration_test.go new file mode 100644 index 000000000..20b93e0f9 --- /dev/null +++ b/langs/languages_integration_test.go @@ -0,0 +1,217 @@ +// Copyright 2025 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package langs_test + +import ( + "strings" + "testing" + + "github.com/gohugoio/hugo/hugolib" +) + +func TestLanguagesContentSimple(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +baseURL = "https://example.org/" +defaultContentLanguage = "en" +defaultContentLanguageInSubdir = true +[languages] +[languages.en] +weight = 2 +[languages.nn] +weight = 1 +-- content/_index.md -- +--- +title: "Home" +--- + +Welcome to the home page. +-- content/_index.nn.md -- +--- +title: "Heim" +--- +Welkomen heim! +-- layouts/all.html -- +title: {{ .Title }}| +` + b := hugolib.Test(t, files) + + b.AssertFileContent("public/en/index.html", `title: Home|`) + b.AssertFileContent("public/nn/index.html", `title: Heim|`) +} + +func TestLanguagesContentSharedResource(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "sitemap", "404"] +baseURL = "https://example.org/" +defaultContentLanguage = "en" +defaultContentLanguageInSubdir = true +[languages] +[languages.en] +weight = 2 +[languages.nn] +weight = 1 +-- content/mytext.txt -- +This is a shared text file. +-- content/_index.md -- +--- +title: "Home" +--- + +Welcome to the home page. +-- content/_index.nn.md -- +--- +title: "Heim" +--- +Welkomen heim! +-- layouts/home.html -- +{{ $text := .Resources.Get "mytext.txt" }} +title: {{ .Title }}|text: {{ with $text }}{{ .Content | safeHTML }}{{ end }}|{{ .Resources | len}} + +` + b := hugolib.Test(t, files) + + b.AssertFileContent("public/nn/index.html", `title: Heim|text: This is a shared text file.|1`) + b.AssertFileContent("public/en/index.html", `title: Home|text: This is a shared text file.|1`) +} + +// Issue 14031 +func TestDraftTermIssue14031(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ['home','rss','section','sitemap'] +capitalizeListTitles = false +pluralizeListTitles = false +defaultContentLanguage = 'en' +defaultContentLanguageInSubdir = true +[languages.en] +weight = 1 +[languages.fr] +weight = 2 +[taxonomies] +tag = 'tags' +-- content/p1.en.md -- +--- +title: P1 (en) +tags: [a,b] +--- +-- content/p1.fr.md -- +--- +title: P1 (fr) +tags: [a,b] +--- +-- content/tags/a/_index.en.md -- +--- +title: a (en) +--- +-- content/tags/a/_index.fr.md -- +--- +title: a (fr) +--- +-- content/tags/b/_index.en.md -- +--- +title: b (en) +--- +-- content/tags/b/_index.fr.md -- +--- +title: b (fr) +draft: false +--- +-- layouts/list.html -- +{{- .Title -}}| +{{- range .Pages -}} +TITLE: {{ .Title }} RELPERMALINK: {{ .RelPermalink }}| +{{- end -}} +-- layouts/page.html -- +{{- .Title -}}| +{{- range .GetTerms "tags" -}} +TITLE: {{ .Title }} RELPERMALINK: {{ .RelPermalink }}| +{{- end -}} +` + + b := hugolib.Test(t, files) + + b.AssertFileExists("public/en/tags/a/index.html", true) + b.AssertFileExists("public/fr/tags/a/index.html", true) + b.AssertFileExists("public/en/tags/b/index.html", true) + b.AssertFileExists("public/fr/tags/b/index.html", true) + + b.AssertFileContentEquals("public/en/p1/index.html", + "P1 (en)|TITLE: a (en) RELPERMALINK: /en/tags/a/|TITLE: b (en) RELPERMALINK: /en/tags/b/|", + ) + b.AssertFileContentEquals("public/fr/p1/index.html", + "P1 (fr)|TITLE: a (fr) RELPERMALINK: /fr/tags/a/|TITLE: b (fr) RELPERMALINK: /fr/tags/b/|", + ) + b.AssertFileContentEquals("public/en/tags/index.html", + "tags|TITLE: a (en) RELPERMALINK: /en/tags/a/|TITLE: b (en) RELPERMALINK: /en/tags/b/|", + ) + b.AssertFileContentEquals("public/fr/tags/index.html", + "tags|TITLE: a (fr) RELPERMALINK: /fr/tags/a/|TITLE: b (fr) RELPERMALINK: /fr/tags/b/|", + ) + b.AssertFileContentEquals("public/en/tags/a/index.html", + "a (en)|TITLE: P1 (en) RELPERMALINK: /en/p1/|", + ) + b.AssertFileContentEquals("public/fr/tags/a/index.html", + "a (fr)|TITLE: P1 (fr) RELPERMALINK: /fr/p1/|", + ) + b.AssertFileContentEquals("public/en/tags/b/index.html", + "b (en)|TITLE: P1 (en) RELPERMALINK: /en/p1/|", + ) + b.AssertFileContentEquals("public/fr/tags/b/index.html", + "b (fr)|TITLE: P1 (fr) RELPERMALINK: /fr/p1/|", + ) + + // Set draft to true on content/tags/b/_index.fr.md. + files = strings.ReplaceAll(files, "draft: false", "draft: true") + + b = hugolib.Test(t, files) + + b.AssertFileExists("public/en/tags/a/index.html", true) + b.AssertFileExists("public/fr/tags/a/index.html", true) + b.AssertFileExists("public/en/tags/b/index.html", true) + b.AssertFileExists("public/fr/tags/b/index.html", false) + + // The assertion below fails. + // Got: P1 (en)|TITLE: a (en) RELPERMALINK: /en/tags/a/| + b.AssertFileContentEquals("public/en/p1/index.html", + "P1 (en)|TITLE: a (en) RELPERMALINK: /en/tags/a/|TITLE: b (en) RELPERMALINK: /en/tags/b/|", + ) + b.AssertFileContentEquals("public/fr/p1/index.html", + "P1 (fr)|TITLE: a (fr) RELPERMALINK: /fr/tags/a/|", + ) + b.AssertFileContentEquals("public/en/tags/index.html", + "tags|TITLE: a (en) RELPERMALINK: /en/tags/a/|TITLE: b (en) RELPERMALINK: /en/tags/b/|", + ) + b.AssertFileContentEquals("public/fr/tags/index.html", + "tags|TITLE: a (fr) RELPERMALINK: /fr/tags/a/|", + ) + b.AssertFileContentEquals("public/en/tags/a/index.html", + "a (en)|TITLE: P1 (en) RELPERMALINK: /en/p1/|", + ) + b.AssertFileContentEquals("public/fr/tags/a/index.html", + "a (fr)|TITLE: P1 (fr) RELPERMALINK: /fr/p1/|", + ) + // The assertion below fails. + // Got: b (en)| + b.AssertFileContentEquals("public/en/tags/b/index.html", + "b (en)|TITLE: P1 (en) RELPERMALINK: /en/p1/|", + ) +} diff --git a/lazy/init.go b/lazy/init.go deleted file mode 100644 index bef3867a9..000000000 --- a/lazy/init.go +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package lazy - -import ( - "context" - "errors" - "sync" - "sync/atomic" - "time" -) - -// New creates a new empty Init. -func New() *Init { - return &Init{} -} - -// Init holds a graph of lazily initialized dependencies. -type Init struct { - // Used mainly for testing. - initCount uint64 - - mu sync.Mutex - - prev *Init - children []*Init - - init OnceMore - out any - err error - f func(context.Context) (any, error) -} - -// Add adds a func as a new child dependency. -func (ini *Init) Add(initFn func(context.Context) (any, error)) *Init { - if ini == nil { - ini = New() - } - return ini.add(false, initFn) -} - -// InitCount gets the number of this this Init has been initialized. -func (ini *Init) InitCount() int { - i := atomic.LoadUint64(&ini.initCount) - return int(i) -} - -// AddWithTimeout is same as Add, but with a timeout that aborts initialization. -func (ini *Init) AddWithTimeout(timeout time.Duration, f func(ctx context.Context) (any, error)) *Init { - return ini.Add(func(ctx context.Context) (any, error) { - return ini.withTimeout(ctx, timeout, f) - }) -} - -// Branch creates a new dependency branch based on an existing and adds -// the given dependency as a child. -func (ini *Init) Branch(initFn func(context.Context) (any, error)) *Init { - if ini == nil { - ini = New() - } - return ini.add(true, initFn) -} - -// BranchWithTimeout is same as Branch, but with a timeout. -func (ini *Init) BranchWithTimeout(timeout time.Duration, f func(ctx context.Context) (any, error)) *Init { - return ini.Branch(func(ctx context.Context) (any, error) { - return ini.withTimeout(ctx, timeout, f) - }) -} - -// Do initializes the entire dependency graph. -func (ini *Init) Do(ctx context.Context) (any, error) { - if ini == nil { - panic("init is nil") - } - - ini.init.Do(func() { - atomic.AddUint64(&ini.initCount, 1) - prev := ini.prev - if prev != nil { - // A branch. Initialize the ancestors. - if prev.shouldInitialize() { - _, err := prev.Do(ctx) - if err != nil { - ini.err = err - return - } - } else if prev.inProgress() { - // Concurrent initialization. The following init func - // may depend on earlier state, so wait. - prev.wait() - } - } - - if ini.f != nil { - ini.out, ini.err = ini.f(ctx) - } - - for _, child := range ini.children { - if child.shouldInitialize() { - _, err := child.Do(ctx) - if err != nil { - ini.err = err - return - } - } - } - }) - - ini.wait() - - return ini.out, ini.err -} - -// TODO(bep) investigate if we can use sync.Cond for this. -func (ini *Init) wait() { - var counter time.Duration - for !ini.init.Done() { - counter += 10 - if counter > 600000000 { - panic("BUG: timed out in lazy init") - } - time.Sleep(counter * time.Microsecond) - } -} - -func (ini *Init) inProgress() bool { - return ini != nil && ini.init.InProgress() -} - -func (ini *Init) shouldInitialize() bool { - return !(ini == nil || ini.init.Done() || ini.init.InProgress()) -} - -// Reset resets the current and all its dependencies. -func (ini *Init) Reset() { - mu := ini.init.ResetWithLock() - ini.err = nil - defer mu.Unlock() - for _, d := range ini.children { - d.Reset() - } -} - -func (ini *Init) add(branch bool, initFn func(context.Context) (any, error)) *Init { - ini.mu.Lock() - defer ini.mu.Unlock() - - if branch { - return &Init{ - f: initFn, - prev: ini, - } - } - - ini.checkDone() - ini.children = append(ini.children, &Init{ - f: initFn, - }) - - return ini -} - -func (ini *Init) checkDone() { - if ini.init.Done() { - panic("init cannot be added to after it has run") - } -} - -func (ini *Init) withTimeout(ctx context.Context, timeout time.Duration, f func(ctx context.Context) (any, error)) (any, error) { - // Create a new context with a timeout not connected to the incoming context. - waitCtx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - c := make(chan verr, 1) - - go func() { - v, err := f(ctx) - select { - case <-waitCtx.Done(): - return - default: - c <- verr{v: v, err: err} - } - }() - - select { - case <-waitCtx.Done(): - //lint:ignore ST1005 end user message. - return nil, errors.New("timed out initializing value. You may have a circular loop in a shortcode, or your site may have resources that take longer to build than the `timeout` limit in your Hugo config file.") - case ve := <-c: - return ve.v, ve.err - } -} - -type verr struct { - v any - err error -} diff --git a/lazy/init_test.go b/lazy/init_test.go deleted file mode 100644 index 94736fab8..000000000 --- a/lazy/init_test.go +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package lazy - -import ( - "context" - "errors" - "math/rand" - "strings" - "sync" - "testing" - "time" - - qt "github.com/frankban/quicktest" -) - -var ( - rnd = rand.New(rand.NewSource(time.Now().UnixNano())) - bigOrSmall = func() int { - if rnd.Intn(10) < 5 { - return 10000 + rnd.Intn(100000) - } - return 1 + rnd.Intn(50) - } -) - -func doWork() { - doWorkOfSize(bigOrSmall()) -} - -func doWorkOfSize(size int) { - _ = strings.Repeat("Hugo Rocks! ", size) -} - -func TestInit(t *testing.T) { - c := qt.New(t) - - var result string - - f1 := func(name string) func(context.Context) (any, error) { - return func(context.Context) (any, error) { - result += name + "|" - doWork() - return name, nil - } - } - - f2 := func() func(context.Context) (any, error) { - return func(context.Context) (any, error) { - doWork() - return nil, nil - } - } - - root := New() - - root.Add(f1("root(1)")) - root.Add(f1("root(2)")) - - branch1 := root.Branch(f1("branch_1")) - branch1.Add(f1("branch_1_1")) - branch1_2 := branch1.Add(f1("branch_1_2")) - branch1_2_1 := branch1_2.Add(f1("branch_1_2_1")) - - var wg sync.WaitGroup - - ctx := context.Background() - - // Add some concurrency and randomness to verify thread safety and - // init order. - for i := range 100 { - wg.Add(1) - go func(i int) { - defer wg.Done() - var err error - if rnd.Intn(10) < 5 { - _, err = root.Do(ctx) - c.Assert(err, qt.IsNil) - } - - // Add a new branch on the fly. - if rnd.Intn(10) > 5 { - branch := branch1_2.Branch(f2()) - _, err = branch.Do(ctx) - c.Assert(err, qt.IsNil) - } else { - _, err = branch1_2_1.Do(ctx) - c.Assert(err, qt.IsNil) - } - _, err = branch1_2.Do(ctx) - c.Assert(err, qt.IsNil) - }(i) - - wg.Wait() - - c.Assert(result, qt.Equals, "root(1)|root(2)|branch_1|branch_1_1|branch_1_2|branch_1_2_1|") - - } -} - -func TestInitAddWithTimeout(t *testing.T) { - c := qt.New(t) - - init := New().AddWithTimeout(100*time.Millisecond, func(ctx context.Context) (any, error) { - return nil, nil - }) - - _, err := init.Do(context.Background()) - - c.Assert(err, qt.IsNil) -} - -func TestInitAddWithTimeoutTimeout(t *testing.T) { - c := qt.New(t) - - init := New().AddWithTimeout(100*time.Millisecond, func(ctx context.Context) (any, error) { - time.Sleep(500 * time.Millisecond) - return nil, nil - }) - - _, err := init.Do(context.Background()) - - c.Assert(err, qt.Not(qt.IsNil)) - - c.Assert(err.Error(), qt.Contains, "timed out") - - time.Sleep(1 * time.Second) -} - -func TestInitAddWithTimeoutError(t *testing.T) { - c := qt.New(t) - - init := New().AddWithTimeout(100*time.Millisecond, func(ctx context.Context) (any, error) { - return nil, errors.New("failed") - }) - - _, err := init.Do(context.Background()) - - c.Assert(err, qt.Not(qt.IsNil)) -} - -type T struct { - sync.Mutex - V1 string - V2 string -} - -func (t *T) Add1(v string) { - t.Lock() - t.V1 += v - t.Unlock() -} - -func (t *T) Add2(v string) { - t.Lock() - t.V2 += v - t.Unlock() -} - -// https://github.com/gohugoio/hugo/issues/5901 -func TestInitBranchOrder(t *testing.T) { - c := qt.New(t) - - base := New() - - work := func(size int, f func()) func(context.Context) (any, error) { - return func(context.Context) (any, error) { - doWorkOfSize(size) - if f != nil { - f() - } - - return nil, nil - } - } - - state := &T{} - - base = base.Add(work(10000, func() { - state.Add1("A") - })) - - inits := make([]*Init, 2) - for i := range inits { - inits[i] = base.Branch(work(i+1*100, func() { - // V1 is A - ab := state.V1 + "B" - state.Add2(ab) - })) - } - - var wg sync.WaitGroup - ctx := context.Background() - - for _, v := range inits { - v := v - wg.Add(1) - go func() { - defer wg.Done() - _, err := v.Do(ctx) - c.Assert(err, qt.IsNil) - }() - } - - wg.Wait() - - c.Assert(state.V2, qt.Equals, "ABAB") -} - -// See issue 7043 -func TestResetError(t *testing.T) { - c := qt.New(t) - r := false - i := New().Add(func(context.Context) (any, error) { - if r { - return nil, nil - } - return nil, errors.New("r is false") - }) - _, err := i.Do(context.Background()) - c.Assert(err, qt.IsNotNil) - i.Reset() - r = true - _, err = i.Do(context.Background()) - c.Assert(err, qt.IsNil) -} diff --git a/lazy/once.go b/lazy/once.go deleted file mode 100644 index dac689df3..000000000 --- a/lazy/once.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package lazy - -import ( - "sync" - "sync/atomic" -) - -// OnceMore is similar to sync.Once. -// -// Additional features are: -// * it can be reset, so the action can be repeated if needed -// * it has methods to check if it's done or in progress - -type OnceMore struct { - mu sync.Mutex - lock uint32 - done uint32 -} - -func (t *OnceMore) Do(f func()) { - if atomic.LoadUint32(&t.done) == 1 { - return - } - - // f may call this Do and we would get a deadlock. - locked := atomic.CompareAndSwapUint32(&t.lock, 0, 1) - if !locked { - return - } - defer atomic.StoreUint32(&t.lock, 0) - - t.mu.Lock() - defer t.mu.Unlock() - - // Double check - if t.done == 1 { - return - } - defer atomic.StoreUint32(&t.done, 1) - f() -} - -func (t *OnceMore) InProgress() bool { - return atomic.LoadUint32(&t.lock) == 1 -} - -func (t *OnceMore) Done() bool { - return atomic.LoadUint32(&t.done) == 1 -} - -func (t *OnceMore) ResetWithLock() *sync.Mutex { - t.mu.Lock() - defer atomic.StoreUint32(&t.done, 0) - return &t.mu -} diff --git a/markup/goldmark/codeblocks/codeblocks_integration_test.go b/markup/goldmark/codeblocks/codeblocks_integration_test.go index 97f4eb384..2fd4fa170 100644 --- a/markup/goldmark/codeblocks/codeblocks_integration_test.go +++ b/markup/goldmark/codeblocks/codeblocks_integration_test.go @@ -87,9 +87,11 @@ echo "l8"; §§§ ` - b := hugolib.Test(t, files) + for range 6 { - b.AssertFileContent("public/p1/index.html", ` + b := hugolib.Test(t, files) + + b.AssertFileContent("public/p1/index.html", ` Goat SVG:Go Code\nGo Code: fmt.Println(\"Hello, World!\");\n|\nGo Language: go|", - "

Golang Code

\nGo Code: fmt.Println(\"Hello, Golang!\");\n|\nGo Language: go|", - "

Bash Code

\n
32echo "l1";\n33",
-	)
+			"Goat SVG:Go Code\nGo Code: fmt.Println(\"Hello, World!\");\n|\nGo Language: go|",
+			"
Golang Code
\nGo Code: fmt.Println(\"Hello, Golang!\");\n|\nGo Language: go|",
+			"
Bash Code
\n
32echo "l1";\n33",
+		)
+	}
 }
 
 func TestHighlightCodeblock(t *testing.T) {
diff --git a/markup/goldmark/tables/tables_integration_test.go b/markup/goldmark/tables/tables_integration_test.go
index 36cf953ae..d42f1195c 100644
--- a/markup/goldmark/tables/tables_integration_test.go
+++ b/markup/goldmark/tables/tables_integration_test.go
@@ -56,18 +56,21 @@ Attributes: {{ .Attributes }}|
 {{ end }}
 
 `
-	b := hugolib.Test(t, files)
 
-	b.AssertFileContent("public/p1/index.html",
-		"Attributes: map[class:foo foo:bar]|",
-		"table-0-thead: 0: 0: left: Item| 1: center: In Stock| 2: right: Price|$",
-		"table-0-tbody: 0: 0: left: Python Hat| 1: center: True| 2: right: 23.99| 1: 0: left: SQL Hat| 1: center: True| 2: right: 23.99| 2: 0: left: Codecademy Tee| 1: center: False| 2: right: 19.99| 3: 0: left: Codecademy Hoodie| 1: center: False| 2: right: 42.99|$",
-	)
+	for range 2 {
+		b := hugolib.Test(t, files)
+
+		b.AssertFileContent("public/p1/index.html",
+			"Attributes: map[class:foo foo:bar]|",
+			"table-0-thead: 0: 0: left: Item| 1: center: In Stock| 2: right: Price|$",
+			"table-0-tbody: 0: 0: left: Python Hat| 1: center: True| 2: right: 23.99| 1: 0: left: SQL Hat| 1: center: True| 2: right: 23.99| 2: 0: left: Codecademy Tee| 1: center: False| 2: right: 19.99| 3: 0: left: Codecademy Hoodie| 1: center: False| 2: right: 42.99|$",
+		)
 
-	b.AssertFileContent("public/p1/index.html",
-		"table-1-thead: 0: 0: : Month| 1: : Savings|$",
-		"table-1-tbody: 0: 0: : January| 1: : $250| 1: 0: : February| 1: : $80| 2: 0: : March| 1: : $420|$",
-	)
+		b.AssertFileContent("public/p1/index.html",
+			"table-1-thead: 0: 0: : Month| 1: : Savings|$",
+			"table-1-tbody: 0: 0: : January| 1: : $250| 1: 0: : February| 1: : $80| 2: 0: : March| 1: : $420|$",
+		)
+	}
 }
 
 func TestTableDefault(t *testing.T) {
diff --git a/modules/client.go b/modules/client.go
index 09adeab90..c3d568188 100644
--- a/modules/client.go
+++ b/modules/client.go
@@ -35,7 +35,7 @@ import (
 	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/config"
 
-	hglob "github.com/gohugoio/hugo/hugofs/glob"
+	hglob "github.com/gohugoio/hugo/hugofs/hglob"
 
 	"github.com/gobwas/glob"
 
diff --git a/modules/client_test.go b/modules/client_test.go
index 78fdb9237..26b9fcacd 100644
--- a/modules/client_test.go
+++ b/modules/client_test.go
@@ -24,7 +24,7 @@ import (
 	"github.com/gohugoio/hugo/common/hexec"
 	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/config/security"
-	"github.com/gohugoio/hugo/hugofs/glob"
+	hglob "github.com/gohugoio/hugo/hugofs/hglob"
 
 	"github.com/gohugoio/hugo/htesting"
 
@@ -156,7 +156,7 @@ project github.com/gohugoio/hugoTestModules1_darwin/modh2_2_2@v1.3.0+vendor
 			c, func(cfg *ClientConfig) {
 				cfg.ModuleConfig = mcfg
 				s := "github.com/gohugoio/hugoTestModules1_darwin/modh1_1v"
-				g, _ := glob.GetGlob(s)
+				g, _ := hglob.GetGlob(s)
 				cfg.IgnoreVendor = g
 			}, "modh1v")
 		defer clean()
@@ -202,7 +202,7 @@ project github.com/gohugoio/hugoTestModules1_darwin/modh2_2_2@v1.3.0+vendor
 	})
 }
 
-var globAll, _ = glob.GetGlob("**")
+var globAll, _ = hglob.GetGlob("**")
 
 func TestGetModlineSplitter(t *testing.T) {
 	c := qt.New(t)
diff --git a/modules/collect.go b/modules/collect.go
index 560c61d6a..bf027b372 100644
--- a/modules/collect.go
+++ b/modules/collect.go
@@ -29,13 +29,13 @@ import (
 	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/common/paths"
+	"github.com/gohugoio/hugo/common/version"
 	"golang.org/x/mod/module"
 
 	"github.com/spf13/cast"
 
 	"github.com/gohugoio/hugo/common/maps"
 
-	"github.com/gohugoio/hugo/common/hugo"
 	"github.com/gohugoio/hugo/parser/metadecoders"
 
 	"github.com/gohugoio/hugo/hugofs/files"
@@ -64,7 +64,7 @@ func (h *Client) Collect() (ModulesConfig, error) {
 		}
 	}
 
-	if err := (&mc).finalize(h.logger); err != nil {
+	if err := (&mc).finalize(); err != nil {
 		return mc, err
 	}
 
@@ -134,25 +134,31 @@ func (m *ModulesConfig) setActiveMods(logger loggers.Logger) error {
 	return nil
 }
 
-func (m *ModulesConfig) finalize(logger loggers.Logger) error {
+func (m *ModulesConfig) finalize() error {
 	for _, mod := range m.AllModules {
 		m := mod.(*moduleAdapter)
-		m.mounts = filterUnwantedMounts(m.mounts)
+		m.mounts = filterDuplicateMounts(m.mounts)
 	}
 	return nil
 }
 
-func filterUnwantedMounts(mounts []Mount) []Mount {
-	// Remove duplicates
-	seen := make(map[string]bool)
-	tmp := mounts[:0]
-	for _, m := range mounts {
-		if !seen[m.key()] {
-			tmp = append(tmp, m)
+// filterDuplicateMounts removes duplicate mounts while preserving order.
+// The input slice will not be modified.
+func filterDuplicateMounts(mounts []Mount) []Mount {
+	var x []Mount
+	for _, m1 := range mounts {
+		var found bool
+		for _, m2 := range x {
+			if m1.Equal(m2) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			x = append(x, m1)
 		}
-		seen[m.key()] = true
 	}
-	return tmp
+	return x
 }
 
 type pathVersionKey struct {
@@ -525,7 +531,7 @@ LOOP:
 		}
 	}
 
-	config, err := decodeConfig(tc.cfg, c.moduleConfig.replacementsMap)
+	config, err := decodeConfig(c.logger.Logger(), tc.cfg, c.moduleConfig.replacementsMap)
 	if err != nil {
 		return err
 	}
@@ -537,7 +543,7 @@ LOOP:
 		// Merge old with new
 		if minVersion, found := themeCfg[oldVersionKey]; found {
 			if config.HugoVersion.Min == "" {
-				config.HugoVersion.Min = hugo.VersionString(cast.ToString(minVersion))
+				config.HugoVersion.Min = version.VersionString(cast.ToString(minVersion))
 			}
 		}
 
@@ -790,6 +796,10 @@ func (c *collector) normalizeMounts(owner *moduleAdapter, mounts []Mount) ([]Mou
 			return nil, fmt.Errorf("%s: mount target must be one of: %v", errMsg, files.ComponentFolders)
 		}
 
+		if err := mnt.init(c.logger.Logger()); err != nil {
+			return nil, fmt.Errorf("%s: %w", errMsg, err)
+		}
+
 		out = append(out, mnt)
 	}
 
diff --git a/modules/collect_test.go b/modules/collect_test.go
index 5debda28c..0f0dc3a0c 100644
--- a/modules/collect_test.go
+++ b/modules/collect_test.go
@@ -43,7 +43,7 @@ func TestFilterUnwantedMounts(t *testing.T) {
 		{Source: "b", Target: "c", Lang: "en"},
 	}
 
-	filtered := filterUnwantedMounts(mounts)
+	filtered := filterDuplicateMounts(mounts)
 
 	c := qt.New(t)
 	c.Assert(len(filtered), qt.Equals, 2)
diff --git a/modules/config.go b/modules/config.go
index b6a825416..a3e8256fe 100644
--- a/modules/config.go
+++ b/modules/config.go
@@ -17,10 +17,18 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 
+	"github.com/bep/logg"
+	"github.com/gohugoio/hugo/common/hstrings"
 	"github.com/gohugoio/hugo/common/hugo"
+	"github.com/gohugoio/hugo/common/types"
+	"github.com/gohugoio/hugo/common/version"
 	"github.com/gohugoio/hugo/hugofs/files"
+	hglob "github.com/gohugoio/hugo/hugofs/hglob"
+	"github.com/gohugoio/hugo/hugolib/sitesmatrix"
+	"github.com/gohugoio/hugo/langs"
 
 	"github.com/gohugoio/hugo/config"
 	"github.com/mitchellh/mapstructure"
@@ -56,7 +64,7 @@ var DefaultModuleConfig = Config{
 
 // ApplyProjectConfigDefaults applies default/missing module configuration for
 // the main project.
-func ApplyProjectConfigDefaults(mod Module, cfgs ...config.AllProvider) error {
+func ApplyProjectConfigDefaults(logger logg.Logger, mod Module, cfgs ...config.AllProvider) error {
 	moda := mod.(*moduleAdapter)
 
 	// To bridge between old and new configuration format we need
@@ -127,49 +135,50 @@ func ApplyProjectConfigDefaults(mod Module, cfgs ...config.AllProvider) error {
 			}
 
 			var lang string
+			var sites sitesmatrix.Sites
+
 			if perLang && !dropLang {
-				lang = cfg.Language().Lang
+				l := cfg.Language().(*langs.Language)
+				lang = l.Lang
+				sites = sitesmatrix.Sites{
+					Matrix: sitesmatrix.StringSlices{
+						Languages: []string{l.Lang},
+					},
+				}
 			}
 
 			// Static mounts are a little special.
 			if component == files.ComponentFolderStatic {
 				staticDirs := cfg.StaticDirs()
 				for _, dir := range staticDirs {
-					mounts = append(mounts, Mount{Lang: lang, Source: dir, Target: component})
+					mounts = append(mounts, Mount{Sites: sites, Source: dir, Target: component})
 				}
 				continue
 			}
 
 			if dir != "" {
-				mounts = append(mounts, Mount{Lang: lang, Source: dir, Target: component})
+				mnt := Mount{Lang: lang, Source: dir, Target: component}
+				if err := mnt.init(logger); err != nil {
+					return fmt.Errorf("failed to init mount %q %d: %w", lang, i, err)
+				}
+				mounts = append(mounts, mnt)
 			}
 		}
 	}
 
 	moda.mounts = append(moda.mounts, mounts...)
 
-	// Temporary: Remove duplicates.
-	seen := make(map[string]bool)
-	var newMounts []Mount
-	for _, m := range moda.mounts {
-		key := m.Source + m.Target + m.Lang
-		if seen[key] {
-			continue
-		}
-		seen[key] = true
-		newMounts = append(newMounts, m)
-	}
-	moda.mounts = newMounts
+	moda.mounts = filterDuplicateMounts(moda.mounts)
 
 	return nil
 }
 
 // DecodeConfig creates a modules Config from a given Hugo configuration.
-func DecodeConfig(cfg config.Provider) (Config, error) {
-	return decodeConfig(cfg, nil)
+func DecodeConfig(logger logg.Logger, cfg config.Provider) (Config, error) {
+	return decodeConfig(logger, cfg, nil)
 }
 
-func decodeConfig(cfg config.Provider, pathReplacements map[string]string) (Config, error) {
+func decodeConfig(logger logg.Logger, cfg config.Provider, pathReplacements map[string]string) (Config, error) {
 	c := DefaultModuleConfig
 	c.replacementsMap = pathReplacements
 
@@ -220,6 +229,9 @@ func decodeConfig(cfg config.Provider, pathReplacements map[string]string) (Conf
 		for i, mnt := range c.Mounts {
 			mnt.Source = filepath.Clean(mnt.Source)
 			mnt.Target = filepath.Clean(mnt.Target)
+			if err := mnt.init(logger); err != nil {
+				return c, fmt.Errorf("failed to init mount %d: %w", i, err)
+			}
 			c.Mounts[i] = mnt
 		}
 
@@ -323,10 +335,10 @@ func (c Config) hasModuleImport() bool {
 // HugoVersion holds Hugo binary version requirements for a module.
 type HugoVersion struct {
 	// The minimum Hugo version that this module works with.
-	Min hugo.VersionString
+	Min version.VersionString
 
 	// The maximum Hugo version that this module works with.
-	Max hugo.VersionString
+	Max version.VersionString
 
 	// Set if the extended version is needed.
 	Extended bool
@@ -408,19 +420,68 @@ type Mount struct {
 	// Any file in this mount will be associated with this language.
 	Lang string
 
+	// Sites defines which sites this mount applies to.
+	Sites sitesmatrix.Sites
+
+	// A slice of Glob patterns (string or slice) to exclude or include in this mount.
+	// To exclude, prefix with "! ".
+	Files []string
+
 	// Include only files matching the given Glob patterns (string or slice).
+	// Deprecated, use Files instead.
 	IncludeFiles any
 
 	// Exclude all files matching the given Glob patterns (string or slice).
+	// Deprecated, use Files instead.
 	ExcludeFiles any
 
 	// Disable watching in watch mode for this mount.
 	DisableWatch bool
 }
 
-// Used as key to remove duplicates.
-func (m Mount) key() string {
-	return strings.Join([]string{m.Lang, m.Source, m.Target}, "/")
+func (m Mount) Equal(o Mount) bool {
+	if m.Lang != o.Lang {
+		return false
+	}
+	if m.Source != o.Source {
+		return false
+	}
+	if m.Target != o.Target {
+		return false
+	}
+	if m.DisableWatch != o.DisableWatch {
+		return false
+	}
+
+	if !m.Sites.Equal(o.Sites) {
+		return false
+	}
+
+	patterns, hasLegacy11, hasLegacy12 := m.FilesToFilter()
+	patterns2, hasLegacy21, hasLegacy22 := o.FilesToFilter()
+
+	if hasLegacy11 != hasLegacy21 || hasLegacy12 != hasLegacy22 {
+		return false
+	}
+
+	return slices.Equal(patterns, patterns2)
+}
+
+func (m Mount) FilesToFilter() (patterns []string, hasLegacyIncludeFiles, hasLegacyExcludeFiles bool) {
+	patterns = m.Files
+
+	// Legacy config, add IncludeFiles first.
+	for _, pattern := range types.ToStringSlicePreserveString(m.IncludeFiles) {
+		hasLegacyIncludeFiles = true
+		patterns = append(patterns, pattern)
+	}
+
+	for _, pattern := range types.ToStringSlicePreserveString(m.ExcludeFiles) {
+		hasLegacyExcludeFiles = true
+		patterns = append(patterns, hglob.NegationPrefix+pattern)
+	}
+
+	return patterns, hasLegacyIncludeFiles, hasLegacyExcludeFiles
 }
 
 func (m Mount) Component() string {
@@ -431,3 +492,17 @@ func (m Mount) ComponentAndName() (string, string) {
 	c, n, _ := strings.Cut(m.Target, fileSeparator)
 	return c, n
 }
+
+func (m *Mount) init(logger logg.Logger) error {
+	if m.Lang != "" {
+		// We moved this to a more flixeble setup in Hugo 0.153.0.
+		m.Sites.Matrix.Languages = append(m.Sites.Matrix.Languages, m.Lang)
+		m.Lang = ""
+
+		hugo.DeprecateWithLogger("module.mounts.lang", "Replaced by the more powerful 'sites.matrix' setting, see https://gohugo.io/configuration/module/#mounts", "v0.153.0", logger)
+	}
+
+	m.Sites.Matrix.Languages = hstrings.UniqueStringsReuse(m.Sites.Matrix.Languages)
+
+	return nil
+}
diff --git a/modules/config_integration_test.go b/modules/config_integration_test.go
new file mode 100644
index 000000000..7c540cf02
--- /dev/null
+++ b/modules/config_integration_test.go
@@ -0,0 +1,69 @@
+// Copyright 2025 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules_test
+
+import (
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+	"github.com/gohugoio/hugo/hugolib"
+)
+
+func TestMountsProjectDefaults(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+[languages]
+[languages.en]
+weight = 1
+[languages.sv]
+weight = 2
+
+[module]
+[[module.mounts]]
+source = 'content'
+target = 'content'
+lang = 'en'
+-- content/p1.md --
+
+`
+	b := hugolib.Test(t, files)
+
+	b.Assert(len(b.H.Configs.Modules), qt.Equals, 1)
+	projectMod := b.H.Configs.Modules[0]
+	b.Assert(projectMod.Path(), qt.Equals, "project")
+	mounts := projectMod.Mounts()
+	b.Assert(len(mounts), qt.Equals, 7)
+	contentMount := mounts[0]
+	b.Assert(contentMount.Source, qt.Equals, "content")
+	b.Assert(contentMount.Sites.Matrix.Languages, qt.DeepEquals, []string{"en"})
+}
+
+func TestMountsLangIsDeprecated(t *testing.T) {
+	t.Parallel()
+	files := `
+-- hugo.toml --
+[module]
+[[module.mounts]]
+source = 'content'
+target = 'content'
+lang = 'en'
+-- layouts/all.html --
+All.
+`
+
+	b := hugolib.Test(t, files, hugolib.TestOptInfo())
+	b.AssertLogContains("deprecated")
+}
diff --git a/modules/config_test.go b/modules/config_test.go
index 6dac14e9a..42f7f629e 100644
--- a/modules/config_test.go
+++ b/modules/config_test.go
@@ -20,6 +20,8 @@ import (
 	"testing"
 
 	"github.com/gohugoio/hugo/common/hugo"
+	"github.com/gohugoio/hugo/common/loggers"
+	"github.com/gohugoio/hugo/common/version"
 
 	"github.com/gohugoio/hugo/config"
 
@@ -76,10 +78,10 @@ lang="en"
 		cfg, err := config.FromConfigString(fmt.Sprintf(tomlConfig, tempDir), "toml")
 		c.Assert(err, qt.IsNil)
 
-		mcfg, err := DecodeConfig(cfg)
+		mcfg, err := DecodeConfig(loggers.NewDefault().Logger(), cfg)
 		c.Assert(err, qt.IsNil)
 
-		v056 := hugo.VersionString("0.56.0")
+		v056 := version.VersionString("0.56.0")
 
 		hv := mcfg.HugoVersion
 
@@ -118,7 +120,7 @@ path="github.com/bep/mycomponent"
 			cfg, err := config.FromConfigString(tomlConfig, "toml")
 			c.Assert(err, qt.IsNil)
 
-			mcfg, err := DecodeConfig(cfg)
+			mcfg, err := DecodeConfig(loggers.NewDefault().Logger(), cfg)
 			c.Assert(err, qt.IsNil)
 			c.Assert(mcfg.Replacements, qt.DeepEquals, []string{"a->b", "github.com/bep/mycomponent->c"})
 			c.Assert(mcfg.replacementsMap, qt.DeepEquals, map[string]string{
@@ -146,7 +148,7 @@ path="a"
 	cfg, err := config.FromConfigString(tomlConfig, "toml")
 	c.Assert(err, qt.IsNil)
 
-	modCfg, err := DecodeConfig(cfg)
+	modCfg, err := DecodeConfig(loggers.NewDefault().Logger(), cfg)
 	c.Assert(err, qt.IsNil)
 	c.Assert(len(modCfg.Imports), qt.Equals, 3)
 	c.Assert(modCfg.Imports[0].Path, qt.Equals, "a")
@@ -162,7 +164,7 @@ theme = ["a", "b"]
 	cfg, err := config.FromConfigString(tomlConfig, "toml")
 	c.Assert(err, qt.IsNil)
 
-	mcfg, err := DecodeConfig(cfg)
+	mcfg, err := DecodeConfig(loggers.NewDefault().Logger(), cfg)
 	c.Assert(err, qt.IsNil)
 
 	c.Assert(len(mcfg.Imports), qt.Equals, 2)
diff --git a/modules/modules_integration_test.go b/modules/modules_integration_test.go
index 6f09a35e0..d3cfc4496 100644
--- a/modules/modules_integration_test.go
+++ b/modules/modules_integration_test.go
@@ -38,10 +38,9 @@ target = "content/v1"
 Title: {{ .Title }}|Summary: {{ .Summary }}|
 Deps: {{ range hugo.Deps}}{{ printf "%s@%s" .Path .Version }}|{{ end }}$
 
-
 `
 
-	b := hugolib.Test(t, files, hugolib.TestOptWithOSFs()).Build()
+	b := hugolib.Test(t, files, hugolib.TestOptWithOSFs())
 
 	b.AssertFileContent("public/index.html", "Deps: project@|github.com/bep/hugo-mod-misc/dummy-content@v0.2.0|github.com/bep/hugo-mod-misc/dummy-content@v0.1.0|$")
 
diff --git a/modules/mount_filters_integration_test.go b/modules/mount_filters_integration_test.go
new file mode 100644
index 000000000..9a057a316
--- /dev/null
+++ b/modules/mount_filters_integration_test.go
@@ -0,0 +1,96 @@
+// Copyright 2025 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules_test
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/hugolib"
+)
+
+func TestModulesFiltersFiles(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+[module]
+[[module.mounts]]
+source = "content"
+target = "content"
+files = ["! **b.md", "**.md"] # The order matters here.
+-- content/a.md --
++++
+title = "A"
++++
+Content A
+-- content/b.md --
++++
+title = "B"
++++
+Content B
+-- content/c.md --
++++
+title = "C"
++++
+Content C
+-- layouts/all.html --
+All. {{ .Title }}|
+
+`
+
+	b := hugolib.Test(t, files, hugolib.TestOptInfo())
+	b.AssertLogContains("! deprecated")
+	b.AssertFileContent("public/a/index.html", "All. A|")
+	b.AssertFileContent("public/c/index.html", "All. C|")
+	b.AssertFileExists("public/b/index.html", false)
+}
+
+// File filter format <= 0.152.0.
+func TestModulesFiltersFilesLegacy(t *testing.T) {
+	// This cannot be parallel.
+
+	files := `
+-- hugo.toml --
+[module]
+[[module.mounts]]
+source = "content"
+target = "content"
+includefiles = ["**{a,c}.md"] #includes was evaluated before excludes <= 0.152.0
+excludefiles = ["**b.md"]
+-- content/a.md --
++++
+title = "A"
++++
+Content A
+-- content/b.md --
++++
+title = "B"
++++
+Content B
+-- content/c.md --
++++
+title = "C"
++++
+Content C
+-- layouts/all.html --
+All. {{ .Title }}|
+
+`
+
+	b := hugolib.Test(t, files, hugolib.TestOptInfo())
+
+	b.AssertFileContent("public/a/index.html", "All. A|")
+	b.AssertFileContent("public/c/index.html", "All. C|")
+	b.AssertFileExists("public/b/index.html", false)
+}
diff --git a/publisher/htmlElementsCollector.go b/publisher/htmlElementsCollector.go
index 11a640550..43eb14fcd 100644
--- a/publisher/htmlElementsCollector.go
+++ b/publisher/htmlElementsCollector.go
@@ -24,8 +24,8 @@ import (
 
 	"golang.org/x/net/html"
 
+	"github.com/gohugoio/hugo/common/hstrings"
 	"github.com/gohugoio/hugo/config"
-	"github.com/gohugoio/hugo/helpers"
 )
 
 const eof = -1
@@ -77,9 +77,9 @@ func (h *HTMLElements) Merge(other HTMLElements) {
 	h.Classes = append(h.Classes, other.Classes...)
 	h.IDs = append(h.IDs, other.IDs...)
 
-	h.Tags = helpers.UniqueStringsReuse(h.Tags)
-	h.Classes = helpers.UniqueStringsReuse(h.Classes)
-	h.IDs = helpers.UniqueStringsReuse(h.IDs)
+	h.Tags = hstrings.UniqueStringsReuse(h.Tags)
+	h.Classes = hstrings.UniqueStringsReuse(h.Classes)
+	h.IDs = hstrings.UniqueStringsReuse(h.IDs)
 }
 
 func (h *HTMLElements) Sort() {
@@ -122,9 +122,9 @@ func (c *htmlElementsCollector) getHTMLElements() HTMLElements {
 		}
 	}
 
-	classes = helpers.UniqueStringsSorted(classes)
-	ids = helpers.UniqueStringsSorted(ids)
-	tags = helpers.UniqueStringsSorted(tags)
+	classes = hstrings.UniqueStringsSorted(classes)
+	ids = hstrings.UniqueStringsSorted(ids)
+	tags = hstrings.UniqueStringsSorted(tags)
 
 	els := HTMLElements{
 		Classes: classes,
diff --git a/releaser/releaser.go b/releaser/releaser.go
index 46cee1b13..74770dfbd 100644
--- a/releaser/releaser.go
+++ b/releaser/releaser.go
@@ -24,7 +24,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/gohugoio/hugo/common/hugo"
+	"github.com/gohugoio/hugo/common/version"
 )
 
 const commitPrefix = "releaser:"
@@ -135,7 +135,7 @@ func (r *ReleaseHandler) Run() error {
 	return nil
 }
 
-func (r *ReleaseHandler) bumpVersions(ver hugo.Version) error {
+func (r *ReleaseHandler) bumpVersions(ver version.Version) error {
 	toDev := ""
 
 	if ver.Suffix != "" {
@@ -166,8 +166,8 @@ func (r *ReleaseHandler) bumpVersions(ver hugo.Version) error {
 	return nil
 }
 
-func (r ReleaseHandler) calculateVersions() (hugo.Version, hugo.Version) {
-	newVersion := hugo.MustParseVersion(r.branchVersion)
+func (r ReleaseHandler) calculateVersions() (version.Version, version.Version) {
+	newVersion := version.MustParseVersion(r.branchVersion)
 	finalVersion := newVersion.Next()
 	finalVersion.PatchLevel = 0
 
diff --git a/resources/images/smartcrop.go b/resources/images/smartcrop.go
index af45c241c..4f706df67 100644
--- a/resources/images/smartcrop.go
+++ b/resources/images/smartcrop.go
@@ -81,7 +81,7 @@ func (p *ImageProcessor) smartCrop(img image.Image, width, height int, filter gi
 	return img.Bounds().Intersect(rect), nil
 }
 
-// Calculates scaling factors using old and new image dimensions.
+// Calculates scaling factors using old and new image sitesmatrix.
 // Code borrowed from https://github.com/nfnt/resize/blob/83c6a9932646f83e3267f353373d47347b6036b2/resize.go#L593
 func calcFactorsNfnt(width, height uint, oldWidth, oldHeight float64) (scaleX, scaleY float64) {
 	if width == 0 {
diff --git a/resources/page/page.go b/resources/page/page.go
index 5eceaf56f..406843283 100644
--- a/resources/page/page.go
+++ b/resources/page/page.go
@@ -20,6 +20,8 @@ import (
 	"fmt"
 	"html/template"
 
+	"github.com/gohugoio/hugo/hugolib/roles"
+	"github.com/gohugoio/hugo/hugolib/sitesmatrix"
 	"github.com/gohugoio/hugo/markup/converter"
 	"github.com/gohugoio/hugo/markup/tableofcontents"
 
@@ -185,6 +187,11 @@ type PageMetaResource interface {
 	resource.Resource
 }
 
+type PageMetaLanguageResource interface {
+	PageMetaResource
+	resource.LanguageProvider
+}
+
 // PageMetaProvider provides page metadata, typically provided via front matter.
 type PageMetaProvider interface {
 	// The 4 page dates
@@ -224,9 +231,6 @@ type PageMetaProvider interface {
 	// IsPage returns whether this is a regular content
 	IsPage() bool
 
-	// Param looks for a param in Page and then in Site config.
-	Param(key any) (any, error)
-
 	// Path gets the relative path, including file name and extension if relevant,
 	// to the source of this Page. It will be relative to any content root.
 	Path() string
@@ -234,9 +238,6 @@ type PageMetaProvider interface {
 	// The slug, typically defined in front matter.
 	Slug() string
 
-	// This page's language code. Will be the same as the site's.
-	Lang() string
-
 	// IsSection returns whether this is a section
 	IsSection() bool
 
@@ -260,7 +261,7 @@ type PageMetaProvider interface {
 // This is currently only used to generate keywords for related content.
 // If nameLower is not one of the metadata interface methods, we
 // look in Params.
-func NamedPageMetaValue(p PageMetaResource, nameLower string) (any, bool, error) {
+func NamedPageMetaValue(p PageMetaLanguageResource, nameLower string) (any, bool, error) {
 	var (
 		v   any
 		err error
@@ -344,7 +345,10 @@ type PageWithoutContent interface {
 	RenderShortcodesProvider
 	resource.Resource
 	PageMetaProvider
+
+	Param(key any) (any, error)
 	PageMetaInternalProvider
+
 	resource.LanguageProvider
 
 	// For pages backed by a file.
@@ -400,6 +404,10 @@ type PageWithoutContent interface {
 	HeadingsFiltered(context.Context) tableofcontents.Headings
 }
 
+type SiteDimensionProvider interface {
+	Role() roles.Role
+}
+
 // Positioner provides next/prev navigation.
 type Positioner interface {
 	// Next points up to the next regular page (sorted by Hugo’s default sort).
@@ -531,6 +539,19 @@ type TreeProvider interface {
 	SectionsPath() string
 }
 
+// SiteVectorProvider provides the dimensions of a Page.
+type SiteVectorProvider interface {
+	SiteVector() sitesmatrix.Vector
+}
+
+// GetSiteVector returns the site vector for a Page.
+func GetSiteVector(p Page) sitesmatrix.Vector {
+	if sp, ok := p.(SiteVectorProvider); ok {
+		return sp.SiteVector()
+	}
+	return sitesmatrix.Vector{}
+}
+
 // PageWithContext is a Page with a context.Context.
 type PageWithContext struct {
 	Page
diff --git a/resources/page/page_lazy_contentprovider.go b/resources/page/page_lazy_contentprovider.go
index 8e66a03e4..78ca09bbe 100644
--- a/resources/page/page_lazy_contentprovider.go
+++ b/resources/page/page_lazy_contentprovider.go
@@ -17,7 +17,7 @@ import (
 	"context"
 	"html/template"
 
-	"github.com/gohugoio/hugo/lazy"
+	"github.com/gohugoio/hugo/common/hsync"
 	"github.com/gohugoio/hugo/markup/converter"
 	"github.com/gohugoio/hugo/markup/tableofcontents"
 )
@@ -48,26 +48,16 @@ type OutputFormatPageContentProvider interface {
 // Used in cases where we cannot guarantee whether the content provider
 // will be needed. Must create via NewLazyContentProvider.
 type LazyContentProvider struct {
-	init *lazy.Init
-	cp   OutputFormatContentProvider
+	init hsync.ValueResetter[OutputFormatContentProvider]
 }
 
 // NewLazyContentProvider returns a LazyContentProvider initialized with
 // function f. The resulting LazyContentProvider calls f in order to
 // retrieve a ContentProvider
-func NewLazyContentProvider(f func() (OutputFormatContentProvider, error)) *LazyContentProvider {
+func NewLazyContentProvider(f func(ctx context.Context) OutputFormatContentProvider) *LazyContentProvider {
 	lcp := LazyContentProvider{
-		init: lazy.New(),
-		cp:   NopCPageContentRenderer,
+		init: hsync.OnceMoreValue(f),
 	}
-	lcp.init.Add(func(context.Context) (any, error) {
-		cp, err := f()
-		if err != nil {
-			return nil, err
-		}
-		lcp.cp = cp
-		return nil, nil
-	})
 	return &lcp
 }
 
@@ -76,91 +66,73 @@ func (lcp *LazyContentProvider) Reset() {
 }
 
 func (lcp *LazyContentProvider) Markup(opts ...any) Markup {
-	lcp.init.Do(context.Background())
-	return lcp.cp.Markup(opts...)
+	return lcp.init.Value(context.Background()).Markup(opts...)
 }
 
 func (lcp *LazyContentProvider) TableOfContents(ctx context.Context) template.HTML {
-	lcp.init.Do(ctx)
-	return lcp.cp.TableOfContents(ctx)
+	return lcp.init.Value(ctx).TableOfContents(ctx)
 }
 
 func (lcp *LazyContentProvider) Fragments(ctx context.Context) *tableofcontents.Fragments {
-	lcp.init.Do(ctx)
-	return lcp.cp.Fragments(ctx)
+	return lcp.init.Value(ctx).Fragments(ctx)
 }
 
 func (lcp *LazyContentProvider) Content(ctx context.Context) (any, error) {
-	lcp.init.Do(ctx)
-	return lcp.cp.Content(ctx)
+	return lcp.init.Value(ctx).Content(ctx)
 }
 
 func (lcp *LazyContentProvider) ContentWithoutSummary(ctx context.Context) (template.HTML, error) {
-	lcp.init.Do(ctx)
-	return lcp.cp.ContentWithoutSummary(ctx)
+	return lcp.init.Value(ctx).ContentWithoutSummary(ctx)
 }
 
 func (lcp *LazyContentProvider) Plain(ctx context.Context) string {
-	lcp.init.Do(ctx)
-	return lcp.cp.Plain(ctx)
+	return lcp.init.Value(ctx).Plain(ctx)
 }
 
 func (lcp *LazyContentProvider) PlainWords(ctx context.Context) []string {
-	lcp.init.Do(ctx)
-	return lcp.cp.PlainWords(ctx)
+	return lcp.init.Value(ctx).PlainWords(ctx)
 }
 
 func (lcp *LazyContentProvider) Summary(ctx context.Context) template.HTML {
-	lcp.init.Do(ctx)
-	return lcp.cp.Summary(ctx)
+	return lcp.init.Value(ctx).Summary(ctx)
 }
 
 func (lcp *LazyContentProvider) Truncated(ctx context.Context) bool {
-	lcp.init.Do(ctx)
-	return lcp.cp.Truncated(ctx)
+	return lcp.init.Value(ctx).Truncated(ctx)
 }
 
 func (lcp *LazyContentProvider) FuzzyWordCount(ctx context.Context) int {
-	lcp.init.Do(ctx)
-	return lcp.cp.FuzzyWordCount(ctx)
+	return lcp.init.Value(ctx).FuzzyWordCount(ctx)
 }
 
 func (lcp *LazyContentProvider) WordCount(ctx context.Context) int {
-	lcp.init.Do(ctx)
-	return lcp.cp.WordCount(ctx)
+	return lcp.init.Value(ctx).WordCount(ctx)
 }
 
 func (lcp *LazyContentProvider) ReadingTime(ctx context.Context) int {
-	lcp.init.Do(ctx)
-	return lcp.cp.ReadingTime(ctx)
+	return lcp.init.Value(ctx).ReadingTime(ctx)
 }
 
 func (lcp *LazyContentProvider) Len(ctx context.Context) int {
-	lcp.init.Do(ctx)
-	return lcp.cp.Len(ctx)
+	return lcp.init.Value(ctx).Len(ctx)
 }
 
 func (lcp *LazyContentProvider) Render(ctx context.Context, layout ...string) (template.HTML, error) {
-	lcp.init.Do(ctx)
-	return lcp.cp.Render(ctx, layout...)
+	return lcp.init.Value(ctx).Render(ctx, layout...)
 }
 
 func (lcp *LazyContentProvider) RenderString(ctx context.Context, args ...any) (template.HTML, error) {
-	lcp.init.Do(ctx)
-	return lcp.cp.RenderString(ctx, args...)
+	return lcp.init.Value(ctx).RenderString(ctx, args...)
 }
 
 func (lcp *LazyContentProvider) ParseAndRenderContent(ctx context.Context, content []byte, renderTOC bool) (converter.ResultRender, error) {
-	lcp.init.Do(ctx)
-	return lcp.cp.ParseAndRenderContent(ctx, content, renderTOC)
+	return lcp.init.Value(ctx).ParseAndRenderContent(ctx, content, renderTOC)
 }
 
 func (lcp *LazyContentProvider) ParseContent(ctx context.Context, content []byte) (converter.ResultParse, bool, error) {
-	lcp.init.Do(ctx)
-	return lcp.cp.ParseContent(ctx, content)
+	return lcp.init.Value(ctx).ParseContent(ctx, content)
 }
 
 func (lcp *LazyContentProvider) RenderContent(ctx context.Context, content []byte, doc any) (converter.ResultRender, bool, error) {
-	lcp.init.Do(ctx)
-	return lcp.cp.RenderContent(ctx, content, doc)
+	return lcp.init.Value(ctx).RenderContent(ctx, content, doc)
 }
diff --git a/resources/page/page_matcher.go b/resources/page/page_matcher.go
index 0ca41aa85..60f0c4654 100644
--- a/resources/page/page_matcher.go
+++ b/resources/page/page_matcher.go
@@ -15,14 +15,19 @@ package page
 
 import (
 	"fmt"
+	"iter"
 	"path/filepath"
 	"slices"
 	"strings"
 
+	"github.com/gohugoio/hugo/common/hashing"
+	"github.com/gohugoio/hugo/common/hstrings"
+	"github.com/gohugoio/hugo/common/hugo"
 	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/common/maps"
 	"github.com/gohugoio/hugo/config"
-	"github.com/gohugoio/hugo/hugofs/glob"
+	"github.com/gohugoio/hugo/hugofs/hglob"
+	"github.com/gohugoio/hugo/hugolib/sitesmatrix"
 	"github.com/gohugoio/hugo/resources/kinds"
 	"github.com/mitchellh/mapstructure"
 )
@@ -40,32 +45,42 @@ type PageMatcher struct {
 	Kind string
 
 	// A Glob pattern matching the Page's language, e.g. "{en,sv}".
+	// Deprecated: use Sites.Matrix instead.
 	Lang string
 
+	// The sites to apply this to.
+	// Note that we currently only use the Matrix field for cascade matching.
+	Sites sitesmatrix.Sites
+
 	// A Glob pattern matching the Page's Environment, e.g. "{production,development}".
 	Environment string
+
+	// Compiled values.
+	// The site vectors to apply this to.
+	SitesMatrixCompiled sitesmatrix.VectorProvider `mapstructure:"-"`
 }
 
-// Matches returns whether p matches this matcher.
 func (m PageMatcher) Matches(p Page) bool {
-	if m.Kind != "" {
-		g, err := glob.GetGlob(m.Kind)
-		if err == nil && !g.Match(p.Kind()) {
+	return m.Match(p.Kind(), p.Path(), p.Site().Hugo().Environment, nil)
+}
+
+func (m PageMatcher) Match(kind, path, environment string, sitesMatrix sitesmatrix.VectorProvider) bool {
+	if sitesMatrix != nil {
+		if m.SitesMatrixCompiled != nil && !m.SitesMatrixCompiled.HasAnyVector(sitesMatrix) {
 			return false
 		}
 	}
-
-	if m.Lang != "" {
-		g, err := glob.GetGlob(m.Lang)
-		if err == nil && !g.Match(p.Lang()) {
+	if m.Kind != "" {
+		g, err := hglob.GetGlob(m.Kind)
+		if err == nil && !g.Match(kind) {
 			return false
 		}
 	}
 
 	if m.Path != "" {
-		g, err := glob.GetGlob(m.Path)
+		g, err := hglob.GetGlob(m.Path)
 		// TODO(bep) Path() vs filepath vs leading slash.
-		p := strings.ToLower(filepath.ToSlash(p.Path()))
+		p := strings.ToLower(filepath.ToSlash(path))
 		if !(strings.HasPrefix(p, "/")) {
 			p = "/" + p
 		}
@@ -75,8 +90,8 @@ func (m PageMatcher) Matches(p Page) bool {
 	}
 
 	if m.Environment != "" {
-		g, err := glob.GetGlob(m.Environment)
-		if err == nil && !g.Match(p.Site().Hugo().Environment) {
+		g, err := hglob.GetGlob(m.Environment)
+		if err == nil && !g.Match(environment) {
 			return false
 		}
 	}
@@ -87,9 +102,10 @@ func (m PageMatcher) Matches(p Page) bool {
 var disallowedCascadeKeys = map[string]bool{
 	// These define the structure of the page tree and cannot
 	// currently be set in the cascade.
-	"kind": true,
-	"path": true,
-	"lang": true,
+	"kind":    true,
+	"path":    true,
+	"lang":    true,
+	"cascade": true,
 }
 
 // See issue 11977.
@@ -99,21 +115,27 @@ func isGlobWithExtension(s string) bool {
 	return strings.Count(last, ".") > 0
 }
 
-func CheckCascadePattern(logger loggers.Logger, m PageMatcher) {
-	if logger != nil && isGlobWithExtension(m.Path) {
-		logger.Erroridf("cascade-pattern-with-extension", "cascade target path %q looks like a path with an extension; since Hugo v0.123.0 this will not match anything, see  https://gohugo.io/methods/page/path/", m.Path)
+func checkCascadePattern(logger loggers.Logger, m PageMatcher) {
+	if m.Lang != "" {
+		hugo.Deprecate("cascade.target.language", "cascade.target.sites.matrix instead, see https://gohugo.io/content-management/front-matter/#target", "v0.150.0")
 	}
 }
 
-func DecodeCascadeConfig(logger loggers.Logger, handleLegacyFormat bool, in any) (*config.ConfigNamespace[[]PageMatcherParamsConfig, *maps.Ordered[PageMatcher, PageMatcherParamsConfig]], error) {
-	buildConfig := func(in any) (*maps.Ordered[PageMatcher, PageMatcherParamsConfig], any, error) {
-		cascade := maps.NewOrdered[PageMatcher, PageMatcherParamsConfig]()
+func AddLangToCascadeTargetMap(lang string, m maps.Params) {
+	maps.SetNestedParamIfNotSet("target.sites.matrix.languages", ".", lang, m)
+}
+
+func DecodeCascadeConfig(in any) (*PageMatcherParamsConfigs, error) {
+	buildConfig := func(in any) (CascadeConfig, any, error) {
+		dec := cascadeConfigDecoder{}
+
 		if in == nil {
-			return cascade, []map[string]any{}, nil
+			return CascadeConfig{}, []map[string]any{}, nil
 		}
+
 		ms, err := maps.ToSliceStringMap(in)
 		if err != nil {
-			return nil, nil, err
+			return CascadeConfig{}, nil, err
 		}
 
 		var cfgs []PageMatcherParamsConfig
@@ -124,55 +146,50 @@ func DecodeCascadeConfig(logger loggers.Logger, handleLegacyFormat bool, in any)
 				c   PageMatcherParamsConfig
 				err error
 			)
-			c, err = mapToPageMatcherParamsConfig(m)
+			c, err = dec.mapToPageMatcherParamsConfig(m)
 			if err != nil {
-				return nil, nil, err
+				return CascadeConfig{}, nil, err
 			}
 			for k := range m {
 				if disallowedCascadeKeys[k] {
-					return nil, nil, fmt.Errorf("key %q not allowed in cascade config", k)
+					return CascadeConfig{}, nil, fmt.Errorf("key %q not allowed in cascade config", k)
 				}
 			}
 			cfgs = append(cfgs, c)
 		}
 
+		if len(cfgs) == 0 {
+			return CascadeConfig{}, nil, nil
+		}
+
+		var n int
 		for _, cfg := range cfgs {
-			m := cfg.Target
-			CheckCascadePattern(logger, m)
-			c, found := cascade.Get(m)
-			if found {
-				// Merge
-				for k, v := range cfg.Params {
-					if _, found := c.Params[k]; !found {
-						c.Params[k] = v
-					}
-				}
-				for k, v := range cfg.Fields {
-					if _, found := c.Fields[k]; !found {
-						c.Fields[k] = v
-					}
-				}
-			} else {
-				cascade.Set(m, cfg)
+			if len(cfg.Params) > 0 || len(cfg.Fields) > 0 {
+				cfgs[n] = cfg
+				n++
 			}
 		}
 
-		return cascade, cfgs, nil
-	}
+		if n == 0 {
+			return CascadeConfig{}, nil, nil
+		}
 
-	return config.DecodeNamespace[[]PageMatcherParamsConfig, *maps.Ordered[PageMatcher, PageMatcherParamsConfig]](in, buildConfig)
-}
+		cfgs = cfgs[:n]
 
-// DecodeCascade decodes in which could be either a map or a slice of maps.
-func DecodeCascade(logger loggers.Logger, handleLegacyFormat bool, in any) (*maps.Ordered[PageMatcher, PageMatcherParamsConfig], error) {
-	conf, err := DecodeCascadeConfig(logger, handleLegacyFormat, in)
-	if err != nil {
+		return CascadeConfig{Cascades: cfgs}, cfgs, nil
+	}
+
+	c, err := config.DecodeNamespace[[]PageMatcherParamsConfig](in, buildConfig)
+	if err != nil || len(c.Config.Cascades) == 0 {
 		return nil, err
 	}
-	return conf.Config, nil
+
+	return &PageMatcherParamsConfigs{c: []*config.ConfigNamespace[[]PageMatcherParamsConfig, CascadeConfig]{c}}, nil
 }
 
-func mapToPageMatcherParamsConfig(m map[string]any) (PageMatcherParamsConfig, error) {
+type cascadeConfigDecoder struct{}
+
+func (d cascadeConfigDecoder) mapToPageMatcherParamsConfig(m map[string]any) (PageMatcherParamsConfig, error) {
 	var pcfg PageMatcherParamsConfig
 	if pcfg.Fields == nil {
 		pcfg.Fields = make(maps.Params)
@@ -180,11 +197,12 @@ func mapToPageMatcherParamsConfig(m map[string]any) (PageMatcherParamsConfig, er
 	if pcfg.Params == nil {
 		pcfg.Params = make(maps.Params)
 	}
+
 	for k, v := range m {
 		switch strings.ToLower(k) {
 		case "_target", "target":
 			var target PageMatcher
-			if err := decodePageMatcher(v, &target); err != nil {
+			if err := d.decodePageMatcher(v, &target); err != nil {
 				return pcfg, err
 			}
 			pcfg.Target = target
@@ -203,14 +221,14 @@ func mapToPageMatcherParamsConfig(m map[string]any) (PageMatcherParamsConfig, er
 }
 
 // decodePageMatcher decodes m into v.
-func decodePageMatcher(m any, v *PageMatcher) error {
+func (d cascadeConfigDecoder) decodePageMatcher(m any, v *PageMatcher) error {
 	if err := mapstructure.WeakDecode(m, v); err != nil {
 		return err
 	}
 
 	v.Kind = strings.ToLower(v.Kind)
 	if v.Kind != "" {
-		g, _ := glob.GetGlob(v.Kind)
+		g, _ := hglob.GetGlob(v.Kind)
 		found := slices.ContainsFunc(kinds.AllKindsInPages, g.Match)
 		if !found {
 			return fmt.Errorf("%q did not match a valid Page Kind", v.Kind)
@@ -219,9 +237,34 @@ func decodePageMatcher(m any, v *PageMatcher) error {
 
 	v.Path = filepath.ToSlash(strings.ToLower(v.Path))
 
+	if v.Lang != "" {
+		v.Sites.Matrix.Languages = append(v.Sites.Matrix.Languages, v.Lang)
+		v.Sites.Matrix.Languages = hstrings.UniqueStringsReuse(v.Sites.Matrix.Languages)
+	}
+
 	return nil
 }
 
+// DecodeCascadeConfigOptions
+func (v *PageMatcher) compileSitesMatrix(configuredDimensions *sitesmatrix.ConfiguredDimensions) error {
+	if v.Sites.Matrix.IsZero() {
+		// Nothing to do.
+		v.SitesMatrixCompiled = nil
+		return nil
+	}
+	intSetsCfg := sitesmatrix.IntSetsConfig{
+		Globs: v.Sites.Matrix,
+	}
+	b := sitesmatrix.NewIntSetsBuilder(configuredDimensions).WithConfig(intSetsCfg).WithAllIfNotSet()
+
+	v.SitesMatrixCompiled = b.Build()
+	return nil
+}
+
+type CascadeConfig struct {
+	Cascades []PageMatcherParamsConfig
+}
+
 type PageMatcherParamsConfig struct {
 	// Apply Params to all Pages matching Target.
 	Params maps.Params
@@ -234,5 +277,87 @@ type PageMatcherParamsConfig struct {
 func (p *PageMatcherParamsConfig) init() error {
 	maps.PrepareParams(p.Params)
 	maps.PrepareParams(p.Fields)
+
+	return nil
+}
+
+type PageMatcherParamsConfigs struct {
+	c []*config.ConfigNamespace[[]PageMatcherParamsConfig, CascadeConfig]
+}
+
+func (c *PageMatcherParamsConfigs) Append(other *PageMatcherParamsConfigs) *PageMatcherParamsConfigs {
+	if c == nil || len(c.c) == 0 {
+		return other
+	}
+	if other == nil || len(other.c) == 0 {
+		return c
+	}
+	return &PageMatcherParamsConfigs{c: slices.Concat(c.c, other.c)}
+}
+
+func (c *PageMatcherParamsConfigs) Prepend(other *PageMatcherParamsConfigs) *PageMatcherParamsConfigs {
+	if c == nil || len(c.c) == 0 {
+		return other
+	}
+	if other == nil || len(other.c) == 0 {
+		return c
+	}
+	return &PageMatcherParamsConfigs{c: slices.Concat(other.c, c.c)}
+}
+
+func (c *PageMatcherParamsConfigs) All() iter.Seq[PageMatcherParamsConfig] {
+	if c == nil {
+		return func(func(PageMatcherParamsConfig) bool) {}
+	}
+	return func(yield func(PageMatcherParamsConfig) bool) {
+		if c == nil {
+			return
+		}
+		for _, v := range c.c {
+			for _, vv := range v.Config.Cascades {
+				if !yield(vv) {
+					return
+				}
+			}
+		}
+	}
+}
+
+func (c *PageMatcherParamsConfigs) Len() int {
+	if c == nil {
+		return 0
+	}
+	var n int
+	for _, v := range c.c {
+		n += len(v.Config.Cascades)
+	}
+	return n
+}
+
+func (c *PageMatcherParamsConfigs) SourceHash() uint64 {
+	if c == nil {
+		return 0
+	}
+	h := hashing.XxHasher()
+	defer h.Close()
+
+	for _, v := range c.c {
+		h.WriteString(v.SourceHash)
+	}
+	return h.Sum64()
+}
+
+func (c *PageMatcherParamsConfigs) InitConfig(logger loggers.Logger, _ sitesmatrix.VectorStore, configuredDimensions *sitesmatrix.ConfiguredDimensions) error {
+	if c == nil {
+		return nil
+	}
+	for _, cc := range c.c {
+		for i := range cc.Config.Cascades {
+			checkCascadePattern(logger, cc.Config.Cascades[i].Target)
+			if err := cc.Config.Cascades[i].Target.compileSitesMatrix(configuredDimensions); err != nil {
+				return fmt.Errorf("failed to compile cascade target %d: %w", i, err)
+			}
+		}
+	}
 	return nil
 }
diff --git a/resources/page/page_matcher_test.go b/resources/page/page_matcher_test.go
index 8b9cbccd4..127af247d 100644
--- a/resources/page/page_matcher_test.go
+++ b/resources/page/page_matcher_test.go
@@ -20,6 +20,7 @@ import (
 	"github.com/gohugoio/hugo/common/hugo"
 	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/common/maps"
+	"github.com/gohugoio/hugo/hugolib/sitesmatrix"
 
 	qt "github.com/frankban/quicktest"
 )
@@ -29,6 +30,8 @@ func TestPageMatcher(t *testing.T) {
 	developmentTestSite := testSite{h: hugo.NewInfo(testConfig{environment: "development"}, nil)}
 	productionTestSite := testSite{h: hugo.NewInfo(testConfig{environment: "production"}, nil)}
 
+	dec := cascadeConfigDecoder{}
+
 	p1, p2, p3 := &testPage{path: "/p1", kind: "section", lang: "en", site: developmentTestSite},
 		&testPage{path: "p2", kind: "page", lang: "no", site: productionTestSite},
 		&testPage{path: "p3", kind: "page", lang: "en"}
@@ -54,11 +57,6 @@ func TestPageMatcher(t *testing.T) {
 		c.Assert(m.Matches(p2), qt.Equals, true)
 		c.Assert(m.Matches(p3), qt.Equals, true)
 
-		m = PageMatcher{Lang: "en"}
-		c.Assert(m.Matches(p1), qt.Equals, true)
-		c.Assert(m.Matches(p2), qt.Equals, false)
-		c.Assert(m.Matches(p3), qt.Equals, true)
-
 		m = PageMatcher{Environment: "development"}
 		c.Assert(m.Matches(p1), qt.Equals, true)
 		c.Assert(m.Matches(p2), qt.Equals, false)
@@ -72,19 +70,19 @@ func TestPageMatcher(t *testing.T) {
 
 	c.Run("Decode", func(c *qt.C) {
 		var v PageMatcher
-		c.Assert(decodePageMatcher(map[string]any{"kind": "foo"}, &v), qt.Not(qt.IsNil))
-		c.Assert(decodePageMatcher(map[string]any{"kind": "{foo,bar}"}, &v), qt.Not(qt.IsNil))
-		c.Assert(decodePageMatcher(map[string]any{"kind": "taxonomy"}, &v), qt.IsNil)
-		c.Assert(decodePageMatcher(map[string]any{"kind": "{taxonomy,foo}"}, &v), qt.IsNil)
-		c.Assert(decodePageMatcher(map[string]any{"kind": "{taxonomy,term}"}, &v), qt.IsNil)
-		c.Assert(decodePageMatcher(map[string]any{"kind": "*"}, &v), qt.IsNil)
-		c.Assert(decodePageMatcher(map[string]any{"kind": "home", "path": filepath.FromSlash("/a/b/**")}, &v), qt.IsNil)
-		c.Assert(v, qt.Equals, PageMatcher{Kind: "home", Path: "/a/b/**"})
+		c.Assert(dec.decodePageMatcher(map[string]any{"kind": "foo"}, &v), qt.Not(qt.IsNil))
+		c.Assert(dec.decodePageMatcher(map[string]any{"kind": "{foo,bar}"}, &v), qt.Not(qt.IsNil))
+		c.Assert(dec.decodePageMatcher(map[string]any{"kind": "taxonomy"}, &v), qt.IsNil)
+		c.Assert(dec.decodePageMatcher(map[string]any{"kind": "{taxonomy,foo}"}, &v), qt.IsNil)
+		c.Assert(dec.decodePageMatcher(map[string]any{"kind": "{taxonomy,term}"}, &v), qt.IsNil)
+		c.Assert(dec.decodePageMatcher(map[string]any{"kind": "*"}, &v), qt.IsNil)
+		c.Assert(dec.decodePageMatcher(map[string]any{"kind": "home", "path": filepath.FromSlash("/a/b/**")}, &v), qt.IsNil)
+		c.Assert(v, qt.DeepEquals, PageMatcher{Kind: "home", Path: "/a/b/**"})
 	})
 
 	c.Run("mapToPageMatcherParamsConfig", func(c *qt.C) {
 		fn := func(m map[string]any) PageMatcherParamsConfig {
-			v, err := mapToPageMatcherParamsConfig(m)
+			v, err := dec.mapToPageMatcherParamsConfig(m)
 			c.Assert(err, qt.IsNil)
 			return v
 		}
@@ -129,13 +127,25 @@ func TestDecodeCascadeConfig(t *testing.T) {
 		},
 	}
 
-	got, err := DecodeCascadeConfig(loggers.NewDefault(), true, in)
-
+	got, err := DecodeCascadeConfig(in)
 	c.Assert(err, qt.IsNil)
 	c.Assert(got, qt.IsNotNil)
-	c.Assert(got.Config.Keys(), qt.DeepEquals, []PageMatcher{{Kind: "page", Environment: "production"}, {Kind: "page"}})
+	c.Assert(got.InitConfig(loggers.NewDefault(), nil, nil), qt.IsNil)
+	c.Assert(got.c[0].Config.Cascades, qt.HasLen, 2)
+	first := got.c[0].Config.Cascades[0]
+	c.Assert(first, qt.DeepEquals, PageMatcherParamsConfig{
+		Params: maps.Params{
+			"a": "av",
+		},
+		Fields: maps.Params{},
+		Target: PageMatcher{
+			Kind:        "page",
+			Sites:       sitesmatrix.Sites{},
+			Environment: "production",
+		},
+	})
 
-	c.Assert(got.SourceStructure, qt.DeepEquals, []PageMatcherParamsConfig{
+	c.Assert(got.c[0].SourceStructure, qt.DeepEquals, []PageMatcherParamsConfig{
 		{
 			Params: maps.Params{"a": string("av")},
 			Fields: maps.Params{},
@@ -144,9 +154,51 @@ func TestDecodeCascadeConfig(t *testing.T) {
 		{Params: maps.Params{"b": string("bv")}, Fields: maps.Params{}, Target: PageMatcher{Kind: "page"}},
 	})
 
-	got, err = DecodeCascadeConfig(loggers.NewDefault(), true, nil)
+	got, err = DecodeCascadeConfig(nil)
+	c.Assert(err, qt.IsNil)
+	c.Assert(got.InitConfig(loggers.NewDefault(), nil, nil), qt.IsNil)
+	c.Assert(got.Len(), qt.Equals, 0)
+}
+
+func TestDecodeCascadeConfigWithSitesMatrix(t *testing.T) {
+	c := qt.New(t)
+
+	in := []map[string]any{
+		{
+			"params": map[string]any{
+				"a": "av",
+			},
+			"sites": map[string]any{
+				"matrix": map[string]any{
+					"roles": "pro",
+				},
+			},
+			"target": map[string]any{
+				"kind":        "page",
+				"Environment": "production",
+				"sites": map[string]any{
+					"matrix": map[string]any{
+						"languages": []string{"en", "{no,sv}"},
+						"versions":  "v1**",
+					},
+				},
+			},
+		},
+	}
+
+	dims := sitesmatrix.NewTestingDimensions([]string{"en", "no", "sv"}, []string{"v1", "v2"}, []string{"free", "pro"})
+
+	got, err := DecodeCascadeConfig(in)
 	c.Assert(err, qt.IsNil)
 	c.Assert(got, qt.IsNotNil)
+	c.Assert(got.InitConfig(loggers.NewDefault(), nil, dims), qt.IsNil)
+	v := got.c[0].Config.Cascades[0]
+	c.Assert(v.Target.Kind, qt.Equals, "page")
+	c.Assert(v.Target.Environment, qt.Equals, "production")
+
+	matrix := v.Target.SitesMatrixCompiled
+	c.Assert(matrix.HasVector(sitesmatrix.Vector{0, 0, 0}), qt.IsTrue)  // en, v1, free
+	c.Assert(matrix.HasVector(sitesmatrix.Vector{0, 1, 0}), qt.IsFalse) // en, v2, free
 }
 
 type testConfig struct {
diff --git a/resources/page/page_nop.go b/resources/page/page_nop.go
index 865470479..297ebda22 100644
--- a/resources/page/page_nop.go
+++ b/resources/page/page_nop.go
@@ -21,6 +21,7 @@ import (
 	"html/template"
 	"time"
 
+	"github.com/gohugoio/hugo/hugolib/roles"
 	"github.com/gohugoio/hugo/markup/converter"
 	"github.com/gohugoio/hugo/markup/tableofcontents"
 
@@ -255,6 +256,10 @@ func (p *nopPage) Language() *langs.Language {
 	return nil
 }
 
+func (p *nopPage) Role() roles.Role {
+	return nil
+}
+
 func (p *nopPage) Lastmod() (t time.Time) {
 	return
 }
diff --git a/resources/page/page_paths.go b/resources/page/page_paths.go
index 354292e1a..3bc3e802e 100644
--- a/resources/page/page_paths.go
+++ b/resources/page/page_paths.go
@@ -14,6 +14,7 @@
 package page
 
 import (
+	"bytes"
 	"path"
 	"path/filepath"
 	"strings"
@@ -295,6 +296,7 @@ func CreateTargetPaths(d TargetPathDescriptor) (tp TargetPaths) {
 // When adding state here, remember to update putPagePathBuilder.
 type pagePathBuilder struct {
 	els []string
+	b   bytes.Buffer
 
 	d TargetPathDescriptor
 
@@ -386,8 +388,17 @@ func (p *pagePathBuilder) Path(upperOffset int) string {
 	if upperOffset > 0 {
 		upper -= upperOffset
 	}
-	pth := path.Join(p.els[:upper]...)
-	return paths.AddLeadingSlash(pth)
+	p.b.Reset()
+
+	var hadTrailingSlash bool
+	for _, el := range p.els[:upper] {
+		if !hadTrailingSlash && !strings.HasPrefix(el, "/") {
+			p.b.WriteByte('/')
+		}
+		hadTrailingSlash = strings.HasSuffix(el, "/")
+		p.b.WriteString(el)
+	}
+	return p.b.String()
 }
 
 func (p *pagePathBuilder) PathDir() string {
diff --git a/resources/page/page_paths_test.go b/resources/page/page_paths_test.go
new file mode 100644
index 000000000..e2765ec65
--- /dev/null
+++ b/resources/page/page_paths_test.go
@@ -0,0 +1,54 @@
+// Copyright 2025 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package page
+
+import (
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestPagePathsBuilder(t *testing.T) {
+	c := qt.New(t)
+
+	d := TargetPathDescriptor{}
+	b := getPagePathBuilder(d)
+	defer putPagePathBuilder(b)
+	b.Add("foo", "bar")
+
+	c.Assert(b.Path(0), qt.Equals, "/foo/bar")
+}
+
+func BenchmarkPagePathsBuilderPath(b *testing.B) {
+	d := TargetPathDescriptor{}
+	pb := getPagePathBuilder(d)
+	defer putPagePathBuilder(pb)
+	pb.Add("foo", "bar")
+
+	for i := 0; i < b.N; i++ {
+		_ = pb.Path(0)
+	}
+}
+
+func BenchmarkPagePathsBuilderPathDir(b *testing.B) {
+	d := TargetPathDescriptor{}
+	pb := getPagePathBuilder(d)
+	defer putPagePathBuilder(pb)
+	pb.Add("foo", "bar")
+	pb.prefixPath = "foo/"
+
+	for i := 0; i < b.N; i++ {
+		_ = pb.PathDir()
+	}
+}
diff --git a/resources/page/pagemeta/page_frontmatter.go b/resources/page/pagemeta/page_frontmatter.go
index 5cbef5681..aa299d0d1 100644
--- a/resources/page/pagemeta/page_frontmatter.go
+++ b/resources/page/pagemeta/page_frontmatter.go
@@ -21,12 +21,15 @@ import (
 	"time"
 
 	"github.com/gohugoio/hugo/common/hreflect"
+	"github.com/gohugoio/hugo/common/hstrings"
 	"github.com/gohugoio/hugo/common/htime"
 	"github.com/gohugoio/hugo/common/hugio"
 	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/common/maps"
 	"github.com/gohugoio/hugo/common/paths"
+	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/hugofs/files"
+	"github.com/gohugoio/hugo/hugolib/sitesmatrix"
 	"github.com/gohugoio/hugo/markup"
 	"github.com/gohugoio/hugo/media"
 	"github.com/gohugoio/hugo/output"
@@ -35,8 +38,6 @@ import (
 	"github.com/gohugoio/hugo/resources/resource"
 	"github.com/mitchellh/mapstructure"
 
-	"github.com/gohugoio/hugo/helpers"
-
 	"github.com/gohugoio/hugo/config"
 	"github.com/spf13/cast"
 )
@@ -55,6 +56,17 @@ type Dates struct {
 	ExpiryDate  time.Time
 }
 
+func (d Dates) String() string {
+	fmtDate := func(t time.Time) string {
+		if t.IsZero() {
+			return ""
+		}
+		return t.Format(time.RFC3339)
+	}
+	return fmt.Sprintf("Date: %s, Lastmod: %s, PublishDate: %s, ExpiryDate: %s",
+		fmtDate(d.Date), fmtDate(d.Lastmod), fmtDate(d.PublishDate), fmtDate(d.ExpiryDate))
+}
+
 func (d Dates) IsDateOrLastModAfter(in Dates) bool {
 	return d.Date.After(in.Date) || d.Lastmod.After(in.Lastmod)
 }
@@ -76,40 +88,87 @@ func (d Dates) IsAllDatesZero() bool {
 	return d.Date.IsZero() && d.Lastmod.IsZero() && d.PublishDate.IsZero() && d.ExpiryDate.IsZero()
 }
 
-// Page config that needs to be set early. These cannot be modified by cascade.
+const (
+	pageMetaKeySites  = "sites"
+	pageMetaKeyMarkup = "markup"
+)
+
+func (pcfg *PageConfigEarly) SetMetaPreFromMap(ext string, frontmatter map[string]any, logger loggers.Logger, conf config.AllProvider) error {
+	if frontmatter != nil {
+		if err := pcfg.setFromFrontMatter(frontmatter); err != nil {
+			return err
+		}
+	}
+	return pcfg.resolveContentType(ext, conf.GetConfigSection("mediaTypes").(media.Types))
+}
+
+func (pcfg *PageConfigEarly) setFromFrontMatter(frontmatter map[string]any) error {
+	// Needed for case insensitive fetching of params values.
+	maps.PrepareParams(frontmatter)
+	pcfg.Frontmatter = frontmatter
+
+	if v, found := frontmatter[pageMetaKeyMarkup]; found {
+		pcfg.Content.Markup = cast.ToString(v)
+	}
+
+	if v, found := frontmatter[pageMetaKeySites]; found {
+		if err := mapstructure.WeakDecode(v, &pcfg.Sites); err != nil {
+			return fmt.Errorf("failed to decode sites from front matter: %w", err)
+		}
+	}
+	return nil
+}
+
+func (p *PageConfigEarly) setCascadeEarlyValueIfNotSet(key string, value any) (done bool) {
+	switch key {
+	case pageMetaKeySites:
+		p.Sites.SetFromParamsIfNotSet(value.(maps.Params))
+	}
+
+	return !p.Sites.Matrix.IsZero()
+}
+
+// Page config that needs to be set early.
 type PageConfigEarly struct {
-	Kind    string // The kind of page, e.g. "page", "section", "home" etc. This is usually derived from the content path.
-	Path    string // The canonical path to the page, e.g. /sect/mypage. Note: Leading slash, no trailing slash, no extensions or language identifiers.
-	Lang    string // The language code for this page. This is usually derived from the module mount or filename.
-	Cascade []map[string]any
+	Kind            string // The kind of page, e.g. "page", "section", "home" etc. This is usually derived from the content path.
+	Path            string // The canonical path to the page, e.g. /sect/mypage. Note: Leading slash, no trailing slash, no extensions or language identifiers.
+	SourceEntryHash uint64 // The source entry hash for content adapters.
 
-	// Content holds the content for this page.
-	Content Source
+	Sites   sitesmatrix.Sites
+	Content Source // Content holds the content for this page.
+
+	Frontmatter maps.Params `mapstructure:"-" json:"-"` // The original front matter or content adapter map.
+
+	// Compiled values.
+	SitesMatrixAndComplements `mapstructure:"-" json:"-"`
+	IsFromContentAdapter      bool       `mapstructure:"-" json:"-"`
+	ContentMediaType          media.Type `mapstructure:"-" json:"-"`
 }
 
 // PageConfig configures a Page, typically from front matter.
 // Note that all the top level fields are reserved Hugo keywords.
 // Any custom configuration needs to be set in the Params map.
-type PageConfig struct {
+type PageConfigLate struct {
 	Dates Dates `json:"-"` // Dates holds the four core dates for this page.
 	DatesStrings
-	PageConfigEarly `mapstructure:",squash"`
-	Title           string   // The title of the page.
-	LinkTitle       string   // The link title of the page.
-	Type            string   // The content type of the page.
-	Layout          string   // The layout to use for to render this page.
-	Weight          int      // The weight of the page, used in sorting if set to a non-zero value.
-	URL             string   // The URL to the rendered page, e.g. /sect/mypage.html.
-	Slug            string   // The slug for this page.
-	Description     string   // The description for this page.
-	Summary         string   // The summary for this page.
-	Draft           bool     // Whether or not the content is a draft.
-	Headless        bool     `json:"-"` // Whether or not the page should be rendered.
-	IsCJKLanguage   bool     // Whether or not the content is in a CJK language.
-	TranslationKey  string   // The translation key for this page.
-	Keywords        []string // The keywords for this page.
-	Aliases         []string // The aliases for this page.
-	Outputs         []string // The output formats to render this page in. If not set, the site's configured output formats for this page kind will be used.
+	Params maps.Params // User defined params.
+
+	Title          string   // The title of the page.
+	LinkTitle      string   // The link title of the page.
+	Type           string   // The content type of the page.
+	Layout         string   // The layout to use for to render this page.
+	Weight         int      // The weight of the page, used in sorting if set to a non-zero value.
+	URL            string   // The URL to the rendered page, e.g. /sect/mypage.html.
+	Slug           string   // The slug for this page.
+	Description    string   // The description for this page.
+	Summary        string   // The summary for this page.
+	Draft          bool     // Whether or not the content is a draft.
+	Headless       bool     `json:"-"` // Whether or not the page should be rendered.
+	IsCJKLanguage  bool     // Whether or not the content is in a CJK language.
+	TranslationKey string   // The translation key for this page.
+	Keywords       []string // The keywords for this page.
+	Aliases        []string // The aliases for this page.
+	Outputs        []string // The output formats to render this page in. If not set, the site's configured output formats for this page kind will be used.
 
 	FrontMatterOnlyValues `mapstructure:"-" json:"-"`
 
@@ -117,97 +176,224 @@ type PageConfig struct {
 	Build   BuildConfig
 	Menus   any // Can be a string, []string or map[string]any.
 
-	// User defined params.
-	Params maps.Params
-
-	// The raw data from the content adapter.
-	// TODO(bep) clean up the ContentAdapterData vs Params.
+	// Set only for pages created from data files.
 	ContentAdapterData map[string]any `mapstructure:"-" json:"-"`
 
 	// Compiled values.
-	CascadeCompiled         *maps.Ordered[page.PageMatcher, page.PageMatcherParamsConfig] `mapstructure:"-" json:"-"`
-	ContentMediaType        media.Type                                                    `mapstructure:"-" json:"-"`
-	ConfiguredOutputFormats output.Formats                                                `mapstructure:"-" json:"-"`
-	IsFromContentAdapter    bool                                                          `mapstructure:"-" json:"-"`
+	ConfiguredOutputFormats output.Formats `mapstructure:"-" json:"-"`
 }
 
-func ClonePageConfigForRebuild(p *PageConfig, params map[string]any) *PageConfig {
-	pp := &PageConfig{
-		PageConfigEarly:      p.PageConfigEarly,
-		IsFromContentAdapter: p.IsFromContentAdapter,
-	}
-	if pp.IsFromContentAdapter {
-		pp.ContentAdapterData = params
-	} else {
-		pp.Params = params
-	}
+// SitesMatrixAndComplements holds a sites matrix and a sites complements configuration.
+type SitesMatrixAndComplements struct {
+	SitesMatrix      sitesmatrix.VectorStore `mapstructure:"-" json:"-"`
+	SitesComplements sitesmatrix.VectorStore `mapstructure:"-" json:"-"`
+}
+
+// MatchSiteVector checks whether the site vector matches the sites matrix.
+func (p *SitesMatrixAndComplements) MatchSiteVector(siteVector sitesmatrix.Vector) bool {
+	return p.SitesMatrix.HasAnyVector(siteVector)
+}
 
-	return pp
+// MatchLanguageCoarse checks whether the language dimension matches either
+// the sites matrix or the sites complements.
+func (p *SitesMatrixAndComplements) MatchLanguageCoarse(siteVector sitesmatrix.Vector) bool {
+	i := siteVector.Language()
+	return p.SitesMatrix.HasLanguage(i) || p.SitesComplements.HasLanguage(i)
 }
 
-var DefaultPageConfig = PageConfig{
-	Build: DefaultBuildConfig,
+// MatchRoleCoarse checks whether the role dimension matches either
+// the sites matrix or the sites complements.
+func (p *SitesMatrixAndComplements) MatchRoleCoarse(siteVector sitesmatrix.Vector) bool {
+	i := siteVector.Role()
+	return p.SitesMatrix.HasRole(i) || p.SitesComplements.HasRole(i)
 }
 
-func (p *PageConfig) Init(pagesFromData bool) error {
+// MatchVersionCoarse checks whether the version dimension matches either
+// the sites matrix or the sites complements.
+func (p *SitesMatrixAndComplements) MatchVersionCoarse(siteVector sitesmatrix.Vector) bool {
+	i := siteVector.Version()
+	return p.SitesMatrix.HasVersion(i) || p.SitesComplements.HasVersion(i)
+}
+
+func DefaultPageConfig() *PageConfigLate {
+	return &PageConfigLate{
+		Build: DefaultBuildConfig,
+	}
+}
+
+func (p *PageConfigEarly) Init(pagesFromData bool) error {
 	if pagesFromData {
 		p.Path = strings.TrimPrefix(p.Path, "/")
 
 		if p.Path == "" && p.Kind != kinds.KindHome {
 			return fmt.Errorf("empty path is reserved for the home page")
 		}
-		if p.Lang != "" {
-			return errors.New("lang must not be set")
-		}
 
 		if p.Content.Markup != "" {
 			return errors.New("markup must not be set, use mediaType")
 		}
 	}
 
-	if p.Cascade != nil {
-		if !kinds.IsBranch(p.Kind) {
-			return errors.New("cascade is only supported for branch nodes")
+	return nil
+}
+
+func (p *PageConfigLate) Init() error {
+	return nil
+}
+
+func buildSitesComplementsFromSitesConfig(
+	conf config.AllProvider,
+	fim *hugofs.FileMeta,
+	sitesConfig sitesmatrix.Sites,
+) *sitesmatrix.IntSets {
+	intsetsCfg := sitesmatrix.IntSetsConfig{
+		Globs: sitesConfig.Complements,
+	}
+
+	sitesComplements := sitesmatrix.NewIntSetsBuilder(conf.ConfiguredDimensions()).WithConfig(intsetsCfg)
+
+	if fim != nil && fim.SitesComplements != nil {
+		sitesComplements.WithDimensionsFromOtherIfNotSet(fim.SitesComplements)
+	}
+
+	return sitesComplements.Build()
+}
+
+func buildSitesMatrixFromVectorIterator(
+	sitesMatrix sitesmatrix.VectorIterator,
+) sitesmatrix.VectorStore {
+	switch v := sitesMatrix.(type) {
+	case sitesmatrix.VectorStore:
+		return v
+	case sitesmatrix.ToVectorStoreProvider:
+		return v.ToVectorStore()
+	default:
+		if v == nil {
+			return nil
 		}
+		panic(fmt.Sprintf("unsupported type %T", v))
+	}
+}
+
+func buildSitesMatrixFromSitesConfig(
+	conf config.AllProvider,
+	sitesMatrixBase sitesmatrix.VectorIterator,
+	sitesConfig sitesmatrix.Sites,
+) *sitesmatrix.IntSets {
+	intsetsCfg := sitesmatrix.IntSetsConfig{
+		Globs: sitesConfig.Matrix,
+	}
+	sitesMatrixPage := sitesmatrix.NewIntSetsBuilder(conf.ConfiguredDimensions()).WithConfig(intsetsCfg)
+
+	if sitesMatrixBase != nil {
+		sitesMatrixPage.WithDimensionsFromOtherIfNotSet(sitesMatrixBase)
+	}
+	sitesMatrixPage.WithDefaultsIfNotSet()
+
+	matrix := sitesMatrixPage.Build()
+
+	return matrix
+}
+
+func (p *PageConfigEarly) CompileEarly(pi *paths.Path, cascades *page.PageMatcherParamsConfigs,
+	conf config.AllProvider, fim *hugofs.FileMeta, sitesMatrixBase sitesmatrix.VectorIterator, sitesMatrixBaseOnly bool,
+) error {
+	// First apply the cascades with no site filtering.
+	for cascade := range cascades.All() {
+		if cascade.Target.SitesMatrixCompiled != nil {
+			continue
+		}
+
+		if !cascade.Target.Match(p.Kind, pi.Base(), conf.Environment(), p.SitesMatrix) {
+			continue
+		}
+
+		for ck, cv := range cascade.Fields {
+			if done := p.setCascadeEarlyValueIfNotSet(ck, cv); done {
+				break
+			}
+		}
+
+	}
+
+	if sitesMatrixBaseOnly {
+		p.SitesMatrix = buildSitesMatrixFromVectorIterator(sitesMatrixBase)
+	} else {
+		p.SitesMatrix = buildSitesMatrixFromSitesConfig(
+			conf,
+			sitesMatrixBase,
+			p.Sites,
+		)
+	}
+
+	// Finally apply the cascades with site filtering.
+	// These may also contain site specific settings,
+	// so preserve the original matrix for comparison.
+	sitesMatrixBefore := p.Sites.Matrix
+	var hadCascadeMatch bool
+
+	for cascade := range cascades.All() {
+		if cascade.Target.SitesMatrixCompiled == nil {
+			continue
+		}
+
+		if !cascade.Target.Match(p.Kind, pi.Base(), conf.Environment(), p.SitesMatrix) {
+			continue
+		}
+
+		hadCascadeMatch = true
+
+		for ck, cv := range cascade.Fields {
+			if done := p.setCascadeEarlyValueIfNotSet(ck, cv); done {
+				break
+			}
+		}
+
+	}
+
+	if hadCascadeMatch && !sitesMatrixBaseOnly && !sitesMatrixBefore.Equal(p.Sites.Matrix) {
+		// Matrix has changed, rebuild.
+		p.SitesMatrix = buildSitesMatrixFromSitesConfig(
+			conf,
+			sitesMatrixBase,
+			p.Sites,
+		)
+	}
+
+	p.SitesComplements = buildSitesComplementsFromSitesConfig(
+		conf,
+		fim,
+		p.Sites,
+	)
+
+	mediaTypes := conf.GetConfigSection("mediaTypes").(media.Types)
+	if err := p.resolveContentType(pi.Ext(), mediaTypes); err != nil {
+		return err
 	}
 
 	return nil
 }
 
-func (p *PageConfig) CompileForPagesFromDataPre(basePath string, logger loggers.Logger, mediaTypes media.Types) error {
+func (p *PageConfigEarly) CompileForPagesFromDataPre(basePath string, logger loggers.Logger, mediaTypes media.Types) error {
 	// In content adapters, we always get relative paths.
 	if basePath != "" {
 		p.Path = path.Join(basePath, p.Path)
 	}
 
-	if p.Params == nil {
-		p.Params = make(maps.Params)
-	} else {
-		p.Params = maps.PrepareParamsClone(p.Params)
-	}
-
 	if p.Kind == "" {
 		p.Kind = kinds.KindPage
 	}
 
-	if p.Cascade != nil {
-		cascade, err := page.DecodeCascade(logger, false, p.Cascade)
-		if err != nil {
-			return fmt.Errorf("failed to decode cascade: %w", err)
-		}
-		p.CascadeCompiled = cascade
-	}
-
 	// Note that NormalizePathStringBasic will make sure that we don't preserve the unnormalized path.
 	// We do that when we create pages from the file system; mostly for backward compatibility,
 	// but also because people tend to use use the filename to name their resources (with spaces and all),
 	// and this isn't relevant when creating resources from an API where it's easy to add textual meta data.
 	p.Path = paths.NormalizePathStringBasic(p.Path)
 
-	return p.compilePrePost("", mediaTypes)
+	return p.resolveContentType("", mediaTypes)
 }
 
-func (p *PageConfig) compilePrePost(ext string, mediaTypes media.Types) error {
+func (p *PageConfigEarly) resolveContentType(ext string, mediaTypes media.Types) error {
 	if p.Content.Markup == "" && p.Content.MediaType == "" {
 		if ext == "" {
 			ext = "md"
@@ -242,14 +428,12 @@ func (p *PageConfig) compilePrePost(ext string, mediaTypes media.Types) error {
 }
 
 // Compile sets up the page configuration after all fields have been set.
-func (p *PageConfig) Compile(ext string, logger loggers.Logger, outputFormats output.Formats, mediaTypes media.Types) error {
-	if p.IsFromContentAdapter {
+func (p *PageConfigLate) Compile(e *PageConfigEarly, logger loggers.Logger, outputFormats output.Formats) error {
+	if e.IsFromContentAdapter {
 		if err := mapstructure.WeakDecode(p.ContentAdapterData, p); err != nil {
 			err = fmt.Errorf("failed to decode page map: %w", err)
 			return err
 		}
-		// Not needed anymore.
-		p.ContentAdapterData = nil
 	}
 
 	if p.Params == nil {
@@ -258,10 +442,6 @@ func (p *PageConfig) Compile(ext string, logger loggers.Logger, outputFormats ou
 		maps.PrepareParams(p.Params)
 	}
 
-	if err := p.compilePrePost(ext, mediaTypes); err != nil {
-		return err
-	}
-
 	if len(p.Outputs) > 0 {
 		outFormats, err := outputFormats.GetByNames(p.Outputs...)
 		if err != nil {
@@ -287,10 +467,13 @@ type ResourceConfig struct {
 	Title   string
 	Params  maps.Params
 	Content Source
+	Sites   sitesmatrix.Sites
 
 	// Compiled values.
-	PathInfo         *paths.Path `mapstructure:"-" json:"-"`
-	ContentMediaType media.Type
+	ContentAdapterSourceEntryHash uint64      `mapstructure:"-" json:"-"`
+	PathInfo                      *paths.Path `mapstructure:"-" json:"-"`
+	ContentMediaType              media.Type  `mapstructure:"-" json:"-"`
+	SitesMatrixAndComplements     `mapstructure:"-" json:"-"`
 }
 
 func (rc *ResourceConfig) Validate() error {
@@ -300,7 +483,7 @@ func (rc *ResourceConfig) Validate() error {
 	return nil
 }
 
-func (rc *ResourceConfig) Compile(basePath string, pathParser *paths.PathParser, mediaTypes media.Types) error {
+func (rc *ResourceConfig) Compile(basePath string, fim hugofs.FileMetaInfo, conf config.AllProvider, mediaTypes media.Types) error {
 	if rc.Params != nil {
 		maps.PrepareParams(rc.Params)
 	}
@@ -310,7 +493,7 @@ func (rc *ResourceConfig) Compile(basePath string, pathParser *paths.PathParser,
 	// but also because people tend to use use the filename to name their resources (with spaces and all),
 	// and this isn't relevant when creating resources from an API where it's easy to add textual meta data.
 	rc.Path = paths.NormalizePathStringBasic(path.Join(basePath, rc.Path))
-	rc.PathInfo = pathParser.Parse(files.ComponentFolderContent, rc.Path)
+	rc.PathInfo = conf.PathParser().Parse(files.ComponentFolderContent, rc.Path)
 	if rc.Content.MediaType != "" {
 		var found bool
 		rc.ContentMediaType, found = mediaTypes.GetByType(rc.Content.MediaType)
@@ -318,6 +501,24 @@ func (rc *ResourceConfig) Compile(basePath string, pathParser *paths.PathParser,
 			return fmt.Errorf("media type %q not found", rc.Content.MediaType)
 		}
 	}
+
+	var sitesMatrixFile sitesmatrix.VectorStore
+	if fim != nil {
+		sitesMatrixFile = fim.Meta().SitesMatrix
+	}
+
+	rc.SitesMatrix = buildSitesMatrixFromSitesConfig(
+		conf,
+		sitesMatrixFile,
+		rc.Sites,
+	)
+
+	rc.SitesComplements = buildSitesComplementsFromSitesConfig(
+		conf,
+		fim.Meta(),
+		rc.Sites,
+	)
+
 	return nil
 }
 
@@ -395,11 +596,11 @@ type FrontMatterDescriptor struct {
 	// May be set from the author date in Git.
 	GitAuthorDate time.Time
 
-	// The below will be modified.
-	PageConfig *PageConfig
+	PageConfigEarly *PageConfigEarly
 
-	// The Location to use to parse dates without time zone info.
-	Location *time.Location
+	// The below will be modified.
+	PageConfigLate *PageConfigLate
+	Location       *time.Location // The Location to use to parse dates without time zone info.
 }
 
 var dateFieldAliases = map[string][]string{
@@ -413,11 +614,11 @@ var dateFieldAliases = map[string][]string{
 // supplied front matter params. Note that this requires all lower-case keys
 // in the params map.
 func (f FrontMatterHandler) HandleDates(d *FrontMatterDescriptor) error {
-	if d.PageConfig == nil {
+	if d.PageConfigLate == nil {
 		panic("missing pageConfig")
 	}
 
-	if d.PageConfig.IsFromContentAdapter {
+	if d.PageConfigEarly.IsFromContentAdapter {
 		if f.contentAdapterDatesHandler == nil {
 			panic("missing content adapter date handler")
 		}
@@ -592,7 +793,7 @@ func addDateFieldAliases(values []string) []string {
 			complete = append(complete, aliases...)
 		}
 	}
-	return helpers.UniqueStringsReuse(complete)
+	return hstrings.UniqueStringsReuse(complete)
 }
 
 func expandDefaultValues(values []string, defaults []string) []string {
@@ -655,7 +856,7 @@ func (f *FrontMatterHandler) createHandlers() error {
 
 	if f.dateHandler, err = f.createDateHandler(f.fmConfig.Date,
 		func(d *FrontMatterDescriptor, t time.Time) {
-			d.PageConfig.Dates.Date = t
+			d.PageConfigLate.Dates.Date = t
 			setParamIfNotSet(fmDate, t, d)
 		}); err != nil {
 		return err
@@ -664,7 +865,7 @@ func (f *FrontMatterHandler) createHandlers() error {
 	if f.lastModHandler, err = f.createDateHandler(f.fmConfig.Lastmod,
 		func(d *FrontMatterDescriptor, t time.Time) {
 			setParamIfNotSet(fmLastmod, t, d)
-			d.PageConfig.Dates.Lastmod = t
+			d.PageConfigLate.Dates.Lastmod = t
 		}); err != nil {
 		return err
 	}
@@ -672,7 +873,7 @@ func (f *FrontMatterHandler) createHandlers() error {
 	if f.publishDateHandler, err = f.createDateHandler(f.fmConfig.PublishDate,
 		func(d *FrontMatterDescriptor, t time.Time) {
 			setParamIfNotSet(fmPubDate, t, d)
-			d.PageConfig.Dates.PublishDate = t
+			d.PageConfigLate.Dates.PublishDate = t
 		}); err != nil {
 		return err
 	}
@@ -680,7 +881,7 @@ func (f *FrontMatterHandler) createHandlers() error {
 	if f.expiryDateHandler, err = f.createDateHandler(f.fmConfig.ExpiryDate,
 		func(d *FrontMatterDescriptor, t time.Time) {
 			setParamIfNotSet(fmExpiryDate, t, d)
-			d.PageConfig.Dates.ExpiryDate = t
+			d.PageConfigLate.Dates.ExpiryDate = t
 		}); err != nil {
 		return err
 	}
@@ -689,14 +890,14 @@ func (f *FrontMatterHandler) createHandlers() error {
 }
 
 func setParamIfNotSet(key string, value any, d *FrontMatterDescriptor) {
-	if _, found := d.PageConfig.Params[key]; found {
+	if _, found := d.PageConfigLate.Params[key]; found {
 		return
 	}
-	d.PageConfig.Params[key] = value
+	d.PageConfigLate.Params[key] = value
 }
 
 func (f FrontMatterHandler) createContentAdapterDatesHandler(fmcfg FrontmatterConfig) (func(d *FrontMatterDescriptor) error, error) {
-	setTime := func(key string, value time.Time, in *PageConfig) {
+	setTime := func(key string, value time.Time, in *PageConfigLate) {
 		switch key {
 		case fmDate:
 			in.Dates.Date = value
@@ -709,7 +910,7 @@ func (f FrontMatterHandler) createContentAdapterDatesHandler(fmcfg FrontmatterCo
 		}
 	}
 
-	getTime := func(key string, in *PageConfig) time.Time {
+	getTime := func(key string, in *PageConfigLate) time.Time {
 		switch key {
 		case fmDate:
 			return in.Dates.Date
@@ -723,33 +924,33 @@ func (f FrontMatterHandler) createContentAdapterDatesHandler(fmcfg FrontmatterCo
 		return time.Time{}
 	}
 
-	createSetter := func(identifiers []string, date string) func(pcfg *PageConfig) {
-		var getTimes []func(in *PageConfig) time.Time
+	createSetter := func(identifiers []string, date string) func(pcfg *PageConfigLate) {
+		var getTimes []func(in *PageConfigLate) time.Time
 		for _, identifier := range identifiers {
 			if strings.HasPrefix(identifier, ":") {
 				continue
 			}
 			switch identifier {
 			case fmDate:
-				getTimes = append(getTimes, func(in *PageConfig) time.Time {
+				getTimes = append(getTimes, func(in *PageConfigLate) time.Time {
 					return getTime(fmDate, in)
 				})
 			case fmLastmod:
-				getTimes = append(getTimes, func(in *PageConfig) time.Time {
+				getTimes = append(getTimes, func(in *PageConfigLate) time.Time {
 					return getTime(fmLastmod, in)
 				})
 			case fmPubDate:
-				getTimes = append(getTimes, func(in *PageConfig) time.Time {
+				getTimes = append(getTimes, func(in *PageConfigLate) time.Time {
 					return getTime(fmPubDate, in)
 				})
 			case fmExpiryDate:
-				getTimes = append(getTimes, func(in *PageConfig) time.Time {
+				getTimes = append(getTimes, func(in *PageConfigLate) time.Time {
 					return getTime(fmExpiryDate, in)
 				})
 			}
 		}
 
-		return func(pcfg *PageConfig) {
+		return func(pcfg *PageConfigLate) {
 			for _, get := range getTimes {
 				if t := get(pcfg); !t.IsZero() {
 					setTime(date, t, pcfg)
@@ -765,7 +966,7 @@ func (f FrontMatterHandler) createContentAdapterDatesHandler(fmcfg FrontmatterCo
 	setExpiryDate := createSetter(fmcfg.ExpiryDate, fmExpiryDate)
 
 	fn := func(d *FrontMatterDescriptor) error {
-		pcfg := d.PageConfig
+		pcfg := d.PageConfigLate
 		setDate(pcfg)
 		setLastmod(pcfg)
 		setPublishDate(pcfg)
@@ -799,7 +1000,13 @@ type frontmatterFieldHandlers int
 
 func (f *frontmatterFieldHandlers) newDateFieldHandler(key string, setter func(d *FrontMatterDescriptor, t time.Time)) frontMatterFieldHandler {
 	return func(d *FrontMatterDescriptor) (bool, error) {
-		v, found := d.PageConfig.Params[key]
+		v, found := d.PageConfigEarly.Frontmatter[key]
+		if found {
+			d.PageConfigLate.Params[key] = v
+		} else {
+			// Reentry from previous handlers.
+			v, found = d.PageConfigLate.Params[key]
+		}
 
 		if !found || v == "" || v == nil {
 			return false, nil
@@ -814,7 +1021,7 @@ func (f *frontmatterFieldHandlers) newDateFieldHandler(key string, setter func(d
 			if err != nil {
 				return false, fmt.Errorf("the %q front matter field is not a parsable date: see %s", key, d.PathOrTitle)
 			}
-			d.PageConfig.Params[key] = date
+			d.PageConfigLate.Params[key] = date
 		}
 
 		// We map several date keys to one, so, for example,
@@ -834,9 +1041,9 @@ func (f *frontmatterFieldHandlers) newDateFilenameHandler(setter func(d *FrontMa
 
 		setter(d, date)
 
-		if _, found := d.PageConfig.Params["slug"]; !found {
+		if _, found := d.PageConfigEarly.Frontmatter["slug"]; !found {
 			// Use slug from filename
-			d.PageConfig.Slug = slug
+			d.PageConfigLate.Slug = slug
 		}
 
 		return true, nil
diff --git a/resources/page/pagemeta/page_frontmatter_test.go b/resources/page/pagemeta/page_frontmatter_test.go
index 8d50f9b57..39074ee15 100644
--- a/resources/page/pagemeta/page_frontmatter_test.go
+++ b/resources/page/pagemeta/page_frontmatter_test.go
@@ -22,7 +22,6 @@ import (
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/config/testconfig"
 	"github.com/gohugoio/hugo/media"
-	"github.com/gohugoio/hugo/output"
 
 	"github.com/gohugoio/hugo/resources/page/pagemeta"
 
@@ -31,7 +30,10 @@ import (
 
 func newTestFd() *pagemeta.FrontMatterDescriptor {
 	return &pagemeta.FrontMatterDescriptor{
-		PageConfig: &pagemeta.PageConfig{
+		PageConfigEarly: &pagemeta.PageConfigEarly{
+			Frontmatter: make(map[string]any),
+		},
+		PageConfigLate: &pagemeta.PageConfigLate{
 			Params: make(map[string]any),
 		},
 		Location: time.UTC,
@@ -107,16 +109,16 @@ func TestFrontMatterDatesHandlers(t *testing.T) {
 		case ":git":
 			d.GitAuthorDate = d1
 		}
-		d.PageConfig.Params["date"] = d2
+		d.PageConfigEarly.Frontmatter["date"] = d2
 		c.Assert(handler.HandleDates(d), qt.IsNil)
-		c.Assert(d.PageConfig.Dates.Date, qt.Equals, d1)
-		c.Assert(d.PageConfig.Params["date"], qt.Equals, d2)
+		c.Assert(d.PageConfigLate.Dates.Date, qt.Equals, d1)
+		c.Assert(d.PageConfigLate.Params["date"], qt.Equals, d2)
 
 		d = newTestFd()
-		d.PageConfig.Params["date"] = d2
+		d.PageConfigEarly.Frontmatter["date"] = d2
 		c.Assert(handler.HandleDates(d), qt.IsNil)
-		c.Assert(d.PageConfig.Dates.Date, qt.Equals, d2)
-		c.Assert(d.PageConfig.Params["date"], qt.Equals, d2)
+		c.Assert(d.PageConfigLate.Dates.Date, qt.Equals, d2)
+		c.Assert(d.PageConfigLate.Params["date"], qt.Equals, d2)
 
 	}
 }
@@ -139,17 +141,17 @@ func TestFrontMatterDatesDefaultKeyword(t *testing.T) {
 
 	testDate, _ := time.Parse("2006-01-02", "2018-02-01")
 	d := newTestFd()
-	d.PageConfig.Params["mydate"] = testDate
-	d.PageConfig.Params["date"] = testDate.Add(1 * 24 * time.Hour)
-	d.PageConfig.Params["mypubdate"] = testDate.Add(2 * 24 * time.Hour)
-	d.PageConfig.Params["publishdate"] = testDate.Add(3 * 24 * time.Hour)
+	d.PageConfigEarly.Frontmatter["mydate"] = testDate
+	d.PageConfigEarly.Frontmatter["date"] = testDate.Add(1 * 24 * time.Hour)
+	d.PageConfigEarly.Frontmatter["mypubdate"] = testDate.Add(2 * 24 * time.Hour)
+	d.PageConfigEarly.Frontmatter["publishdate"] = testDate.Add(3 * 24 * time.Hour)
 
 	c.Assert(handler.HandleDates(d), qt.IsNil)
 
-	c.Assert(d.PageConfig.Dates.Date.Day(), qt.Equals, 1)
-	c.Assert(d.PageConfig.Dates.Lastmod.Day(), qt.Equals, 2)
-	c.Assert(d.PageConfig.Dates.PublishDate.Day(), qt.Equals, 4)
-	c.Assert(d.PageConfig.Dates.ExpiryDate.IsZero(), qt.Equals, true)
+	c.Assert(d.PageConfigLate.Dates.Date.Day(), qt.Equals, 1)
+	c.Assert(d.PageConfigLate.Dates.Lastmod.Day(), qt.Equals, 2)
+	c.Assert(d.PageConfigLate.Dates.PublishDate.Day(), qt.Equals, 4)
+	c.Assert(d.PageConfigLate.Dates.ExpiryDate.IsZero(), qt.Equals, true)
 }
 
 func TestContentMediaTypeFromMarkup(t *testing.T) {
@@ -174,9 +176,9 @@ func TestContentMediaTypeFromMarkup(t *testing.T) {
 		{"pdc", "text/pandoc"},
 		{"rst", "text/rst"},
 	} {
-		var pc pagemeta.PageConfig
+		var pc pagemeta.PageConfigEarly
 		pc.Content.Markup = test.in
-		c.Assert(pc.Compile("", logger, output.DefaultFormats, media.DefaultTypes), qt.IsNil)
+		c.Assert(pc.CompileForPagesFromDataPre("", logger, media.DefaultTypes), qt.IsNil)
 		c.Assert(pc.ContentMediaType.Type, qt.Equals, test.expected)
 	}
 }
diff --git a/resources/page/pagemeta/pagemeta.go b/resources/page/pagemeta/pagemeta.go
index b6b953231..8ae2e3cd9 100644
--- a/resources/page/pagemeta/pagemeta.go
+++ b/resources/page/pagemeta/pagemeta.go
@@ -28,7 +28,6 @@ var DefaultBuildConfig = BuildConfig{
 	List:             Always,
 	Render:           Always,
 	PublishResources: true,
-	set:              true,
 }
 
 // BuildConfig holds configuration options about how to handle a Page in Hugo's
@@ -52,8 +51,6 @@ type BuildConfig struct {
 	// but enabling this can be useful if the originals (e.g. images) are
 	// never used.
 	PublishResources bool
-
-	set bool // BuildCfg is non-zero if this is set to true.
 }
 
 // Disable sets all options to their off value.
@@ -61,11 +58,10 @@ func (b *BuildConfig) Disable() {
 	b.List = Never
 	b.Render = Never
 	b.PublishResources = false
-	b.set = true
 }
 
 func (b BuildConfig) IsZero() bool {
-	return !b.set
+	return b == BuildConfig{}
 }
 
 func DecodeBuildConfig(m any) (BuildConfig, error) {
diff --git a/resources/page/pagemeta/pagemeta_test.go b/resources/page/pagemeta/pagemeta_test.go
index 80c19d07f..e2c4ebf15 100644
--- a/resources/page/pagemeta/pagemeta_test.go
+++ b/resources/page/pagemeta/pagemeta_test.go
@@ -46,38 +46,32 @@ publishResources = true`
 				Render:           Always,
 				List:             Always,
 				PublishResources: true,
-				set:              true,
 			},
 		},
 		{[]any{"true", "false"}, BuildConfig{
 			Render:           Always,
 			List:             Never,
 			PublishResources: true,
-			set:              true,
 		}},
 		{[]any{`"always"`, `"always"`}, BuildConfig{
 			Render:           Always,
 			List:             Always,
 			PublishResources: true,
-			set:              true,
 		}},
 		{[]any{`"never"`, `"never"`}, BuildConfig{
 			Render:           Never,
 			List:             Never,
 			PublishResources: true,
-			set:              true,
 		}},
 		{[]any{`"link"`, `"local"`}, BuildConfig{
 			Render:           Link,
 			List:             ListLocally,
 			PublishResources: true,
-			set:              true,
 		}},
 		{[]any{`"always"`, `"asdfadf"`}, BuildConfig{
 			Render:           Always,
 			List:             Always,
 			PublishResources: true,
-			set:              true,
 		}},
 	} {
 		cfg, err := config.FromConfigString(fmt.Sprintf(configTempl, test.args...), "toml")
diff --git a/resources/page/pages_sort.go b/resources/page/pages_sort.go
index e77bb7e7c..da3386a47 100644
--- a/resources/page/pages_sort.go
+++ b/resources/page/pages_sort.go
@@ -18,6 +18,7 @@ import (
 	"sort"
 
 	"github.com/gohugoio/hugo/common/collections"
+	"github.com/gohugoio/hugo/common/types"
 	"github.com/gohugoio/hugo/langs"
 
 	"github.com/gohugoio/hugo/resources/resource"
@@ -54,14 +55,14 @@ func getOrdinals(p1, p2 Page) (int, int) {
 	return p1o.Ordinal(), p2o.Ordinal()
 }
 
-func getWeight0s(p1, p2 Page) (int, int) {
-	p1w, ok1 := p1.(resource.Weight0Provider)
+func getWeight0s(p1, p2 Page) (w1 int, w2 int) {
+	p1w, ok1 := p1.(types.Weight0Provider)
 	if !ok1 {
-		return -1, -1
+		return
 	}
-	p2w, ok2 := p2.(resource.Weight0Provider)
+	p2w, ok2 := p2.(types.Weight0Provider)
 	if !ok2 {
-		return -1, -1
+		return
 	}
 
 	return p1w.Weight0(), p2w.Weight0()
@@ -155,6 +156,16 @@ var (
 	lessPagePubDate = func(p1, p2 Page) bool {
 		return p1.PublishDate().Unix() < p2.PublishDate().Unix()
 	}
+
+	lessPageDims = func(p1, p2 Page) bool {
+		d1, d2 := GetSiteVector(p1), GetSiteVector(p2)
+		if n := d1.Compare(d2); n != 0 {
+			return n < 0
+		}
+
+		// Fall back to the default sort order.
+		return DefaultPageSort(p1, p2)
+	}
 )
 
 func (ps *pageSorter) Len() int      { return len(ps.pages) }
@@ -360,6 +371,11 @@ func SortByLanguage(pages Pages) {
 	pageBy(lessPageLanguage).Sort(pages)
 }
 
+// SortByDims sorts the pages by sitesmatrix.
+func SortByDims(pages Pages) {
+	pageBy(lessPageDims).Sort(pages)
+}
+
 // Reverse reverses the order in Pages and returns a copy.
 //
 // Adjacent invocations on the same receiver will return a cached result.
diff --git a/resources/page/site.go b/resources/page/site.go
index 4b48ad538..33dd94736 100644
--- a/resources/page/site.go
+++ b/resources/page/site.go
@@ -20,6 +20,8 @@ import (
 	"github.com/gohugoio/hugo/common/maps"
 	"github.com/gohugoio/hugo/config/privacy"
 	"github.com/gohugoio/hugo/config/services"
+	"github.com/gohugoio/hugo/hugolib/roles"
+	"github.com/gohugoio/hugo/hugolib/versions"
 	"github.com/gohugoio/hugo/identity"
 
 	"github.com/gohugoio/hugo/config"
@@ -34,6 +36,15 @@ type Site interface {
 	// Returns the Language configured for this Site.
 	Language() *langs.Language
 
+	// Returns the Site in one of dimensions language, version or role.
+	Dimension(string) SiteDimension
+
+	// Returns the role configured for this Site.
+	Role() roles.Role
+
+	// Returns the version configured for this Site.
+	Version() versions.Version
+
 	// Returns all the languages configured for all sites.
 	Languages() langs.Languages
 
@@ -124,6 +135,9 @@ type Site interface {
 	LanguagePrefix() string
 
 	hstore.StoreProvider
+	// String returns a string representation of the site.
+	// Note that this represenetation may change in the future.
+	String() string
 
 	// For internal use only.
 	// This will panic if the site is not fully initialized.
@@ -132,6 +146,11 @@ type Site interface {
 	CheckReady()
 }
 
+// SiteDimension represents a dimension of the site.
+type SiteDimension interface {
+	Name() string
+}
+
 // Sites represents an ordered list of sites (languages).
 type Sites []Site
 
@@ -147,6 +166,11 @@ func (s Sites) Default() Site {
 	if len(s) == 0 {
 		return nil
 	}
+	for _, site := range s {
+		if site.Language().IsDefault() {
+			return site
+		}
+	}
 	return s[0]
 }
 
@@ -195,6 +219,18 @@ func (s *siteWrapper) Languages() langs.Languages {
 	return s.s.Languages()
 }
 
+func (s *siteWrapper) Role() roles.Role {
+	return s.s.Role()
+}
+
+func (s *siteWrapper) Dimension(d string) SiteDimension {
+	return s.s.Dimension(d)
+}
+
+func (s *siteWrapper) Version() versions.Version {
+	return s.s.Version()
+}
+
 func (s *siteWrapper) AllPages() Pages {
 	return s.s.AllPages()
 }
@@ -301,6 +337,10 @@ func (s *siteWrapper) Store() *hstore.Scratch {
 	return s.s.Store()
 }
 
+func (s *siteWrapper) String() string {
+	return s.s.String()
+}
+
 // For internal use only.
 func (s *siteWrapper) ForEeachIdentityByName(name string, f func(identity.Identity) bool) {
 	s.s.(identity.ForEeachIdentityByNameProvider).ForEeachIdentityByName(name, f)
@@ -384,6 +424,10 @@ func (t testSite) Languages() langs.Languages {
 	return nil
 }
 
+func (t testSite) Dimension(d string) SiteDimension {
+	return nil
+}
+
 func (t testSite) MainSections() []string {
 	return nil
 }
@@ -392,6 +436,14 @@ func (t testSite) Language() *langs.Language {
 	return t.l
 }
 
+func (t testSite) Role() roles.Role {
+	return nil
+}
+
+func (t testSite) Version() versions.Version {
+	return nil
+}
+
 func (t testSite) Home() Page {
 	return nil
 }
@@ -449,6 +501,10 @@ func (s testSite) Store() *hstore.Scratch {
 	return hstore.NewScratch()
 }
 
+func (s testSite) String() string {
+	return "testSite"
+}
+
 func (s testSite) CheckReady() {
 }
 
diff --git a/resources/resource.go b/resources/resource.go
index f6e5b9d73..0e7b36f85 100644
--- a/resources/resource.go
+++ b/resources/resource.go
@@ -24,11 +24,11 @@ import (
 	"sync/atomic"
 
 	"github.com/gohugoio/hugo/identity"
-	"github.com/gohugoio/hugo/lazy"
 	"github.com/gohugoio/hugo/resources/internal"
 
 	"github.com/gohugoio/hugo/common/hashing"
 	"github.com/gohugoio/hugo/common/herrors"
+	"github.com/gohugoio/hugo/common/hsync"
 	"github.com/gohugoio/hugo/common/paths"
 
 	"github.com/gohugoio/hugo/media"
@@ -185,7 +185,7 @@ func (fd *ResourceSourceDescriptor) init(r *Spec) error {
 	fd.MediaType = mediaType
 
 	if fd.DependencyManager == nil {
-		fd.DependencyManager = r.Cfg.NewIdentityManager("resource")
+		fd.DependencyManager = r.Cfg.NewIdentityManager()
 	}
 
 	return nil
@@ -358,7 +358,7 @@ func GetTestInfoForResource(r resource.Resource) GenericResourceTestInfo {
 
 // genericResource represents a generic linkable resource.
 type genericResource struct {
-	publishInit *lazy.OnceMore
+	publishInit *hsync.OnceMore
 
 	key     string
 	keyInit *sync.Once
@@ -636,7 +636,7 @@ func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResour
 }
 
 func (l genericResource) clone() *genericResource {
-	l.publishInit = &lazy.OnceMore{}
+	l.publishInit = &hsync.OnceMore{}
 	l.keyInit = &sync.Once{}
 	return &l
 }
diff --git a/resources/resource/resources.go b/resources/resource/resources.go
index 6b7311bad..013d9b097 100644
--- a/resources/resource/resources.go
+++ b/resources/resource/resources.go
@@ -17,14 +17,14 @@ package resource
 import (
 	"fmt"
 	"path"
+	"slices"
 	"strings"
 
 	"github.com/gohugoio/hugo/common/hreflect"
 	"github.com/gohugoio/hugo/common/maps"
 	"github.com/gohugoio/hugo/common/paths"
-	"github.com/gohugoio/hugo/hugofs/glob"
+	"github.com/gohugoio/hugo/hugofs/hglob"
 	"github.com/spf13/cast"
-	"slices"
 )
 
 var _ ResourceFinder = (*Resources)(nil)
@@ -155,7 +155,7 @@ func (r Resources) GetMatch(pattern any) Resource {
 		panic(err)
 	}
 
-	g, err := glob.GetGlob(paths.AddLeadingSlash(patternstr))
+	g, err := hglob.GetGlob(paths.AddLeadingSlash(patternstr))
 	if err != nil {
 		panic(err)
 	}
@@ -193,7 +193,7 @@ func (r Resources) Match(pattern any) Resources {
 		panic(err)
 	}
 
-	g, err := glob.GetGlob(paths.AddLeadingSlash(patternstr))
+	g, err := hglob.GetGlob(paths.AddLeadingSlash(patternstr))
 	if err != nil {
 		panic(err)
 	}
diff --git a/resources/resource/resourcetypes.go b/resources/resource/resourcetypes.go
index 51255c612..e55e7169f 100644
--- a/resources/resource/resourcetypes.go
+++ b/resources/resource/resourcetypes.go
@@ -173,17 +173,6 @@ type TransientIdentifier interface {
 	TransientKey() string
 }
 
-// WeightProvider provides a weight.
-type WeightProvider interface {
-	Weight() int
-}
-
-// Weight0Provider provides a weight that's considered before the WeightProvider in sorting.
-// This allows the weight set on a given term to win.
-type Weight0Provider interface {
-	Weight0() int
-}
-
 // ContentResource represents a Resource that provides a way to get to its content.
 // Most Resource types in Hugo implements this interface, including Page.
 type ContentResource interface {
@@ -219,6 +208,7 @@ type LengthProvider interface {
 // LanguageProvider is a Resource in a language.
 type LanguageProvider interface {
 	Language() *langs.Language
+	Lang() string
 }
 
 // TranslationKeyProvider connects translations of the same Resource.
diff --git a/resources/resource_factories/bundler/bundler.go b/resources/resource_factories/bundler/bundler.go
index aef644b7f..19150525b 100644
--- a/resources/resource_factories/bundler/bundler.go
+++ b/resources/resource_factories/bundler/bundler.go
@@ -94,7 +94,7 @@ func (c *Client) Concat(targetPath string, r resource.Resources) (resource.Resou
 			resolvedm = rr.MediaType()
 		}
 
-		idm := c.rs.Cfg.NewIdentityManager("concat")
+		idm := c.rs.Cfg.NewIdentityManager()
 
 		// Re-create on structural changes.
 		idm.AddIdentity(identity.StructuralChangeAdd, identity.StructuralChangeRemove)
diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go
index e8a0b4c4d..d51039eba 100644
--- a/resources/resource_factories/create/create.go
+++ b/resources/resource_factories/create/create.go
@@ -28,7 +28,7 @@ import (
 	"github.com/bep/logg"
 	"github.com/gohugoio/httpcache"
 	hhttpcache "github.com/gohugoio/hugo/cache/httpcache"
-	"github.com/gohugoio/hugo/hugofs/glob"
+	"github.com/gohugoio/hugo/hugofs/hglob"
 	"github.com/gohugoio/hugo/identity"
 
 	"github.com/gohugoio/hugo/hugofs"
@@ -206,8 +206,8 @@ func (c *Client) getOrCreateFileResource(info hugofs.FileMetaInfo) (resource.Res
 }
 
 func (c *Client) match(name, pattern string, matchFunc func(r resource.Resource) bool, firstOnly bool) (resource.Resources, error) {
-	pattern = glob.NormalizePath(pattern)
-	partitions := glob.FilterGlobParts(strings.Split(pattern, "/"))
+	pattern = hglob.NormalizePath(pattern)
+	partitions := hglob.FilterGlobParts(strings.Split(pattern, "/"))
 	key := path.Join(name, path.Join(partitions...))
 	key = path.Join(key, pattern)
 
diff --git a/resources/resource_metadata.go b/resources/resource_metadata.go
index 2a4faa315..955809c93 100644
--- a/resources/resource_metadata.go
+++ b/resources/resource_metadata.go
@@ -19,16 +19,17 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/gohugoio/hugo/hugofs/glob"
+	"github.com/gohugoio/hugo/hugofs/hglob"
 	"github.com/gohugoio/hugo/media"
 	"github.com/gohugoio/hugo/resources/page/pagemeta"
 	"github.com/gohugoio/hugo/resources/resource"
 
 	"github.com/spf13/cast"
 
+	maps0 "maps"
+
 	"github.com/gohugoio/hugo/common/maps"
 	"github.com/gohugoio/hugo/common/paths"
-	maps0 "maps"
 )
 
 var (
@@ -161,7 +162,7 @@ func assignMetadata(metadata []map[string]any, ma *metaResource) error {
 
 		srcKey := strings.ToLower(cast.ToString(src))
 
-		glob, err := glob.GetGlob(srcKey)
+		glob, err := hglob.GetGlob(srcKey)
 		if err != nil {
 			return fmt.Errorf("failed to match resource with metadata: %w", err)
 		}
diff --git a/resources/resource_spec.go b/resources/resource_spec.go
index 806a5ff8d..00fb637ae 100644
--- a/resources/resource_spec.go
+++ b/resources/resource_spec.go
@@ -20,7 +20,6 @@ import (
 
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/config/allconfig"
-	"github.com/gohugoio/hugo/lazy"
 	"github.com/gohugoio/hugo/output"
 	"github.com/gohugoio/hugo/resources/internal"
 	"github.com/gohugoio/hugo/resources/jsconfig"
@@ -28,6 +27,7 @@ import (
 
 	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/common/hexec"
+	"github.com/gohugoio/hugo/common/hsync"
 	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/common/paths"
 	"github.com/gohugoio/hugo/common/types"
@@ -190,7 +190,7 @@ func (r *Spec) NewResource(rd ResourceSourceDescriptor) (resource.Resource, erro
 	gr := &genericResource{
 		Staler:           &AtomicStaler{},
 		h:                &resourceHash{},
-		publishInit:      &lazy.OnceMore{},
+		publishInit:      &hsync.OnceMore{},
 		keyInit:          &sync.Once{},
 		includeHashInKey: isImage,
 		paths:            rp,
diff --git a/resources/resources_integration_test.go b/resources/resources_integration_test.go
index 0d02b45d5..e13ae72b3 100644
--- a/resources/resources_integration_test.go
+++ b/resources/resources_integration_test.go
@@ -134,7 +134,6 @@ iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAA
 
 		b.AssertFileCount("resources/_gen/images", 6)
 		b.AssertFileCount("public/images", 1)
-		b.Build()
 	}
 }
 
diff --git a/source/content_directory_test.go b/source/content_directory_test.go
index 96ee22bc7..fea83fd52 100644
--- a/source/content_directory_test.go
+++ b/source/content_directory_test.go
@@ -65,7 +65,7 @@ func TestIgnoreDotFilesAndDirectories(t *testing.T) {
 			afs := afero.NewMemMapFs()
 			conf := testconfig.GetTestConfig(afs, v)
 			fs := hugofs.NewFromOld(afs, v)
-			ps, err := helpers.NewPathSpec(fs, conf, nil)
+			ps, err := helpers.NewPathSpec(fs, conf, nil, nil)
 			c.Assert(err, qt.IsNil)
 
 			s := source.NewSourceSpec(ps, nil, fs.Source)
diff --git a/source/fileInfo.go b/source/fileInfo.go
index fff78e2fe..10e809ba5 100644
--- a/source/fileInfo.go
+++ b/source/fileInfo.go
@@ -18,6 +18,7 @@ import (
 	"sync"
 
 	"github.com/bep/gitmap"
+	"github.com/bep/logg"
 	"github.com/gohugoio/hugo/common/hashing"
 	"github.com/gohugoio/hugo/common/hugo"
 	"github.com/gohugoio/hugo/common/paths"
@@ -61,8 +62,9 @@ func (fi *File) Ext() string { return fi.p().Ext() }
 // Lang returns a file's language (e.g. "sv").
 // Deprecated: Use .Page.Language.Lang instead.
 func (fi *File) Lang() string {
-	hugo.Deprecate(".Page.File.Lang", "Use .Page.Language.Lang instead.", "v0.123.0")
-	return fi.fim.Meta().Lang
+	// From Hugo 0.149.0 a file may have multiple languages.
+	hugo.DeprecateLevelMin(".File.Lang", "Use e.g. Page.Site.Language.Lang", "v0.149.0", logg.LevelError)
+	return ""
 }
 
 // LogicalName returns a file's name and extension (e.g. "page.sv.md").
diff --git a/source/sourceSpec.go b/source/sourceSpec.go
index ea1b977f3..f7afa2fc6 100644
--- a/source/sourceSpec.go
+++ b/source/sourceSpec.go
@@ -18,11 +18,10 @@ import (
 	"path/filepath"
 	"runtime"
 
-	"github.com/gohugoio/hugo/hugofs/glob"
-
 	"github.com/spf13/afero"
 
 	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/hugofs/hglob"
 )
 
 // SourceSpec abstracts language-specific file creation.
@@ -36,7 +35,7 @@ type SourceSpec struct {
 }
 
 // NewSourceSpec initializes SourceSpec using languages the given filesystem and PathSpec.
-func NewSourceSpec(ps *helpers.PathSpec, inclusionFilter *glob.FilenameFilter, fs afero.Fs) *SourceSpec {
+func NewSourceSpec(ps *helpers.PathSpec, inclusionFilter *hglob.FilenameFilter, fs afero.Fs) *SourceSpec {
 	shouldInclude := func(filename string) bool {
 		if !inclusionFilter.Match(filename, false) {
 			return false
diff --git a/tpl/collections/collections.go b/tpl/collections/collections.go
index 47f4f139a..6b580b31c 100644
--- a/tpl/collections/collections.go
+++ b/tpl/collections/collections.go
@@ -37,7 +37,7 @@ import (
 
 // New returns a new instance of the collections-namespaced template functions.
 func New(deps *deps.Deps) *Namespace {
-	language := deps.Conf.Language()
+	language := deps.Conf.Language().(*langs.Language)
 	if language == nil {
 		panic("language must be set")
 	}
diff --git a/tpl/collections/sort.go b/tpl/collections/sort.go
index 544cb0013..2a0a4f3bb 100644
--- a/tpl/collections/sort.go
+++ b/tpl/collections/sort.go
@@ -50,7 +50,7 @@ func (ns *Namespace) Sort(ctx context.Context, l any, args ...any) (any, error)
 		return nil, errors.New("can't sort " + reflect.ValueOf(l).Type().String())
 	}
 
-	collator := langs.GetCollator1(ns.deps.Conf.Language())
+	collator := langs.GetCollator1(ns.deps.Conf.Language().(*langs.Language))
 
 	// Create a list of pairs that will be used to do the sort
 	p := pairList{Collator: collator, sortComp: ns.sortComp, SortAsc: true, SliceType: sliceType}
diff --git a/tpl/compare/compare_test.go b/tpl/compare/compare_test.go
index c55322fe3..a94ac5db7 100644
--- a/tpl/compare/compare_test.go
+++ b/tpl/compare/compare_test.go
@@ -22,7 +22,7 @@ import (
 	"time"
 
 	qt "github.com/frankban/quicktest"
-	"github.com/gohugoio/hugo/common/hugo"
+	"github.com/gohugoio/hugo/common/version"
 	"github.com/gohugoio/hugo/htesting/hqt"
 	"github.com/spf13/cast"
 )
@@ -238,17 +238,17 @@ func doTestCompare(t *testing.T, tp tstCompareType, funcUnderTest func(a, b any)
 		{tstEqerType1("a"), tstEqerType2("a"), 0},
 		{tstEqerType2("a"), tstEqerType1("a"), 0},
 		{tstEqerType2("a"), tstEqerType1("b"), -1},
-		{hugo.MustParseVersion("0.32.1").Version(), hugo.MustParseVersion("0.32").Version(), 1},
-		{hugo.MustParseVersion("0.35").Version(), hugo.MustParseVersion("0.32").Version(), 1},
-		{hugo.MustParseVersion("0.36").Version(), hugo.MustParseVersion("0.36").Version(), 0},
-		{hugo.MustParseVersion("0.32").Version(), hugo.MustParseVersion("0.36").Version(), -1},
-		{hugo.MustParseVersion("0.32").Version(), "0.36", -1},
-		{"0.36", hugo.MustParseVersion("0.32").Version(), 1},
-		{"0.36", hugo.MustParseVersion("0.36").Version(), 0},
-		{"0.37", hugo.MustParseVersion("0.37-DEV").Version(), 1},
-		{"0.37-DEV", hugo.MustParseVersion("0.37").Version(), -1},
-		{"0.36", hugo.MustParseVersion("0.37-DEV").Version(), -1},
-		{"0.37-DEV", hugo.MustParseVersion("0.37-DEV").Version(), 0},
+		{version.MustParseVersion("0.32.1").Version(), version.MustParseVersion("0.32").Version(), 1},
+		{version.MustParseVersion("0.35").Version(), version.MustParseVersion("0.32").Version(), 1},
+		{version.MustParseVersion("0.36").Version(), version.MustParseVersion("0.36").Version(), 0},
+		{version.MustParseVersion("0.32").Version(), version.MustParseVersion("0.36").Version(), -1},
+		{version.MustParseVersion("0.32").Version(), "0.36", -1},
+		{"0.36", version.MustParseVersion("0.32").Version(), 1},
+		{"0.36", version.MustParseVersion("0.36").Version(), 0},
+		{"0.37", version.MustParseVersion("0.37-DEV").Version(), 1},
+		{"0.37-DEV", version.MustParseVersion("0.37").Version(), -1},
+		{"0.36", version.MustParseVersion("0.37-DEV").Version(), -1},
+		{"0.37-DEV", version.MustParseVersion("0.37-DEV").Version(), 0},
 		// https://github.com/gohugoio/hugo/issues/5905
 		{nil, nil, 0},
 		{testT.NonEmptyInterfaceNil, nil, 0},
diff --git a/tpl/compare/init.go b/tpl/compare/init.go
index f70b19254..4d789af3b 100644
--- a/tpl/compare/init.go
+++ b/tpl/compare/init.go
@@ -25,7 +25,7 @@ const name = "compare"
 
 func init() {
 	f := func(d *deps.Deps) *internal.TemplateFuncsNamespace {
-		language := d.Conf.Language()
+		language := d.Conf.Language().(*langs.Language)
 		if language == nil {
 			panic("language must be set")
 		}
diff --git a/tpl/lang/init.go b/tpl/lang/init.go
index cad4eee09..0c07daa38 100644
--- a/tpl/lang/init.go
+++ b/tpl/lang/init.go
@@ -25,7 +25,7 @@ const name = "lang"
 
 func init() {
 	f := func(d *deps.Deps) *internal.TemplateFuncsNamespace {
-		ctx := New(d, langs.GetTranslator(d.Conf.Language()))
+		ctx := New(d, langs.GetTranslator(d.Conf.Language().(*langs.Language)))
 
 		ns := &internal.TemplateFuncsNamespace{
 			Name:    name,
diff --git a/tpl/page/page_integration_test.go b/tpl/page/page_integration_test.go
index f96c87f98..3f2154921 100644
--- a/tpl/page/page_integration_test.go
+++ b/tpl/page/page_integration_test.go
@@ -121,7 +121,7 @@ Bundled page: {{ $p2_1.Content }}
 
   `
 
-	for _, multilingual := range []bool{false, true} {
+	for _, multilingual := range []bool{true, false} {
 		t.Run(fmt.Sprintf("multilingual-%t", multilingual), func(t *testing.T) {
 			// Fenced code blocks.
 			files := strings.ReplaceAll(filesTemplate, "$$$", "```")
@@ -138,14 +138,16 @@ weight = 2
 				files = strings.ReplaceAll(files, "LANG_CONFIG", "")
 			}
 
-			b := hugolib.NewIntegrationTestBuilder(
-				hugolib.IntegrationTestConfig{
-					T:           t,
-					TxtarString: files,
-				},
-			).Build()
+			for range 1 {
 
-			b.AssertFileContent("public/index.html", `
+				b := hugolib.NewIntegrationTestBuilder(
+					hugolib.IntegrationTestConfig{
+						T:           t,
+						TxtarString: files,
+					},
+				).Build()
+
+				b.AssertFileContent("public/index.html", `
 Heading OK.
 Image OK.
 Link OK.
@@ -162,15 +164,16 @@ Render OK.
 Shortcode in bundled page OK.
 	`)
 
-			b.AssertFileContent("public/404.html", `404 Page OK.`)
-			b.AssertFileContent("public/robots.txt", `Robots OK.`)
-			b.AssertFileContent("public/homealias/index.html", `Alias OK.`)
-			b.AssertFileContent("public/page/1/index.html", `Alias OK.`)
-			b.AssertFileContent("public/page/2/index.html", `Page OK.`)
-			if multilingual {
-				b.AssertFileContent("public/sitemap.xml", `SitemapIndex OK: sitemapindex`)
-			} else {
-				b.AssertFileContent("public/sitemap.xml", `Sitemap OK.`)
+				b.AssertFileContent("public/404.html", `404 Page OK.`)
+				b.AssertFileContent("public/robots.txt", `Robots OK.`)
+				b.AssertFileContent("public/homealias/index.html", `Alias OK.`)
+				b.AssertFileContent("public/page/1/index.html", `Alias OK.`)
+				b.AssertFileContent("public/page/2/index.html", `Page OK.`)
+				if multilingual {
+					b.AssertFileContent("public/sitemap.xml", `SitemapIndex OK: sitemapindex`)
+				} else {
+					b.AssertFileContent("public/sitemap.xml", `Sitemap OK.`)
+				}
 			}
 		})
 	}
diff --git a/tpl/partials/partials.go b/tpl/partials/partials.go
index 63daadb42..022e498c5 100644
--- a/tpl/partials/partials.go
+++ b/tpl/partials/partials.go
@@ -155,6 +155,7 @@ func (ns *Namespace) doInclude(ctx context.Context, key string, templ *tplimpl.T
 	var w io.Writer
 
 	if info.HasReturn {
+
 		// Wrap the context sent to the template to capture the return value.
 		// Note that the template is rewritten to make sure that the dot (".")
 		// and the $ variable points to Arg.
@@ -221,7 +222,7 @@ func (ns *Namespace) IncludeCached(ctx context.Context, name string, context any
 		if ns.deps.Conf.Watching() {
 			// We need to create a shared dependency manager to pass downwards
 			// and add those same dependencies to any cached invocation of this partial.
-			depsManagerShared = identity.NewManager("partials")
+			depsManagerShared = identity.NewManager()
 			ctx = tpl.Context.DependencyManagerScopedProvider.Set(ctx, depsManagerShared.(identity.DependencyManagerScopedProvider))
 		}
 		r := ns.doInclude(ctx, keyString, ti, context)
diff --git a/tpl/time/init.go b/tpl/time/init.go
index b13d0ab89..525be2cf5 100644
--- a/tpl/time/init.go
+++ b/tpl/time/init.go
@@ -29,7 +29,8 @@ func init() {
 		if d.Conf.Language() == nil {
 			panic("Language must be set")
 		}
-		ctx := New(langs.GetTimeFormatter(d.Conf.Language()), langs.GetLocation(d.Conf.Language()), d)
+		lang := d.Conf.Language().(*langs.Language)
+		ctx := New(langs.GetTimeFormatter(lang), langs.GetLocation(lang), d)
 
 		ns := &internal.TemplateFuncsNamespace{
 			Name: name,
diff --git a/tpl/tplimpl/legacy.go b/tpl/tplimpl/legacy.go
index ee9a6e5d1..e67fee7bc 100644
--- a/tpl/tplimpl/legacy.go
+++ b/tpl/tplimpl/legacy.go
@@ -68,7 +68,6 @@ type legacyTargetPathIdentifiers struct {
 	targetPath     string
 	targetCategory Category
 	kind           string
-	lang           string
 	outputFormat   string
 	ext            string
 }
diff --git a/tpl/tplimpl/shortcodes_integration_test.go b/tpl/tplimpl/shortcodes_integration_test.go
index 244a2592b..2a165735c 100644
--- a/tpl/tplimpl/shortcodes_integration_test.go
+++ b/tpl/tplimpl/shortcodes_integration_test.go
@@ -794,3 +794,67 @@ myshortcode.en.html
 	b.AssertFileContent("public/pl/index.html", "myshortcode.html")
 	b.AssertFileContent("public/en/index.html", "myshortcode.en.html")
 }
+
+func TestHasShortcodeSamePageSourceDifferentShortcodes(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+disableKinds = ['home','rss','section','sitemap','taxonomy','term', '404']
+defaultContentLanguage = 'en'
+defaultContentLanguageInSubdir = true
+markup.goldmark.renderer.unsafe = true
+
+[languages]
+[languages.en]
+weight = 1
+[languages.sv]
+weight = 2
+-- content/inc/i1.md --
+---
+headless: true
+sites:
+  matrix:
+    languages:
+      - '**'	
+---
+{{% sc1 %}}
+-- content/inc/i2.md --
+---
+headless: true
+sites:
+  matrix:
+    languages:
+      - '**'	
+---
+{{% sc2 %}}
+-- content/p1.md --
+---
+title: p1
+---
+P1.
+{{% inc %}}
+-- content/p1.sv.md --
+---
+title: p1 sv
+---
+P1.
+{{% inc %}}
+-- layouts/_shortcodes/sc1.html --
+sc1
+-- layouts/_shortcodes/sc2.html --
+sc2
+-- layouts/_shortcodes/inc.html --
+{{ with site.GetPage "inc/i1.md" }}i1.RenderShortcodes: {{ .RenderShortcodes }}{{ end }}
+-- layouts/_shortcodes/inc.sv.html --
+{{ with site.GetPage "inc/i2.md" }}i1.RenderShortcodes: {{ .RenderShortcodes }}{{ end }}
+-- layouts/all.html --
+Content: {{ .Content }}|sc1: {{ .HasShortcode "sc1" }}|sc2: {{ .HasShortcode "sc2" }}|inc: {{ .HasShortcode "inc" }}|
+
+`
+
+	b := hugolib.Test(t, files)
+
+	b.AssertFileContent("public/en/p1/index.html", " |sc1: true|sc2: false|inc: true|")
+	b.AssertFileContent("public/sv/p1/index.html", " |sc1: false|sc2: true|inc: true|")
+}
diff --git a/tpl/tplimpl/templatedescriptor.go b/tpl/tplimpl/templatedescriptor.go
index fd86f15fa..8832b29bb 100644
--- a/tpl/tplimpl/templatedescriptor.go
+++ b/tpl/tplimpl/templatedescriptor.go
@@ -14,6 +14,8 @@
 package tplimpl
 
 import (
+	"github.com/gohugoio/hugo/common/types"
+	"github.com/gohugoio/hugo/hugolib/sitesmatrix"
 	"github.com/gohugoio/hugo/resources/kinds"
 )
 
@@ -29,7 +31,8 @@ type TemplateDescriptor struct {
 	// Group 2.
 	OutputFormat string // rss, csv ...
 	MediaType    string // text/html, text/plain, ...
-	Lang         string // en, nn, fr, ...
+
+	SitesHash uint64
 
 	Variant1 string // contextual variant, e.g. "link" in render hooks."
 	Variant2 string // contextual variant, e.g. "id" in render.
@@ -60,12 +63,12 @@ type descriptorHandler struct {
 
 // Note that this in this setup is usually a descriptor constructed from a page,
 // so we want to find the best match for that page.
-func (s descriptorHandler) compareDescriptors(category Category, isEmbedded bool, this, other TemplateDescriptor) weight {
+func (s descriptorHandler) compareDescriptors(category Category, this, other TemplateDescriptor, sitesMatrixThis, sitesMatrixOther sitesmatrix.VectorProvider) weight {
 	if this.LayoutFromUserMustMatch && this.LayoutFromUser != other.LayoutFromTemplate {
 		return weightNoMatch
 	}
 
-	w := this.doCompare(category, s.opts.DefaultContentLanguage, other)
+	w := this.doCompare(category, other, sitesMatrixThis, sitesMatrixOther)
 
 	if w.w1 <= 0 {
 		if category == CategoryMarkup && (this.Variant1 == other.Variant1) && (this.Variant2 == other.Variant2 || this.Variant2 != "" && other.Variant2 == "") {
@@ -88,7 +91,7 @@ func (s descriptorHandler) compareDescriptors(category Category, isEmbedded bool
 }
 
 //lint:ignore ST1006 this vs other makes it easier to reason about.
-func (this TemplateDescriptor) doCompare(category Category, defaultContentLanguage string, other TemplateDescriptor) weight {
+func (this TemplateDescriptor) doCompare(category Category, other TemplateDescriptor, sitesMatrixThis, sitesMatrixOther sitesmatrix.VectorProvider) weight {
 	w := weightNoMatch
 
 	if !this.AlwaysAllowPlainText {
@@ -110,8 +113,13 @@ func (this TemplateDescriptor) doCompare(category Category, defaultContentLangua
 		}
 	}
 
-	if other.Lang != "" && other.Lang != this.Lang {
-		return w
+	if sitesMatrixOther != nil {
+		// sitesMatrixThis is usually a single Site.
+		// But we also use this method to find all base template variants for a given template,
+		// and in that case we may get multiple vectors (e.g. multiple languages).
+		if sitesMatrixThis == nil || !sitesMatrixOther.HasAnyVector(sitesMatrixThis) {
+			return w
+		}
 	}
 
 	if other.OutputFormat != "" && other.OutputFormat != this.OutputFormat {
@@ -156,7 +164,7 @@ func (this TemplateDescriptor) doCompare(category Category, defaultContentLangua
 		weightLayoutAll      = 2 // the "all" layout
 		weightOutputFormat   = 4 // a configured output format (e.g. rss, html, json)
 		weightMediaType      = 1 // a configured media type (e.g. text/html, text/plain)
-		weightLang           = 1 // a configured language (e.g. en, nn, fr, ...)
+		weightSitesMatrix    = 1 // a configured language (e.g. en, nn, fr, ...)
 		weightVariant1       = 6 // currently used for render hooks, e.g. "link", "image"
 		weightVariant2       = 4 // currently used for render hooks, e.g. the language "go" in code blocks.
 
@@ -170,7 +178,7 @@ func (this TemplateDescriptor) doCompare(category Category, defaultContentLangua
 		weight2Group1 = 1 // kind, standardl layout (single,list,all)
 		weight2Group2 = 2 // custom layout (mylayout)
 
-		weight3 = 1 // for media type, lang, output format.
+		weight3 = 1 // for media type, output format, and dimensions (if not provided).
 	)
 
 	// Now we now know that the other descriptor is a subset of this.
@@ -196,9 +204,16 @@ func (this TemplateDescriptor) doCompare(category Category, defaultContentLangua
 		w.w2 = weight2Group2
 	}
 
-	if (other.Lang != "" && other.Lang == this.Lang) || (other.Lang == "" && this.Lang == defaultContentLanguage) {
-		w.w1 += weightLang
-		w.w3 += weight3
+	if sitesMatrixOther != nil {
+		// sitesMatrixThis is usually a single Site.
+		if sitesMatrixThis != nil && sitesMatrixOther.HasAnyVector(sitesMatrixThis) {
+			w.w1 += weightSitesMatrix
+			if wp, ok := sitesMatrixOther.(types.WeightProvider); ok {
+				w.wsm = wp.Weight()
+			} else {
+				w.wsm = weight3
+			}
+		}
 	}
 
 	if other.OutputFormat != "" && other.OutputFormat == this.OutputFormat {
diff --git a/tpl/tplimpl/templatedescriptor_test.go b/tpl/tplimpl/templatedescriptor_test.go
index 20ab47fba..ed8763c87 100644
--- a/tpl/tplimpl/templatedescriptor_test.go
+++ b/tpl/tplimpl/templatedescriptor_test.go
@@ -20,14 +20,14 @@ func TestTemplateDescriptorCompare(t *testing.T) {
 
 	less := func(category Category, this, other1, other2 TemplateDescriptor) {
 		c.Helper()
-		result1 := dh.compareDescriptors(category, false, this, other1)
-		result2 := dh.compareDescriptors(category, false, this, other2)
+		result1 := dh.compareDescriptors(category, this, other1, nil, nil)
+		result2 := dh.compareDescriptors(category, this, other2, nil, nil)
 		c.Assert(result1.w1 < result2.w1, qt.IsTrue, qt.Commentf("%d < %d", result1, result2))
 	}
 
 	check := func(category Category, this, other TemplateDescriptor, less bool) {
 		c.Helper()
-		result := dh.compareDescriptors(category, false, this, other)
+		result := dh.compareDescriptors(category, this, other, nil, nil)
 		if less {
 			c.Assert(result.w1 < 0, qt.IsTrue, qt.Commentf("%d", result))
 		} else {
@@ -38,15 +38,15 @@ func TestTemplateDescriptorCompare(t *testing.T) {
 	check(
 
 		CategoryBaseof,
-		TemplateDescriptor{Kind: "", LayoutFromTemplate: "", Lang: "", OutputFormat: "404", MediaType: "text/html"},
-		TemplateDescriptor{Kind: "", LayoutFromTemplate: "", Lang: "", OutputFormat: "html", MediaType: "text/html"},
+		TemplateDescriptor{Kind: "", LayoutFromTemplate: "", OutputFormat: "404", MediaType: "text/html"},
+		TemplateDescriptor{Kind: "", LayoutFromTemplate: "", OutputFormat: "html", MediaType: "text/html"},
 		false,
 	)
 
 	check(
 		CategoryLayout,
-		TemplateDescriptor{Kind: "", Lang: "en", OutputFormat: "404", MediaType: "text/html"},
-		TemplateDescriptor{Kind: "", LayoutFromTemplate: "", Lang: "", OutputFormat: "alias", MediaType: "text/html"},
+		TemplateDescriptor{Kind: "", OutputFormat: "404", MediaType: "text/html"},
+		TemplateDescriptor{Kind: "", LayoutFromTemplate: "", OutputFormat: "alias", MediaType: "text/html"},
 		true,
 	)
 
@@ -78,27 +78,27 @@ func BenchmarkCompareDescriptors(b *testing.B) {
 		d1, d2 TemplateDescriptor
 	}{
 		{
-			TemplateDescriptor{Kind: "", LayoutFromTemplate: "", OutputFormat: "404", MediaType: "text/html", Lang: "en", Variant1: "", Variant2: "", LayoutFromUserMustMatch: false, IsPlainText: false},
-			TemplateDescriptor{Kind: "", LayoutFromTemplate: "", OutputFormat: "rss", MediaType: "application/rss+xml", Lang: "", Variant1: "", Variant2: "", LayoutFromUserMustMatch: false, IsPlainText: false},
+			TemplateDescriptor{Kind: "", LayoutFromTemplate: "", OutputFormat: "404", MediaType: "text/html", Variant1: "", Variant2: "", LayoutFromUserMustMatch: false, IsPlainText: false},
+			TemplateDescriptor{Kind: "", LayoutFromTemplate: "", OutputFormat: "rss", MediaType: "application/rss+xml", Variant1: "", Variant2: "", LayoutFromUserMustMatch: false, IsPlainText: false},
 		},
 		{
-			TemplateDescriptor{Kind: "page", LayoutFromTemplate: "single", OutputFormat: "html", MediaType: "text/html", Lang: "en", Variant1: "", Variant2: "", LayoutFromUserMustMatch: false, IsPlainText: false},
-			TemplateDescriptor{Kind: "", LayoutFromTemplate: "list", OutputFormat: "", MediaType: "application/rss+xml", Lang: "", Variant1: "", Variant2: "", LayoutFromUserMustMatch: false, IsPlainText: false},
+			TemplateDescriptor{Kind: "page", LayoutFromTemplate: "single", OutputFormat: "html", MediaType: "text/html", Variant1: "", Variant2: "", LayoutFromUserMustMatch: false, IsPlainText: false},
+			TemplateDescriptor{Kind: "", LayoutFromTemplate: "list", OutputFormat: "", MediaType: "application/rss+xml", Variant1: "", Variant2: "", LayoutFromUserMustMatch: false, IsPlainText: false},
 		},
 		{
-			TemplateDescriptor{Kind: "page", LayoutFromTemplate: "single", OutputFormat: "html", MediaType: "text/html", Lang: "en", Variant1: "", Variant2: "", LayoutFromUserMustMatch: false, IsPlainText: false},
-			TemplateDescriptor{Kind: "", LayoutFromTemplate: "", OutputFormat: "alias", MediaType: "text/html", Lang: "", Variant1: "", Variant2: "", LayoutFromUserMustMatch: false, IsPlainText: false},
+			TemplateDescriptor{Kind: "page", LayoutFromTemplate: "single", OutputFormat: "html", MediaType: "text/html", Variant1: "", Variant2: "", LayoutFromUserMustMatch: false, IsPlainText: false},
+			TemplateDescriptor{Kind: "", LayoutFromTemplate: "", OutputFormat: "alias", MediaType: "text/html", Variant1: "", Variant2: "", LayoutFromUserMustMatch: false, IsPlainText: false},
 		},
 		{
-			TemplateDescriptor{Kind: "page", LayoutFromTemplate: "single", OutputFormat: "rss", MediaType: "application/rss+xml", Lang: "en", Variant1: "", Variant2: "", LayoutFromUserMustMatch: false, IsPlainText: false},
-			TemplateDescriptor{Kind: "", LayoutFromTemplate: "single", OutputFormat: "rss", MediaType: "application/rss+xml", Lang: "nn", Variant1: "", Variant2: "", LayoutFromUserMustMatch: false, IsPlainText: false},
+			TemplateDescriptor{Kind: "page", LayoutFromTemplate: "single", OutputFormat: "rss", MediaType: "application/rss+xml", Variant1: "", Variant2: "", LayoutFromUserMustMatch: false, IsPlainText: false},
+			TemplateDescriptor{Kind: "", LayoutFromTemplate: "single", OutputFormat: "rss", MediaType: "application/rss+xml", Variant1: "", Variant2: "", LayoutFromUserMustMatch: false, IsPlainText: false},
 		},
 	}
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		for _, pair := range pairs {
-			_ = dh.compareDescriptors(CategoryLayout, false, pair.d1, pair.d2)
+			_ = dh.compareDescriptors(CategoryLayout, pair.d1, pair.d2, nil, nil)
 		}
 	}
 }
diff --git a/tpl/tplimpl/templates.go b/tpl/tplimpl/templates.go
index 7aeb7e2b9..5510a67b4 100644
--- a/tpl/tplimpl/templates.go
+++ b/tpl/tplimpl/templates.go
@@ -169,6 +169,7 @@ func (t *templateNamespace) applyBaseTemplate(overlay *TemplInfo, base keyTempla
 		PathInfo: overlay.PathInfo,
 		Fi:       overlay.Fi,
 		D:        overlay.D,
+		matrix:   overlay.matrix,
 		noBaseOf: true,
 	}
 
diff --git a/tpl/tplimpl/templatestore.go b/tpl/tplimpl/templatestore.go
index 0c03b2cc1..f1211d5b0 100644
--- a/tpl/tplimpl/templatestore.go
+++ b/tpl/tplimpl/templatestore.go
@@ -36,6 +36,7 @@ import (
 	"time"
 
 	"github.com/gohugoio/hugo/common/herrors"
+	"github.com/gohugoio/hugo/common/hstrings"
 	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/common/maps"
 	"github.com/gohugoio/hugo/common/paths"
@@ -43,6 +44,7 @@ import (
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/hugofs/files"
 	"github.com/gohugoio/hugo/hugolib/doctree"
+	"github.com/gohugoio/hugo/hugolib/sitesmatrix"
 	"github.com/gohugoio/hugo/identity"
 	gc "github.com/gohugoio/hugo/markup/goldmark/goldmark_config"
 	"github.com/gohugoio/hugo/media"
@@ -179,7 +181,7 @@ type StoreOptions struct {
 	// The logger to use.
 	Log loggers.Logger
 
-	// The path parser to use.
+	// The path handler to use.
 	PathParser *paths.PathParser
 
 	// Set when --enableTemplateMetrics is set.
@@ -191,9 +193,6 @@ type StoreOptions struct {
 	// All configured media types.
 	MediaTypes media.Types
 
-	// The default content language.
-	DefaultContentLanguage string
-
 	// The default output format.
 	DefaultOutputFormat string
 
@@ -251,6 +250,8 @@ type TemplInfo struct {
 	// The descriptior that this template represents.
 	D TemplateDescriptor
 
+	matrix sitesmatrix.VectorProvider // language, version, role.
+
 	// Parser state.
 	ParseInfo ParseInfo
 
@@ -323,7 +324,7 @@ func (ti *TemplInfo) String() string {
 	return ti.PathInfo.String()
 }
 
-func (ti *TemplInfo) findBestMatchBaseof(s *TemplateStore, d1 TemplateDescriptor, k1 string, slashCountK1 int, best *bestMatch) {
+func (ti *TemplInfo) findBestMatchBaseof(s *TemplateStore, d1 TemplateDescriptor, dims1 sitesmatrix.VectorProvider, k1 string, slashCountK1 int, best *bestMatch) {
 	if ti.baseVariants == nil {
 		return
 	}
@@ -336,7 +337,9 @@ func (ti *TemplInfo) findBestMatchBaseof(s *TemplateStore, d1 TemplateDescriptor
 		distance := slashCountK1 - slashCountK2
 
 		for d2, vv := range v {
-			weight := s.dh.compareDescriptors(CategoryBaseof, false, d1, d2)
+
+			weight := s.dh.compareDescriptors(CategoryBaseof, d1, d2, dims1, vv.Base.matrix)
+
 			weight.distance = distance
 			if best.isBetter(weight, vv.Template) {
 				best.updateValues(weight, k2, d2, vv.Template)
@@ -390,6 +393,9 @@ type TemplateQuery struct {
 	// The category to look in.
 	Category Category
 
+	// The sites variants to consider.
+	Sites sitesmatrix.VectorProvider
+
 	// The template descriptor to match against.
 	Desc TemplateDescriptor
 
@@ -459,22 +465,22 @@ func (s *TemplateStore) NewFromOpts() (*TemplateStore, error) {
 }
 
 // In the previous implementation of base templates in Hugo, we parsed and applied these base templates on
-// request, e.g. in the middle of rendering. The idea was that we coulnd't know upfront which layoyt/base template
+// request, e.g. in the middle of rendering. The idea was that we couldn't know upfront which layoyt/base template
 // combination that would be used.
 // This, however, added a lot of complexity involving a careful dance of template cloning and parsing
 // (Go HTML tenplates cannot be parsed after any of the templates in the tree have been executed).
 // FindAllBaseTemplateCandidates finds all base template candidates for the given descriptor so we can apply them upfront.
 // In this setup we may end up with unused base templates, but not having to do the cloning should more than make up for that.
-func (s *TemplateStore) FindAllBaseTemplateCandidates(overlayKey string, desc TemplateDescriptor) []keyTemplateInfo {
+func (s *TemplateStore) FindAllBaseTemplateCandidates(overlayKey string, d1 TemplateDescriptor, dims1 sitesmatrix.VectorProvider) []keyTemplateInfo {
 	var result []keyTemplateInfo
-	descBaseof := desc
+
 	s.treeMain.Walk(func(k string, v map[nodeKey]*TemplInfo) (bool, error) {
 		for _, vv := range v {
 			if vv.category != CategoryBaseof {
 				continue
 			}
 
-			if vv.D.isKindInLayout(desc.LayoutFromTemplate) && s.dh.compareDescriptors(CategoryBaseof, false, descBaseof, vv.D).w1 > 0 {
+			if vv.D.isKindInLayout(d1.LayoutFromTemplate) && s.dh.compareDescriptors(CategoryBaseof, d1, vv.D, dims1, vv.matrix).w1 > 0 {
 				result = append(result, keyTemplateInfo{Key: k, Info: vv})
 			}
 		}
@@ -581,17 +587,18 @@ func (s *TemplateStore) LookupPagesLayout(q TemplateQuery) *TemplInfo {
 		return m
 	}
 	best1.reset()
-	m.findBestMatchBaseof(s, q.Desc, key, slashCountKey, best1)
+	m.findBestMatchBaseof(s, q.Desc, q.Sites, key, slashCountKey, best1)
 	if best1.w.w1 <= 0 {
 		return nil
 	}
+
 	return best1.templ
 }
 
 func (s *TemplateStore) LookupPartial(pth string) *TemplInfo {
 	ti, _ := s.cacheLookupPartials.GetOrCreate(pth, func() (*TemplInfo, error) {
 		pi := s.opts.PathParser.Parse(files.ComponentFolderLayouts, pth).ForType(paths.TypePartial)
-		k1, _, _, desc, err := s.toKeyCategoryAndDescriptor(pi)
+		k1, _, _, desc, matrix, err := s.toKeyCategoryAndDescriptor(pi, nil)
 		if err != nil {
 			return nil, err
 		}
@@ -604,7 +611,7 @@ func (s *TemplateStore) LookupPartial(pth string) *TemplInfo {
 
 		best := s.getBest()
 		defer s.putBest(best)
-		s.findBestMatchGet(s.key(path.Join(containerPartials, k1)), CategoryPartial, nil, desc, best)
+		s.findBestMatchGet(s.key(path.Join(containerPartials, k1)), CategoryPartial, nil, desc, matrix, best)
 		return best.templ, nil
 	})
 
@@ -647,7 +654,7 @@ func (s *TemplateStore) LookupShortcode(q TemplateQuery) (*TemplInfo, error) {
 				continue
 			}
 
-			weight := s.dh.compareDescriptors(q.Category, vv.subCategory == SubCategoryEmbedded, q.Desc, k)
+			weight := s.dh.compareDescriptors(q.Category, q.Desc, k, q.Sites, vv.matrix)
 			weight.distance = distance
 			isBetter := best.isBetter(weight, vv)
 			if isBetter {
@@ -687,7 +694,7 @@ func (s *TemplateStore) PrintDebug(prefix string, category Category, w io.Writer
 			return
 		}
 		s := strings.ReplaceAll(strings.TrimSpace(vv.content), "\n", " ")
-		ts := fmt.Sprintf("kind: %q layout: %q lang: %q content: %.30s", vv.D.Kind, vv.D.LayoutFromTemplate, vv.D.Lang, s)
+		ts := fmt.Sprintf("kind: %q layout: %q content: %.30s", vv.D.Kind, vv.D.LayoutFromTemplate, s)
 		fmt.Fprintf(w, "%s%s %s\n", strings.Repeat(" ", level), key, ts)
 	}
 	s.treeMain.WalkPrefix(prefix, func(key string, v map[nodeKey]*TemplInfo) (bool, error) {
@@ -794,7 +801,9 @@ func (s TemplateStore) WithSiteOpts(opts SiteOptions) *TemplateStore {
 	return &s
 }
 
-func (s *TemplateStore) findBestMatchGet(key string, category Category, consider func(candidate *TemplInfo) bool, desc TemplateDescriptor, best *bestMatch) {
+func (s *TemplateStore) findBestMatchGet(key string, category Category,
+	consider func(candidate *TemplInfo) bool, d1 TemplateDescriptor, dims1 sitesmatrix.VectorProvider, best *bestMatch,
+) {
 	key = strings.ToLower(key)
 
 	v := s.treeMain.Get(key)
@@ -811,7 +820,8 @@ func (s *TemplateStore) findBestMatchGet(key string, category Category, consider
 			continue
 		}
 
-		weight := s.dh.compareDescriptors(category, vv.subCategory == SubCategoryEmbedded, desc, k.d)
+		weight := s.dh.compareDescriptors(category, d1, k.d, dims1, vv.matrix)
+
 		if best.isBetter(weight, vv) {
 			best.updateValues(weight, key, k.d, vv)
 		}
@@ -842,7 +852,7 @@ func (s *TemplateStore) findBestMatchWalkPath(q TemplateQuery, k1 string, slashC
 				continue
 			}
 
-			weight := s.dh.compareDescriptors(q.Category, vv.subCategory == SubCategoryEmbedded, q.Desc, k.d)
+			weight := s.dh.compareDescriptors(q.Category, q.Desc, k.d, q.Sites, vv.matrix)
 
 			weight.distance = distance
 			isBetter := best.isBetter(weight, vv)
@@ -1107,10 +1117,11 @@ func (s *TemplateStore) setTemplateByPath(p string, ti *TemplInfo) {
 }
 
 func (s *TemplateStore) insertShortcode(pi *paths.Path, fi hugofs.FileMetaInfo, replace bool, tree doctree.Tree[map[string]map[TemplateDescriptor]*TemplInfo]) (*TemplInfo, error) {
-	k1, k2, _, d, err := s.toKeyCategoryAndDescriptor(pi)
+	k1, k2, _, d, matrix, err := s.toKeyCategoryAndDescriptor(pi, fi)
 	if err != nil {
 		return nil, err
 	}
+
 	m := tree.Get(k1)
 	if m == nil {
 		m = make(map[string]map[TemplateDescriptor]*TemplInfo)
@@ -1133,6 +1144,7 @@ func (s *TemplateStore) insertShortcode(pi *paths.Path, fi hugofs.FileMetaInfo,
 		PathInfo: pi,
 		Fi:       fi,
 		D:        d,
+		matrix:   matrix,
 		category: CategoryShortcode,
 		noBaseOf: true,
 	}
@@ -1152,7 +1164,7 @@ func (s *TemplateStore) insertShortcode(pi *paths.Path, fi hugofs.FileMetaInfo,
 }
 
 func (s *TemplateStore) insertTemplate(pi *paths.Path, fi hugofs.FileMetaInfo, subCategory SubCategory, replace bool, tree doctree.Tree[map[nodeKey]*TemplInfo]) (*TemplInfo, error) {
-	key, _, category, d, err := s.toKeyCategoryAndDescriptor(pi)
+	key, _, category, d, matrix, err := s.toKeyCategoryAndDescriptor(pi, fi)
 	// See #13577. Warn for now.
 	if err != nil {
 		var loc string
@@ -1165,7 +1177,7 @@ func (s *TemplateStore) insertTemplate(pi *paths.Path, fi hugofs.FileMetaInfo, s
 		return nil, nil
 	}
 
-	return s.insertTemplate2(pi, fi, key, category, subCategory, d, replace, false, tree)
+	return s.insertTemplate2(pi, fi, key, category, subCategory, d, matrix, replace, false, tree)
 }
 
 func (s *TemplateStore) insertTemplate2(
@@ -1175,6 +1187,7 @@ func (s *TemplateStore) insertTemplate2(
 	category Category,
 	subCategory SubCategory,
 	d TemplateDescriptor,
+	matrix sitesmatrix.VectorStore,
 	replace, isLegacyMapped bool,
 	tree doctree.Tree[map[nodeKey]*TemplInfo],
 ) (*TemplInfo, error) {
@@ -1196,7 +1209,27 @@ func (s *TemplateStore) insertTemplate2(
 		tree.Insert(key, m)
 	}
 
-	nkExisting, existingFound := m[nk]
+	var (
+		nkExisting    *TemplInfo
+		existingFound bool
+	)
+
+	if d.SitesHash == 0 {
+		// inline partials.
+		for k, v := range m {
+			d2 := v.D
+			d2.SitesHash = 0
+			if d == d2 {
+				nkExisting = v
+				nk = k
+				existingFound = true
+				break
+			}
+		}
+	} else {
+		nkExisting, existingFound = m[nk]
+	}
+
 	if !replace && existingFound && fi != nil && nkExisting.Fi != nil {
 		// See issue #13715.
 		// We do the merge on the file system level, but from Hugo v0.146.0 we have a situation where
@@ -1223,6 +1256,7 @@ func (s *TemplateStore) insertTemplate2(
 		PathInfo:       pi,
 		Fi:             fi,
 		D:              d,
+		matrix:         matrix,
 		category:       category,
 		noBaseOf:       category > CategoryLayout,
 		isLegacyMapped: isLegacyMapped,
@@ -1256,7 +1290,7 @@ func (s *TemplateStore) insertTemplates(include func(fi hugofs.FileMetaInfo) boo
 
 	legacyOrdinalMappings := map[legacyTargetPathIdentifiers]legacyOrdinalMappingFi{}
 
-	walker := func(pth string, fi hugofs.FileMetaInfo) error {
+	walker := func(ctx context.Context, pth string, fi hugofs.FileMetaInfo) error {
 		if fi.IsDir() {
 			return nil
 		}
@@ -1318,7 +1352,6 @@ func (s *TemplateStore) insertTemplates(include func(fi hugofs.FileMetaInfo) boo
 					targetPath:     m1.mapping.targetPath,
 					targetCategory: m1.mapping.targetCategory,
 					kind:           m1.mapping.targetDesc.Kind,
-					lang:           pi.Lang(),
 					ext:            pi.Ext(),
 					outputFormat:   pi.OutputFormat(),
 				}
@@ -1374,7 +1407,7 @@ func (s *TemplateStore) insertTemplates(include func(fi hugofs.FileMetaInfo) boo
 				identifiers = append(identifiers, pi.Section())
 			}
 
-			identifiers = helpers.UniqueStrings(identifiers)
+			identifiers = hstrings.UniqueStrings(identifiers)
 
 			// Tokens on e.g. form /SECTIONKIND/THESECTION
 			insertSectionTokens := func(section string) []string {
@@ -1396,7 +1429,7 @@ func (s *TemplateStore) insertTemplates(include func(fi hugofs.FileMetaInfo) boo
 					ss = append(ss, s1)
 				}
 
-				helpers.UniqueStringsReuse(ss)
+				hstrings.UniqueStringsReuse(ss)
 
 				return ss
 			}
@@ -1482,12 +1515,12 @@ func (s *TemplateStore) insertTemplates(include func(fi hugofs.FileMetaInfo) boo
 		category := m.targetCategory
 		desc := m.targetDesc
 		desc.Kind = k.kind
-		desc.Lang = k.lang
 		desc.OutputFormat = outputFormat.Name
 		desc.IsPlainText = outputFormat.IsPlainText
 		desc.MediaType = mediaType.Type
+		desc.SitesHash = fi.Meta().SitesMatrix.MustHash()
 
-		ti, err := s.insertTemplate2(pi, fi, targetPath, category, SubCategoryMain, desc, true, true, s.treeMain)
+		ti, err := s.insertTemplate2(pi, fi, targetPath, category, SubCategoryMain, desc, fi.Meta().SitesMatrix, true, true, s.treeMain)
 		if err != nil {
 			return err
 		}
@@ -1556,7 +1589,7 @@ func (s *TemplateStore) parseTemplates(replace bool) error {
 				if !vv.noBaseOf {
 					d := vv.D
 					// Find all compatible base templates.
-					baseTemplates := s.FindAllBaseTemplateCandidates(key, d)
+					baseTemplates := s.FindAllBaseTemplateCandidates(key, d, vv.matrix)
 					if len(baseTemplates) == 0 {
 						// The regular expression used to detect if a template needs a base template has some
 						// rare false positives. Assume we don't need one.
@@ -1686,15 +1719,22 @@ func (s *TemplateStore) templates() iter.Seq[*TemplInfo] {
 	}
 }
 
-func (s *TemplateStore) toKeyCategoryAndDescriptor(p *paths.Path) (string, string, Category, TemplateDescriptor, error) {
+func (s *TemplateStore) toKeyCategoryAndDescriptor(p *paths.Path, fi hugofs.FileMetaInfo) (string, string, Category, TemplateDescriptor, sitesmatrix.VectorStore, error) {
 	k1 := p.Dir()
 	k2 := ""
 
 	outputFormat, mediaType := s.resolveOutputFormatAndOrMediaType(p.OutputFormat(), p.Ext())
 	nameNoIdentifier := p.NameNoIdentifier()
 
+	var vactorStore sitesmatrix.VectorStore
+	if fi != nil {
+		vactorStore = fi.Meta().SitesMatrix
+	} else {
+		vactorStore = s.opts.PathParser.SitesMatrixFromPath(p)
+	}
+
 	d := TemplateDescriptor{
-		Lang:               p.Lang(),
+		SitesHash:          vactorStore.MustHash(),
 		OutputFormat:       p.OutputFormat(),
 		MediaType:          mediaType.Type,
 		Kind:               p.Kind(),
@@ -1769,7 +1809,7 @@ func (s *TemplateStore) toKeyCategoryAndDescriptor(p *paths.Path) (string, strin
 		k1 = strings.TrimSuffix(k1, "/_markup")
 		v, found := strings.CutPrefix(d.LayoutFromTemplate, "render-")
 		if !found {
-			return "", "", 0, TemplateDescriptor{}, fmt.Errorf("unrecognized render hook template")
+			return "", "", 0, TemplateDescriptor{}, nil, fmt.Errorf("unrecognized render hook template")
 		}
 		hyphenIdx := strings.Index(v, "-")
 
@@ -1782,7 +1822,7 @@ func (s *TemplateStore) toKeyCategoryAndDescriptor(p *paths.Path) (string, strin
 		d.LayoutFromTemplate = "" // This allows using page layout as part of the key for lookups.
 	}
 
-	return k1, k2, category, d, nil
+	return k1, k2, category, d, vactorStore, nil
 }
 
 func (s *TemplateStore) transformTemplates() error {
@@ -1956,7 +1996,7 @@ func (best *bestMatch) isBetter(w weight, ti *TemplInfo) bool {
 	}
 
 	// Note that for render hook templates, we need to make
-	// the embedded render hook template wih if they're a better match,
+	// the embedded render hook template win if they're a better match,
 	// e.g. render-codeblock-goat.html.
 	if best.templ.category != CategoryMarkup && best.w.w1 > 0 {
 		currentBestIsEmbedded := best.templ.subCategory == SubCategoryEmbedded
@@ -1973,9 +2013,13 @@ func (best *bestMatch) isBetter(w weight, ti *TemplInfo) bool {
 	}
 
 	if w.distance < best.w.distance {
+		if w.wsm < best.w.wsm {
+			return false
+		}
 		if w.w2 < best.w.w2 {
 			return false
 		}
+
 		if w.w3 < best.w.w3 {
 			return false
 		}
@@ -1986,7 +2030,10 @@ func (best *bestMatch) isBetter(w weight, ti *TemplInfo) bool {
 	}
 
 	if w.isEqualWeights(best.w) {
-		// Tie breakers.
+		if ti.subCategory != SubCategoryEmbedded && best.templ.subCategory == SubCategoryEmbedded {
+			return true
+		}
+
 		if w.distance < best.w.distance {
 			return true
 		}
@@ -2036,6 +2083,7 @@ type weight struct {
 	w1       int
 	w2       int
 	w3       int
+	wsm      int
 	distance int
 }
 
diff --git a/tpl/tplimpl/templatestore_integration_test.go b/tpl/tplimpl/templatestore_integration_test.go
index 09b65d6e1..90fc9614b 100644
--- a/tpl/tplimpl/templatestore_integration_test.go
+++ b/tpl/tplimpl/templatestore_integration_test.go
@@ -543,6 +543,178 @@ func TestCreateManyTemplateStores(t *testing.T) {
 	}
 }
 
+func TestLayoutWithLanguagesVersionsAndRoles(t *testing.T) {
+	t.Parallel()
+	files := `
+-- hugo.toml --
+disableKinds = ["taxonomy", "term", "sitemap", "rss"]
+defaultContentLanguage = "en"
+defaultContentLanguageInSubdir = true
+defaultContentVersion = "v2.0.0"
+defaultContentVersionInSubdir = true
+defaultContentRole = "admin"
+defaultContentRoleInSubdir = true
+
+[languages]
+[languages.en]
+weight = 1
+[languages.nn]
+weight = 2
+
+[versions."v2.0.0"]
+[versions."v1.0.0"]
+
+[roles.admin]
+weight = 1
+[roles.user]
+weight = 2
+
+
+[[module.mounts]]
+source = 'content/en'
+target = 'content'
+[module.mounts.sites.matrix]
+languages = ['en']
+versions = ['**']
+[[module.mounts]]
+source = 'content/nn'
+target = 'content'
+[module.mounts.sites.matrix]
+languages = ['nn']
+versions = ['**']
+[[module.mounts]]
+source = 'layouts/v1'
+target = 'layouts'
+[module.mounts.sites.matrix]
+weight = 10
+versions = ['v1**']
+languages = ['**']
+[[module.mounts]]
+source = 'layouts/en'
+target = 'layouts'
+[module.mounts.sites.matrix]
+languages = ['en']
+versions = ['**']
+[[module.mounts]]
+source = 'layouts/nn'
+target = 'layouts'
+[module.mounts.sites.matrix]
+languages = ['nn']
+versions = ['**']
+-- layouts/v1/all.html --
+layouts/v1/all.html
+-- layouts/en/all.html --
+layouts/en/all.html
+-- layouts/nn/all.html --
+layouts/nn/all.html
+
+
+`
+
+	b := hugolib.Test(t, files)
+	// b.AssertPublishDir("asdf")
+	b.AssertFileContent("public/admin/v2.0.0/en/index.html", "layouts/en/all.html")
+	b.AssertFileContent("public/admin/v2.0.0/nn/index.html", "layouts/nn/all.html")
+	b.AssertFileContent("public/admin/v1.0.0/nn/index.html", "layouts/v1/all.html")
+	b.AssertFileContent("public/admin/v1.0.0/en/index.html", "layouts/v1/all.html")
+}
+
+func TestLayoutWithLanguagesLegacy(t *testing.T) {
+	t.Parallel()
+	files := `
+-- hugo.toml --
+disableKinds = ["taxonomy", "term", "sitemap", "rss"]
+defaultContentLanguage = "en"
+defaultContentLanguageInSubdir = true
+[languages]
+[languages.en]
+weight = 1
+[languages.nn]
+weight = 2
+[languages.sv]
+weight = 3
+-- layouts/_default/single.en.html --
+layouts/_default/single.html
+-- layouts/_default/single.nn.html --
+layouts/_default/single.nn.html
+-- layouts/_default/single.sv.html --
+layouts/_default/single.sv.html
+-- content/p1.md --
+---
+title: "P1"
+---
+-- content/p1.nn.md --
+---
+title: "P1 NN"
+---
+-- content/p1.sv.md --
+---
+title: "P1 SV"
+---
+`
+	b := hugolib.Test(t, files)
+
+	b.AssertFileContent("public/en/p1/index.html", "layouts/_default/single.html")
+	b.AssertFileContent("public/nn/p1/index.html", "layouts/_default/single.nn.html")
+	b.AssertFileContent("public/sv/p1/index.html", "layouts/_default/single.sv.html")
+}
+
+func TestLayoutWithLanguagesLegacyMounts(t *testing.T) {
+	t.Parallel()
+	files := `
+-- hugo.toml --
+disableKinds = ["taxonomy", "term", "sitemap", "rss"]
+defaultContentLanguage = "en"
+defaultContentLanguageInSubdir = true
+[languages]
+[languages.en]
+weight = 1
+[languages.nn]
+weight = 2
+[languages.sv]
+weight = 3
+
+
+[[module.mounts]]
+source = 'layouts/en'
+target = 'layouts'
+lang = 'en'
+
+[[module.mounts]]
+source = 'layouts/nn'
+target = 'layouts'
+lang = 'nn'
+[[module.mounts]]
+source = 'layouts/sv'
+target = 'layouts'
+lang = 'sv'
+
+-- layouts/en/_default/single.html --
+layouts/en/_default/single.html
+-- layouts/nn/_default/single.html --
+layouts/nn/_default/single.html
+-- layouts/sv/_default/single.html --
+layouts/sv/_default/single.html
+-- content/p1.md --
+---
+title: "P1"
+---
+-- content/p1.nn.md --
+---
+title: "P1 NN"
+---
+-- content/p1.sv.md --
+---
+title: "P1 SV"
+---
+`
+	b := hugolib.Test(t, files)
+
+	b.AssertFileContent("public/en/p1/index.html", "layouts/en/_default/single.html")
+	b.AssertFileContent("public/nn/p1/index.html", "layouts/nn/_default/single.html")
+	b.AssertFileContent("public/sv/p1/index.html", "layouts/sv/_default/single.html")
+}
+
 func BenchmarkLookupPagesLayout(b *testing.B) {
 	files := `
 -- hugo.toml --
@@ -702,42 +874,47 @@ layout: mylayout
 
 `
 
-	b := hugolib.Test(t, files, hugolib.TestOptWarn())
+	for range 2 {
 
-	b.AssertLogContains("! WARN")
+		b := hugolib.Test(t, files, hugolib.TestOptWarn())
 
-	// Single pages.
-	// output format: html.
-	b.AssertFileContent("public/en/p/index.html", "layouts/single.html",
-		"layouts/_markup/render-codeblock.html|",
-		"layouts/_markup/render-codeblock-go.html|go|",
-		"layouts/shortcodes/myshortcode.html",
-	)
-	b.AssertFileContent("public/en/foo/p/index.html", "layouts/single.html")
-	b.AssertFileContent("public/en/foo/bar/p/index.html", "layouts/foo/bar/page.html")
-	b.AssertFileContent("public/en/foo/bar/withmylayout/index.html", "layouts/mylayout.html")
-	b.AssertFileContent("public/en/foo/bar/baz/p/index.html", "layouts/foo/bar/baz/single.html", "layouts/foo/bar/_shortcodes/myshortcode.html")
-	b.AssertFileContent("public/en/qux/quux/withmylayout/index.html", "layouts/qux/mylayout.html")
-	// output format: amp.
-	b.AssertFileContent("public/en/amp/p/index.html", "layouts/single.html")
-	b.AssertFileContent("public/en/amp/foo/p/index.html", "layouts/foo/single.amp.html")
-	// output format: rss.
-	b.AssertFileContent("public/en/p/index.xml", "layouts/single.rss.xml")
-	b.AssertFileContent("public/en/foo/p/index.xml", "layouts/foo/single.rss.xml")
-	b.AssertFileContent("public/nn/foo/p/index.xml", "layouts/single.nn.rss.xml")
-
-	// Note: There is qux/single.xml that's closer, but the one in the root is used becaulse of the output format match.
-	b.AssertFileContent("public/en/qux/p/index.xml", "layouts/single.rss.xml")
-
-	// Note.
-	b.AssertFileContent("public/nn/qux/quux/withmylayout/index.html", "layouts/mylayout.nn.html")
-
-	// Section pages.
-	// output format: html.
-	b.AssertFileContent("public/en/foo/index.html", "layouts/list.html")
-	b.AssertFileContent("public/en/qux/index.html", "layouts/qux/mylayout.section.html")
-	// output format: rss.
-	b.AssertFileContent("public/en/foo/index.xml", "layouts/list.xml")
+		b.AssertLogContains("! WARN")
+
+		// Single pages.
+		// output format: html.
+		b.AssertFileContent("public/en/p/index.html", "layouts/single.html",
+			"layouts/_markup/render-codeblock.html|",
+			"layouts/_markup/render-codeblock-go.html|go|",
+			"layouts/shortcodes/myshortcode.html",
+		)
+		b.AssertFileContent("public/en/foo/p/index.html", "layouts/single.html")
+		b.AssertFileContent("public/en/foo/bar/p/index.html", "layouts/foo/bar/page.html")
+		b.AssertFileContent("public/en/foo/bar/withmylayout/index.html", "layouts/mylayout.html")
+		b.AssertFileContent("public/en/foo/bar/baz/p/index.html", "layouts/foo/bar/baz/single.html", "layouts/foo/bar/_shortcodes/myshortcode.html")
+		b.AssertFileContent("public/en/qux/quux/withmylayout/index.html", "layouts/qux/mylayout.html")
+		// output format: amp.
+		b.AssertFileContent("public/en/amp/p/index.html", "layouts/single.html")
+		b.AssertFileContent("public/en/amp/foo/p/index.html", "layouts/foo/single.amp.html")
+		// output format: rss.
+		b.AssertFileContent("public/en/p/index.xml", "layouts/single.rss.xml")
+		b.AssertFileContent("public/en/foo/p/index.xml", "layouts/foo/single.rss.xml")
+		// Note: THere is layouts/foo/single.rss.xml, but we use the root because of a better language match.
+		b.AssertFileContent("public/nn/foo/p/index.xml", "layouts/single.nn.rss.xml")
+
+		// Note: There is qux/single.xml that's closer, but the one in the root is used because of the output format match.
+		b.AssertFileContent("public/en/qux/p/index.xml", "layouts/single.rss.xml")
+
+		// Note.
+		b.AssertFileContent("public/nn/qux/quux/withmylayout/index.html", "layouts/mylayout.nn.html")
+
+		// Section pages.
+		// output format: html.
+		b.AssertFileContent("public/en/foo/index.html", "layouts/list.html")
+		b.AssertFileContent("public/en/qux/index.html", "layouts/qux/mylayout.section.html")
+		// output format: rss.
+		b.AssertFileContent("public/en/foo/index.xml", "layouts/list.xml")
+
+	}
 }
 
 func TestLookupOrderIssue13636(t *testing.T) {
@@ -793,7 +970,7 @@ L3
 	}
 
 	for i, test := range tests {
-		if i != 8 {
+		if i != 10 {
 			// continue
 		}
 		files := strings.ReplaceAll(filesTemplate, "L1", test.L1)
@@ -801,7 +978,7 @@ L3
 		files = strings.ReplaceAll(files, "L3", test.L3)
 		t.Logf("Test %d: %s %s %s %s", i, test.Lang, test.L1, test.L2, test.L3)
 
-		for range 3 {
+		for range 1 {
 			b := hugolib.Test(t, files)
 			b.Assert(len(b.H.Sites), qt.Equals, 2)