From 26f31ff6ce6c69f663b4ea1e62cae95cd6ab7b6d Mon Sep 17 00:00:00 2001 From: =?utf8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Wed, 12 Nov 2025 12:55:27 +0100 Subject: [PATCH] hugolib: Improve performance of content trees with many sections MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Hugo's build process is roughly divided into three steps: 1. Process content (walk file system and insert source nodes into content tree) 2. Assemble content (assemble pages and resources according to sites matrix) 3. Render content In #13679 we consolidated the page creation logic into one place (the assemble step). This made it much simpler to reason about, but it lost us some performance esp. in big content trees. This commit re-introduces parallelization in the first step in the assemble step by handling each top level section in its own goroutine. This gives significant performance improvements for content trees with many sections. Compared to master: ``` AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=100-10 19.26m ± ∞ ¹ 14.54m ± ∞ ¹ -24.52% (p=0.029 n=4) AssembleDeepSiteWithManySections/depth=2/sectionsPerLevel=2/pagesPerSection=100-10 19.74m ± ∞ ¹ 16.45m ± ∞ ¹ -16.71% (p=0.029 n=4) AssembleDeepSiteWithManySections/depth=2/sectionsPerLevel=6/pagesPerSection=100-10 106.18m ± ∞ ¹ 71.23m ± ∞ ¹ -32.91% (p=0.029 n=4) AssembleDeepSiteWithManySections/depth=3/sectionsPerLevel=2/pagesPerSection=100-10 38.85m ± ∞ ¹ 30.47m ± ∞ ¹ -21.59% (p=0.029 n=4) ``` --- common/collections/stack.go | 13 +- common/hiter/iter.go | 26 +++ hugolib/content_map_page_assembler.go | 208 +++++++++++------- .../content_map_page_contentnodeshifter.go | 2 +- hugolib/doctree/nodeshiftree_test.go | 2 +- hugolib/doctree/nodeshifttree.go | 31 ++- hugolib/doctree/support.go | 51 ++++- hugolib/hugo_sites_build.go | 2 +- hugolib/hugo_sites_build_test.go | 1 + .../sitematrix_integration_test.go | 24 ++ 10 files changed, 257 insertions(+), 103 deletions(-) diff --git a/common/collections/stack.go b/common/collections/stack.go index ff0db2f02..0713d196c 100644 --- a/common/collections/stack.go +++ b/common/collections/stack.go @@ -13,9 +13,13 @@ package collections -import "slices" +import ( + "iter" + "slices" + "sync" -import "sync" + "github.com/gohugoio/hugo/common/hiter" +) // Stack is a simple LIFO stack that is safe for concurrent use. type Stack[T any] struct { @@ -60,6 +64,11 @@ func (s *Stack[T]) Len() int { return len(s.items) } +// All returns all items in the stack, from bottom to top. +func (s *Stack[T]) All() iter.Seq2[int, T] { + return hiter.Lock2(slices.All(s.items), s.mu.RLock, s.mu.RUnlock) +} + func (s *Stack[T]) Drain() []T { s.mu.Lock() defer s.mu.Unlock() diff --git a/common/hiter/iter.go b/common/hiter/iter.go index dd221e18c..dd2a41835 100644 --- a/common/hiter/iter.go +++ b/common/hiter/iter.go @@ -38,3 +38,29 @@ func Concat2[K, V any](seqs ...iter.Seq2[K, V]) iter.Seq2[K, V] { } } } + +// Lock returns an iterator that locks before iterating and unlocks after. +func Lock[V any](seq iter.Seq[V], lock, unlock func()) iter.Seq[V] { + return func(yield func(V) bool) { + lock() + defer unlock() + for e := range seq { + if !yield(e) { + return + } + } + } +} + +// Lock2 returns an iterator that locks before iterating and unlocks after. +func Lock2[K, V any](seq iter.Seq2[K, V], lock, unlock func()) iter.Seq2[K, V] { + return func(yield func(K, V) bool) { + lock() + defer unlock() + for k, v := range seq { + if !yield(k, v) { + return + } + } + } +} diff --git a/hugolib/content_map_page_assembler.go b/hugolib/content_map_page_assembler.go index a659f1b24..47e03d782 100644 --- a/hugolib/content_map_page_assembler.go +++ b/hugolib/content_map_page_assembler.go @@ -14,14 +14,18 @@ package hugolib import ( + "cmp" "context" "fmt" "path" "strings" "time" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/para" "github.com/gohugoio/hugo/common/paths" "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/hugofs/files" "github.com/gohugoio/hugo/hugolib/doctree" "github.com/gohugoio/hugo/hugolib/sitesmatrix" @@ -41,14 +45,18 @@ type allPagesAssembler struct { ctx context.Context h *HugoSites m *pageMap + r para.Runner - assembleChanges *WhatChanged - seenTerms map[term]sitesmatrix.Vectors - droppedPages map[*Site][]string // e.g. drafts, expired, future. + assembleChanges *WhatChanged + seenTerms map[term]sitesmatrix.Vectors + droppedPages map[*Site][]string // e.g. drafts, expired, future. + seenRootSections *maps.Cache[string, bool] + seenHome bool + assembleSectionsInParallel bool // Internal state. - pw *doctree.NodeShiftTreeWalker[contentNode] // walks pages. - rw *doctree.NodeShiftTreeWalker[contentNode] // walks resources. + pwRoot *doctree.NodeShiftTreeWalker[contentNode] // walks pages. + rwRoot *doctree.NodeShiftTreeWalker[contentNode] // walks resources. } func newAllPagesAssembler( @@ -66,16 +74,20 @@ func newAllPagesAssembler( pw := rw.Extend() pw.Tree = m.treePages + seenRootSections := maps.NewCache[string, bool]() + seenRootSections.Set("", true) // home. + return &allPagesAssembler{ - ctx: ctx, - h: h, - m: m, - assembleChanges: assembleChanges, - seenTerms: map[term]sitesmatrix.Vectors{}, - droppedPages: map[*Site][]string{}, - - pw: pw, - rw: rw, + ctx: ctx, + h: h, + m: m, + assembleChanges: assembleChanges, + seenTerms: map[term]sitesmatrix.Vectors{}, + droppedPages: map[*Site][]string{}, + seenRootSections: seenRootSections, + assembleSectionsInParallel: true, + pwRoot: pw, + rwRoot: rw, } } @@ -86,23 +98,7 @@ type sitePagesAssembler struct { ctx context.Context } -// No locking. func (a *allPagesAssembler) createAllPages() error { - var ( - sites = a.h.sitesVersionsRolesMap - h = a.h - treePages = a.m.treePages - - getViews = func(vec sitesmatrix.Vector) []viewName { - return h.languageSiteForSiteVector(vec).pageMap.cfg.taxonomyConfig.views - } - ) - - resourceOwnerInfo := struct { - n contentNode - s string - }{} - defer func() { for site, dropped := range a.droppedPages { for _, s := range dropped { @@ -119,15 +115,15 @@ func (a *allPagesAssembler) createAllPages() error { for t := range a.h.previousSeenTerms { if _, found := a.seenTerms[t]; !found { // This term has been removed. - a.pw.Tree.Delete(t.view.pluralTreeKey) - a.rw.Tree.DeletePrefix(t.view.pluralTreeKey + "/") + a.pwRoot.Tree.Delete(t.view.pluralTreeKey) + a.rwRoot.Tree.DeletePrefix(t.view.pluralTreeKey + "/") } } // Find new terms. for t := range a.seenTerms { if _, found := a.h.previousSeenTerms[t]; !found { // This term is new. - if n, ok := a.pw.Tree.GetRaw(t.view.pluralTreeKey); ok { + if n, ok := a.pwRoot.Tree.GetRaw(t.view.pluralTreeKey); ok { a.assembleChanges.Add(cnh.GetIdentity(n)) } } @@ -136,6 +132,52 @@ func (a *allPagesAssembler) createAllPages() error { a.h.previousSeenTerms = a.seenTerms }() } + workers := para.New(config.GetNumWorkerMultiplier()) + a.r, _ = workers.Start(context.Background()) + if err := cmp.Or(a.doCreatePages(""), a.r.Wait()); err != nil { + return err + } + if err := a.pwRoot.WalkContext.HandleHooks1AndEventsAndHooks2(); err != nil { + return err + } + return nil +} + +func (a *allPagesAssembler) doCreatePages(prefix string) error { + var ( + sites = a.h.sitesVersionsRolesMap + h = a.h + treePages = a.m.treePages + + getViews = func(vec sitesmatrix.Vector) []viewName { + return h.languageSiteForSiteVector(vec).pageMap.cfg.taxonomyConfig.views + } + + pw *doctree.NodeShiftTreeWalker[contentNode] // walks pages. + rw *doctree.NodeShiftTreeWalker[contentNode] // walks resources. + + isRootWalk = prefix == "" + ) + + if isRootWalk { + pw = a.pwRoot + rw = a.rwRoot + } else { + // Sub-walkers for a specific prefix. + pw = a.pwRoot.Extend() + pw.Prefix = prefix + "/" + + rw = a.rwRoot.Extend() // rw will get its prefix(es) set later. + + pw.TransformDelayInsert = true + rw.TransformDelayInsert = true + + } + + resourceOwnerInfo := struct { + n contentNode + s string + }{} newHomePageMetaSource := func() *pageMetaSource { pi := a.h.Conf.PathParser().Parse(files.ComponentFolderContent, "/_index.md") @@ -149,14 +191,16 @@ func (a *allPagesAssembler) createAllPages() error { } } - if err := a.createMissingPages(); err != nil { - return err - } + if isRootWalk { + if err := a.createMissingPages(); err != nil { + return err + } - if treePages.Len() == 0 { - // No pages, insert a home page to get something to walk on. - p := newHomePageMetaSource() - treePages.InsertRaw(p.pathInfo.Base(), p) + if treePages.Len() == 0 { + // No pages, insert a home page to get something to walk on. + p := newHomePageMetaSource() + treePages.InsertRaw(p.pathInfo.Base(), p) + } } getCascades := func(wctx *doctree.WalkContext[contentNode], s string) *page.PageMatcherParamsConfigs { @@ -201,7 +245,7 @@ func (a *allPagesAssembler) createAllPages() error { if s == "" || cascades.Len() > cascadesLen { // New cascade values added, pass them downwards. - a.rw.WalkContext.Data().Insert(paths.AddTrailingSlash(s), cascades) + rw.WalkContext.Data().Insert(paths.AddTrailingSlash(s), cascades) } }() @@ -351,13 +395,6 @@ func (a *allPagesAssembler) createAllPages() error { } } - var ( - seenRootSections = map[string]bool{ - "": true, // The home section. - } - seenHome bool - ) - var unpackPageMetaSources func(n contentNode) contentNode unpackPageMetaSources = func(n contentNode) contentNode { switch nn := n.(type) { @@ -430,18 +467,17 @@ func (a *allPagesAssembler) createAllPages() error { level := strings.Count(s, "/") if s == "" { - seenHome = true + a.seenHome = true } - if !isResource && s != "" && !seenHome { - + if !isResource && s != "" && !a.seenHome { var homePages contentNode homePages, ns, err = transformPages("", newHomePageMetaSource(), cascades) if err != nil { return } treePages.InsertRaw("", homePages) - seenHome = true + a.seenHome = true } n2, ns, err = transformPages(s, n, cascades) @@ -460,16 +496,15 @@ func (a *allPagesAssembler) createAllPages() error { } isTaxonomy := !a.h.getFirstTaxonomyConfig(s).IsZero() - isRootSetion := !isTaxonomy && level == 1 && cnh.isBranchNode(n) + isRootSection := !isTaxonomy && level == 1 && cnh.isBranchNode(n) - if isRootSetion { + if isRootSection { // This is a root section. - seenRootSections[cnh.PathInfo(n).Section()] = true + a.seenRootSections.SetIfAbsent(cnh.PathInfo(n).Section(), true) } else if !isTaxonomy { p := cnh.PathInfo(n) rootSection := p.Section() - if !seenRootSections[rootSection] { - seenRootSections[rootSection] = true + _, err := a.seenRootSections.GetOrCreate(rootSection, func() (bool, error) { // Try to preserve the original casing if possible. sectionUnnormalized := p.Unnormalized().Section() rootSectionPath := a.h.Conf.PathParser().Parse(files.ComponentFolderContent, "/"+sectionUnnormalized+"/_index.md") @@ -482,15 +517,20 @@ func (a *allPagesAssembler) createAllPages() error { }, }, cascades) if err != nil { - return + return true, err } treePages.InsertRaw(rootSectionPath.Base(), rootSectionPages) + return true, nil + }) + if err != nil { + return nil, 0, err } + } const eventNameSitesMatrix = "sitesmatrix" - if s == "" || isRootSetion { + if s == "" || isRootSection { // Every page needs a home and a root section (.FirstSection). // We don't know yet what language, version, role combination that will @@ -509,7 +549,7 @@ func (a *allPagesAssembler) createAllPages() error { return true }) } else { - a.pw.WalkContext.AddEventListener(eventNameSitesMatrix, s, + pw.WalkContext.AddEventListener(eventNameSitesMatrix, s, func(e *doctree.Event[contentNode]) { n := e.Source e.StopPropagation() @@ -525,7 +565,7 @@ func (a *allPagesAssembler) createAllPages() error { } // We need to wait until after the walk to have a complete set. - a.pw.WalkContext.AddPostHook( + pw.WalkContext.HooksPost2().Push( func() error { if i := len(missingVectorsForHomeOrRootSection); i > 0 { // Pick one, the rest will be created later. @@ -556,7 +596,7 @@ func (a *allPagesAssembler) createAllPages() error { } if replaced { - a.pw.Tree.InsertRaw(s, nm) + pw.Tree.InsertRaw(s, nm) } } return nil @@ -565,14 +605,14 @@ func (a *allPagesAssembler) createAllPages() error { } if s != "" { - a.rw.WalkContext.SendEvent(&doctree.Event[contentNode]{Source: n2, Path: s, Name: eventNameSitesMatrix}) + rw.WalkContext.SendEvent(&doctree.Event[contentNode]{Source: n2, Path: s, Name: eventNameSitesMatrix}) } return } transformPagesAndCreateMissingStructuralNodes := func(s string, n contentNode, isResource bool) (n2 contentNode, ns doctree.NodeTransformState, err error) { - cascades := getCascades(a.pw.WalkContext, s) + cascades := getCascades(pw.WalkContext, s) n2, ns, err = transformPagesAndCreateMissingHome(s, n, isResource, cascades) if err != nil || ns >= doctree.NodeTransformStateSkip { return @@ -643,7 +683,7 @@ func (a *allPagesAssembler) createAllPages() error { } // Stop walking downwards, someone else owns this resource. - a.rw.SkipPrefix(ownerKey + "/") + rw.SkipPrefix(ownerKey + "/") return doctree.NodeTransformStateSkip } return @@ -665,7 +705,7 @@ func (a *allPagesAssembler) createAllPages() error { return true } - a.rw.Transform = func(s string, n contentNode) (n2 contentNode, ns doctree.NodeTransformState, err error) { + rw.Transform = func(s string, n contentNode) (n2 contentNode, ns doctree.NodeTransformState, err error) { if ns = shouldSkipOrTerminate(s); ns >= doctree.NodeTransformStateSkip { return } @@ -710,7 +750,7 @@ func (a *allPagesAssembler) createAllPages() error { } // Create missing term pages. - a.pw.WalkContext.AddPostHook( + pw.WalkContext.HooksPost2().Push( func() error { for k, v := range a.seenTerms { viewTermKey := "/" + k.view.plural + "/" + k.term @@ -718,7 +758,7 @@ func (a *allPagesAssembler) createAllPages() error { pi := a.h.Conf.PathParser().Parse(files.ComponentFolderContent, viewTermKey+"/_index.md") termKey := pi.Base() - n, found := a.pw.Tree.GetRaw(termKey) + n, found := pw.Tree.GetRaw(termKey) if found { // Merge. @@ -742,14 +782,14 @@ func (a *allPagesAssembler) createAllPages() error { if found { n2 = contentNodes{n, p} } - n2, ns, err := transformPages(termKey, n2, getCascades(a.pw.WalkContext, termKey)) + n2, ns, err := transformPages(termKey, n2, getCascades(pw.WalkContext, termKey)) if err != nil { return fmt.Errorf("failed to create term page %q: %w", termKey, err) } switch ns { case doctree.NodeTransformStateReplaced: - a.pw.Tree.InsertRaw(termKey, n2) + pw.Tree.InsertRaw(termKey, n2) } } @@ -758,7 +798,7 @@ func (a *allPagesAssembler) createAllPages() error { }, ) - a.pw.Transform = func(s string, n contentNode) (n2 contentNode, ns doctree.NodeTransformState, err error) { + pw.Transform = func(s string, n contentNode) (n2 contentNode, ns doctree.NodeTransformState, err error) { n2, ns, err = transformPagesAndCreateMissingStructuralNodes(s, n, false) if err != nil || ns >= doctree.NodeTransformStateSkip { @@ -776,21 +816,25 @@ func (a *allPagesAssembler) createAllPages() error { // Walk nested resources. resourceOwnerInfo.s = s resourceOwnerInfo.n = n2 - a.rw.Prefix = s + "/" - if err := a.rw.Walk(a.ctx); err != nil { + rw = rw.WithPrefix(s + "/") + if err := rw.Walk(a.ctx); err != nil { return nil, 0, err } + if a.assembleSectionsInParallel && prefix == "" && s != "" && cnh.isBranchNode(n) && a.h.getFirstTaxonomyConfig(s).IsZero() { + // Handle this branch's descendants in its own goroutine. + pw.SkipPrefix(s + "/") + a.r.Run(func() error { + return a.doCreatePages(s) + }) + } + return } - a.pw.Handle = nil - - if err := a.pw.Walk(a.ctx); err != nil { - return err - } + pw.Handle = nil - if err := a.pw.WalkContext.HandleEventsAndHooks(); err != nil { + if err := pw.Walk(a.ctx); err != nil { return err } @@ -829,7 +873,7 @@ func (sa *sitePagesAssembler) applyAggregates() error { oldDates := pageBundle.m.pageConfig.Dates // We need to wait until after the walk to determine if any of the dates have changed. - pw.WalkContext.AddPostHook( + pw.WalkContext.HooksPost2().Push( func() error { if oldDates != pageBundle.m.pageConfig.Dates { sa.assembleChanges.Add(pageBundle) @@ -910,7 +954,7 @@ func (sa *sitePagesAssembler) applyAggregates() error { return err } - if err := pw.WalkContext.HandleEventsAndHooks(); err != nil { + if err := pw.WalkContext.HandleHooks1AndEventsAndHooks2(); err != nil { return err } @@ -997,7 +1041,7 @@ func (sa *sitePagesAssembler) applyAggregatesToTaxonomiesAndTerms() error { } } - if err := walkContext.HandleEventsAndHooks(); err != nil { + if err := walkContext.HandleHooks1AndEventsAndHooks2(); err != nil { return err } diff --git a/hugolib/content_map_page_contentnodeshifter.go b/hugolib/content_map_page_contentnodeshifter.go index b9d183ddd..c026997fc 100644 --- a/hugolib/content_map_page_contentnodeshifter.go +++ b/hugolib/content_map_page_contentnodeshifter.go @@ -146,7 +146,7 @@ func (s *contentNodeShifter) Shift(n contentNode, siteVector sitesmatrix.Vector, return vv, true } default: - panic(fmt.Sprintf("Shift: unknown type %T", n)) + panic(fmt.Sprintf("Shift: unknown type %T for %q", n, n.Path())) } if !fallback { diff --git a/hugolib/doctree/nodeshiftree_test.go b/hugolib/doctree/nodeshiftree_test.go index 74ce26043..b75586ce1 100644 --- a/hugolib/doctree/nodeshiftree_test.go +++ b/hugolib/doctree/nodeshiftree_test.go @@ -147,7 +147,7 @@ func TestTreeEvents(t *testing.T) { } c.Assert(w.Walk(context.Background()), qt.IsNil) - c.Assert(w.WalkContext.HandleEventsAndHooks(), qt.IsNil) + c.Assert(w.WalkContext.HandleHooks1AndEventsAndHooks2(), qt.IsNil) c.Assert(tree.Get("/a").Weight, eq, 9) c.Assert(tree.Get("/a/s1").Weight, eq, 9) diff --git a/hugolib/doctree/nodeshifttree.go b/hugolib/doctree/nodeshifttree.go index aa1b6a723..8f64be396 100644 --- a/hugolib/doctree/nodeshifttree.go +++ b/hugolib/doctree/nodeshifttree.go @@ -381,6 +381,9 @@ type NodeShiftTreeWalker[T any] struct { // the third bool indicates if the walk should terminate. Transform func(s string, v1 T) (v2 T, state NodeTransformState, err error) + // When set, will add inserts to WalkContext.HooksPost1 to be performed after the walk. + TransformDelayInsert bool + // Handle will be called for each node in the main tree. // If the callback returns true, the walk will stop. // The callback can optionally return a callback for the nested tree. @@ -405,8 +408,8 @@ type NodeShiftTreeWalker[T any] struct { // When set, no dimension shifting will be performed. NoShift bool - // WHen set, will try to fall back to alternative match, - // typically a shared resoures common for all languages. + // When set, will try to fall back to alternative match, + // typically a shared resource common for all languages. Fallback bool // Used in development only. @@ -423,14 +426,21 @@ type NodeShiftTreeWalker[T any] struct { skipPrefixes []string } -// Extend returns a new NodeShiftTreeWalker with the same configuration as the +// Extend returns a new NodeShiftTreeWalker with the same configuration // and the same WalkContext as the original. // Any local state is reset. func (r NodeShiftTreeWalker[T]) Extend() *NodeShiftTreeWalker[T] { - r.resetLocalState() + r.skipPrefixes = nil return &r } +// WithPrefix returns a new NodeShiftTreeWalker with the given prefix. +func (r *NodeShiftTreeWalker[T]) WithPrefix(prefix string) *NodeShiftTreeWalker[T] { + r2 := r.Extend() + r2.Prefix = prefix + return r2 +} + // SkipPrefix adds a prefix to be skipped in the walk. func (r *NodeShiftTreeWalker[T]) SkipPrefix(prefix ...string) { r.skipPrefixes = append(r.skipPrefixes, prefix...) @@ -474,7 +484,18 @@ func (r *NodeShiftTreeWalker[T]) Walk(ctx context.Context) error { switch ns { case NodeTransformStateReplaced: - r.Tree.tree.Insert(s, t) + // Delay insert until after the walk to + // avoid locking issues. + if r.TransformDelayInsert { + r.WalkContext.HooksPost1().Push( + func() error { + r.Tree.InsertRaw(s, t) + return nil + }, + ) + } else { + r.Tree.InsertRaw(s, t) + } case NodeTransformStateDeleted: // Delay delete until after the walk. deletes = append(deletes, s) diff --git a/hugolib/doctree/support.go b/hugolib/doctree/support.go index 9cdc36fda..3692b1167 100644 --- a/hugolib/doctree/support.go +++ b/hugolib/doctree/support.go @@ -19,6 +19,7 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/hugolib/sitesmatrix" ) @@ -34,6 +35,9 @@ const ( // AddEventListener adds an event listener to the tree. // Note that the handler func may not add listeners. func (ctx *WalkContext[T]) AddEventListener(event, path string, handler func(*Event[T])) { + ctx.eventHandlersMu.Lock() + defer ctx.eventHandlersMu.Unlock() + if ctx.eventHandlers == nil { ctx.eventHandlers = make(eventHandlers[T]) } @@ -56,12 +60,6 @@ func (ctx *WalkContext[T]) AddEventListener(event, path string, handler func(*Ev ) } -// AddPostHook adds a post hook to the tree. -// This will be run after the tree has been walked. -func (ctx *WalkContext[T]) AddPostHook(handler func() error) { - ctx.HooksPost = append(ctx.HooksPost, handler) -} - func (ctx *WalkContext[T]) Data() *SimpleThreadSafeTree[any] { ctx.dataInit.Do(func() { ctx.data = NewSimpleThreadSafeTree[any]() @@ -94,6 +92,8 @@ func (ctx *WalkContext[T]) DataRawForEeach() iter.Seq2[sitesmatrix.Vector, *Simp // SendEvent sends an event up the tree. func (ctx *WalkContext[T]) SendEvent(event *Event[T]) { + ctx.eventMu.Lock() + defer ctx.eventMu.Unlock() ctx.events = append(ctx.events, event) } @@ -213,10 +213,16 @@ type WalkContext[T any] struct { dataRaw *maps.Cache[sitesmatrix.Vector, *SimpleThreadSafeTree[any]] dataRawInit sync.Once - eventHandlers eventHandlers[T] - events []*Event[T] + eventHandlersMu sync.Mutex + eventHandlers eventHandlers[T] + eventMu sync.Mutex + events []*Event[T] + + hooksPost1Init sync.Once + hooksPost1 *collections.Stack[func() error] - HooksPost []func() error + hooksPost2Init sync.Once + hooksPost2 *collections.Stack[func() error] } type eventHandlers[T any] map[string][]func(*Event[T]) @@ -233,6 +239,9 @@ func cleanKey(key string) string { } func (ctx *WalkContext[T]) HandleEvents() error { + ctx.eventHandlersMu.Lock() + defer ctx.eventHandlersMu.Unlock() + for len(ctx.events) > 0 { event := ctx.events[0] ctx.events = ctx.events[1:] @@ -250,12 +259,32 @@ func (ctx *WalkContext[T]) HandleEvents() error { return nil } -func (ctx *WalkContext[T]) HandleEventsAndHooks() error { +func (ctx *WalkContext[T]) HooksPost1() *collections.Stack[func() error] { + ctx.hooksPost1Init.Do(func() { + ctx.hooksPost1 = collections.NewStack[func() error]() + }) + return ctx.hooksPost1 +} + +func (ctx *WalkContext[T]) HooksPost2() *collections.Stack[func() error] { + ctx.hooksPost2Init.Do(func() { + ctx.hooksPost2 = collections.NewStack[func() error]() + }) + return ctx.hooksPost2 +} + +func (ctx *WalkContext[T]) HandleHooks1AndEventsAndHooks2() error { + for _, hook := range ctx.HooksPost1().All() { + if err := hook(); err != nil { + return err + } + } + if err := ctx.HandleEvents(); err != nil { return err } - for _, hook := range ctx.HooksPost { + for _, hook := range ctx.HooksPost2().All() { if err := hook(); err != nil { return err } diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index b0157bb59..87321d471 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -336,7 +336,7 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil if h.Conf.Watching() { defer func() { // Store previous walk context to detect cascade changes on next rebuild. - h.previousPageTreesWalkContext = apa.rw.WalkContext + h.previousPageTreesWalkContext = apa.rwRoot.WalkContext }() } diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 29cdaa482..4ef48e6f4 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -204,6 +204,7 @@ func BenchmarkAssembleDeepSiteWithManySections(b *testing.B) { }) } + runOne(1, 1, 20) runOne(1, 6, 100) runOne(2, 2, 100) runOne(2, 6, 100) diff --git a/hugolib/sitesmatrix/sitematrix_integration_test.go b/hugolib/sitesmatrix/sitematrix_integration_test.go index 048fcc791..1b1174614 100644 --- a/hugolib/sitesmatrix/sitematrix_integration_test.go +++ b/hugolib/sitesmatrix/sitematrix_integration_test.go @@ -1231,3 +1231,27 @@ func BenchmarkSitesMatrixContent(b *testing.B) { } } } + +// Just a test to test the development of concurrency in page assembly. +func TestCreateAllPagesPartitionSections(t *testing.T) { + files := ` +-- hugo.toml -- +baseURL = "https://example.org/" +disableKinds = ["rss", "sitemap", "taxonomy", "term"] +-- content/_index.md -- +-- content/s1/_index.md -- +-- content/s1/p1.md -- +-- content/s1/p2.md -- +-- content/s2/_index.md -- +-- content/s2/p1.md -- +-- content/s2_1/_index.md -- +-- content/s2_1/p1.md -- +-- layouts/all.html -- +{{ .Kind }}|{{ .RelPermalink }}| +` + + for range 3 { + b := hugolib.Test(t, files) + b.AssertFileContent("public/s1/index.html", "section|/s1/|") + } +} -- 2.39.5