]> git.maquefel.me Git - brevno-suite/hugo/commitdiff
tpl: Add a partial lookup cache
authorBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Thu, 10 Apr 2025 07:22:29 +0000 (09:22 +0200)
committerBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Thu, 10 Apr 2025 09:07:19 +0000 (11:07 +0200)
````
                 │ stash.bench  │          perf-v146.bench           │
                 │    sec/op    │   sec/op     vs base               │
LookupPartial-10   248.00n ± 0%   14.75n ± 2%  -94.05% (p=0.002 n=6)

                 │ stash.bench │          perf-v146.bench          │
                 │    B/op     │   B/op     vs base                │
LookupPartial-10    48.00 ± 0%   0.00 ± 0%  -100.00% (p=0.002 n=6)

                 │ stash.bench │          perf-v146.bench           │
                 │  allocs/op  │ allocs/op   vs base                │
LookupPartial-10    3.000 ± 0%   0.000 ± 0%  -100.00% (p=0.002 n=6)
```

THe speedup above assumes reuse of the same partials over and over again, which I think is not uncommon.

This commits also adds some more lookup benchmarks. The current output of these on my MacBook looks decent:

```
BenchmarkLookupPagesLayout/Single_root-10                3031562               395.5 ns/op             0 B/op          0 allocs/op
BenchmarkLookupPagesLayout/Single_sub_folder-10          2515915               480.9 ns/op             0 B/op          0 allocs/op
BenchmarkLookupPartial-10                               84808112                14.13 ns/op            0 B/op          0 allocs/op
BenchmarkLookupShortcode/toplevelpage-10                 8111779               148.2 ns/op             0 B/op          0 allocs/op
BenchmarkLookupShortcode/nestedpage-10                   8088183               148.6 ns/op             0 B/op          0 allocs/op
```

Note that in the above the partial lookups are cahced, the others not (they are harder to cache because of the page path).

Closes #13571

common/maps/cache.go
hugolib/alias.go
hugolib/page.go
hugolib/page__common.go
hugolib/page__new.go
hugolib/page__per_output.go
hugolib/shortcode.go
resources/page/page.go
resources/page/pages_related.go
tpl/tplimpl/templatestore.go
tpl/tplimpl/templatestore_integration_test.go

index 7304a7a6f4e9a45cb3a3efffddfd9d25b43ad81b..de1535994260d2799ec8bf13270f2414d9d13d09 100644 (file)
@@ -160,7 +160,7 @@ func (c *Cache[K, T]) Len() int {
 
 func (c *Cache[K, T]) Reset() {
        c.Lock()
-       c.m = make(map[K]T)
+       clear(c.m)
        c.hasBeenInitialized = false
        c.Unlock()
 }
index 0d182042a59a26ef7cc2da96af78f9a72058ded0..3beee44dbb739d435817390a2e6bd0609594dc86 100644 (file)
@@ -51,7 +51,7 @@ func (a aliasHandler) renderAlias(permalink string, p page.Page) (io.Reader, err
        var templateDesc tplimpl.TemplateDescriptor
        var base string = ""
        if ps, ok := p.(*pageState); ok {
-               base, templateDesc = ps.getTemplateBasePathAndDescriptor()
+               base, templateDesc = ps.GetInternalTemplateBasePathAndDescriptor()
        }
        templateDesc.Layout = ""
        templateDesc.Kind = ""
index 41519909b163b81a167d460b12e65ab1194d78e8..b36b9712ca9e471371e9349b8d281991b9772ceb 100644 (file)
@@ -476,7 +476,8 @@ func (ps *pageState) initCommonProviders(pp pagePaths) error {
        return nil
 }
 
-func (po *pageOutput) getTemplateBasePathAndDescriptor() (string, tplimpl.TemplateDescriptor) {
+// Exported so it can be used in integration tests.
+func (po *pageOutput) GetInternalTemplateBasePathAndDescriptor() (string, tplimpl.TemplateDescriptor) {
        p := po.p
        f := po.f
        base := p.PathInfo().BaseReTyped(p.m.pageConfig.Type)
@@ -491,7 +492,7 @@ func (po *pageOutput) getTemplateBasePathAndDescriptor() (string, tplimpl.Templa
 }
 
 func (p *pageState) resolveTemplate(layouts ...string) (*tplimpl.TemplInfo, bool, error) {
-       dir, d := p.getTemplateBasePathAndDescriptor()
+       dir, d := p.GetInternalTemplateBasePathAndDescriptor()
 
        if len(layouts) > 0 {
                d.Layout = layouts[0]
index 27d2c3089b2ab605266e36ecd103229dafcdb630..f6f01bbe20492ed28774c5576e40b2bdc67784f4 100644 (file)
@@ -97,7 +97,7 @@ type pageCommon struct {
        pageMenus *pageMenus
 
        // Internal use
-       page.InternalDependencies
+       page.RelatedDocsHandlerProvider
 
        contentConverterInit sync.Once
        contentConverter     converter.Converter
index b8c3483add689b71b0ec9f5eec57845d98510a04..10336e255832d1a0f31a4966904ae4c7eafe4002 100644 (file)
@@ -209,11 +209,11 @@ func (h *HugoSites) doNewPage(m *pageMeta) (*pageState, *paths.Path, error) {
                                ShortcodeInfoProvider:     page.NopPage,
                                LanguageProvider:          m.s,
 
-                               InternalDependencies: m.s,
-                               init:                 lazy.New(),
-                               m:                    m,
-                               s:                    m.s,
-                               sWrapped:             page.WrapSite(m.s),
+                               RelatedDocsHandlerProvider: m.s,
+                               init:                       lazy.New(),
+                               m:                          m,
+                               s:                          m.s,
+                               sWrapped:                   page.WrapSite(m.s),
                        },
                }
 
index 71bc51828a2c26614d5895a00676e4b2f1d69e71..1f7f3411e35975fd158e6c3046d880cdc5f1f954 100644 (file)
@@ -275,7 +275,7 @@ func (pco *pageContentOutput) initRenderHooks() error {
                        // Inherit the descriptor from the page/current output format.
                        // This allows for fine-grained control of the template used for
                        // rendering of e.g. links.
-                       base, layoutDescriptor := pco.po.p.getTemplateBasePathAndDescriptor()
+                       base, layoutDescriptor := pco.po.p.GetInternalTemplateBasePathAndDescriptor()
 
                        switch tp {
                        case hooks.LinkRendererType:
index 7a23b15d2f2afb52ceb37fec9b5122e2ce75fbdf..3ac0940e2cd4e4ed3594fcd10d08dae45a963ec2 100644 (file)
@@ -397,7 +397,7 @@ func doRenderShortcode(
                        ofCount[match.D.OutputFormat]++
                        return true
                }
-               base, layoutDescriptor := po.getTemplateBasePathAndDescriptor()
+               base, layoutDescriptor := po.GetInternalTemplateBasePathAndDescriptor()
                q := tplimpl.TemplateQuery{
                        Path:     base,
                        Name:     sc.name,
index da7b6beece600c6e3aca4b58eea31fc82894ac2e..cbcfad5579652f541e334746913db5178b6246c5 100644 (file)
@@ -148,8 +148,8 @@ type InSectionPositioner interface {
        PrevInSection() Page
 }
 
-// InternalDependencies is considered an internal interface.
-type InternalDependencies interface {
+// RelatedDocsHandlerProvider is considered an internal interface.
+type RelatedDocsHandlerProvider interface {
        // GetInternalRelatedDocsHandler is for internal use only.
        GetInternalRelatedDocsHandler() *RelatedDocsHandler
 }
index 9debda32456701eea4730449c0e6a702a5f0dd4e..402ed905f5f477c09b2d8a2a4f10f9b127ade656 100644 (file)
@@ -124,7 +124,7 @@ func (p Pages) withInvertedIndex(ctx context.Context, search func(idx *related.I
                return nil, nil
        }
 
-       d, ok := p[0].(InternalDependencies)
+       d, ok := p[0].(RelatedDocsHandlerProvider)
        if !ok {
                return nil, fmt.Errorf("invalid type %T in related search", p[0])
        }
index fe31806609b8fc50c39bce1148301d270c822769..f6a119e9b5a1c50334466fdc603304bb5b80a4f3 100644 (file)
@@ -97,16 +97,16 @@ func NewStore(opts StoreOptions, siteOpts SiteOptions) (*TemplateStore, error) {
                panic("HTML output format not found")
        }
        s := &TemplateStore{
-               opts:                     opts,
-               siteOpts:                 siteOpts,
-               optsOrig:                 opts,
-               siteOptsOrig:             siteOpts,
-               htmlFormat:               html,
-               storeSite:                configureSiteStorage(siteOpts, opts.Watching),
-               treeMain:                 doctree.NewSimpleTree[map[nodeKey]*TemplInfo](),
-               treeShortcodes:           doctree.NewSimpleTree[map[string]map[TemplateDescriptor]*TemplInfo](),
-               templatesByPath:          maps.NewCache[string, *TemplInfo](),
-               templateDescriptorByPath: maps.NewCache[string, PathTemplateDescriptor](),
+               opts:                opts,
+               siteOpts:            siteOpts,
+               optsOrig:            opts,
+               siteOptsOrig:        siteOpts,
+               htmlFormat:          html,
+               storeSite:           configureSiteStorage(siteOpts, opts.Watching),
+               treeMain:            doctree.NewSimpleTree[map[nodeKey]*TemplInfo](),
+               treeShortcodes:      doctree.NewSimpleTree[map[string]map[TemplateDescriptor]*TemplInfo](),
+               templatesByPath:     maps.NewCache[string, *TemplInfo](),
+               cacheLookupPartials: maps.NewCache[string, *TemplInfo](),
 
                // Note that the funcs passed below is just for name validation.
                tns: newTemplateNamespace(siteOpts.TemplateFuncs),
@@ -400,10 +400,9 @@ type TemplateStore struct {
        siteOpts   SiteOptions
        htmlFormat output.Format
 
-       treeMain                 *doctree.SimpleTree[map[nodeKey]*TemplInfo]
-       treeShortcodes           *doctree.SimpleTree[map[string]map[TemplateDescriptor]*TemplInfo]
-       templatesByPath          *maps.Cache[string, *TemplInfo]
-       templateDescriptorByPath *maps.Cache[string, PathTemplateDescriptor]
+       treeMain        *doctree.SimpleTree[map[nodeKey]*TemplInfo]
+       treeShortcodes  *doctree.SimpleTree[map[string]map[TemplateDescriptor]*TemplInfo]
+       templatesByPath *maps.Cache[string, *TemplInfo]
 
        dh descriptorHandler
 
@@ -417,6 +416,9 @@ type TemplateStore struct {
        // For testing benchmarking.
        optsOrig     StoreOptions
        siteOptsOrig SiteOptions
+
+       // caches. These need to be refreshed when the templates are refreshed.
+       cacheLookupPartials *maps.Cache[string, *TemplInfo]
 }
 
 // NewFromOpts creates a new store with the same configuration as the original.
@@ -540,15 +542,19 @@ func (s *TemplateStore) LookupPagesLayout(q TemplateQuery) *TemplInfo {
 }
 
 func (s *TemplateStore) LookupPartial(pth string) *TemplInfo {
-       d := s.templateDescriptorFromPath(pth)
-       desc := d.Desc
-       if desc.Layout != "" {
-               panic("shortcode template descriptor must not have a layout")
-       }
-       best := s.getBest()
-       defer s.putBest(best)
-       s.findBestMatchGet(s.key(path.Join(containerPartials, d.Path)), CategoryPartial, nil, desc, best)
-       return best.templ
+       ti, _ := s.cacheLookupPartials.GetOrCreate(pth, func() (*TemplInfo, error) {
+               d := s.templateDescriptorFromPath(pth)
+               desc := d.Desc
+               if desc.Layout != "" {
+                       panic("shortcode template descriptor must not have a layout")
+               }
+               best := s.getBest()
+               defer s.putBest(best)
+               s.findBestMatchGet(s.key(path.Join(containerPartials, d.Path)), CategoryPartial, nil, desc, best)
+               return best.templ, nil
+       })
+
+       return ti
 }
 
 func (s *TemplateStore) LookupShortcode(q TemplateQuery) *TemplInfo {
@@ -619,8 +625,14 @@ func (s *TemplateStore) PrintDebug(prefix string, category Category, w io.Writer
        })
 }
 
+func (s *TemplateStore) clearCaches() {
+       s.cacheLookupPartials.Reset()
+}
+
 // RefreshFiles refreshes this store for the files matching the given predicate.
 func (s *TemplateStore) RefreshFiles(include func(fi hugofs.FileMetaInfo) bool) error {
+       s.clearCaches()
+
        if err := s.tns.createPrototypesParse(); err != nil {
                return err
        }
@@ -1370,43 +1382,38 @@ type PathTemplateDescriptor struct {
 // templateDescriptorFromPath returns a template descriptor from the given path.
 // This is currently used in partial lookups only.
 func (s *TemplateStore) templateDescriptorFromPath(pth string) PathTemplateDescriptor {
-       // Check cache first.
-       d, _ := s.templateDescriptorByPath.GetOrCreate(pth, func() (PathTemplateDescriptor, error) {
-               var (
-                       mt media.Type
-                       of output.Format
-               )
-
-               // Common cases.
-               dotCount := strings.Count(pth, ".")
-               if dotCount <= 1 {
-                       if dotCount == 0 {
-                               // Asume HTML.
-                               of, mt = s.resolveOutputFormatAndOrMediaType("html", "")
-                       } else {
-                               pth = strings.TrimPrefix(pth, "/")
-                               ext := path.Ext(pth)
-                               pth = strings.TrimSuffix(pth, ext)
-                               ext = ext[1:]
-                               of, mt = s.resolveOutputFormatAndOrMediaType("", ext)
-                       }
+       var (
+               mt media.Type
+               of output.Format
+       )
+
+       // Common cases.
+       dotCount := strings.Count(pth, ".")
+       if dotCount <= 1 {
+               if dotCount == 0 {
+                       // Asume HTML.
+                       of, mt = s.resolveOutputFormatAndOrMediaType("html", "")
                } else {
-                       path := s.opts.PathParser.Parse(files.ComponentFolderLayouts, pth)
-                       pth = path.PathNoIdentifier()
-                       of, mt = s.resolveOutputFormatAndOrMediaType(path.OutputFormat(), path.Ext())
-               }
-
-               return PathTemplateDescriptor{
-                       Path: pth,
-                       Desc: TemplateDescriptor{
-                               OutputFormat: of.Name,
-                               MediaType:    mt.Type,
-                               IsPlainText:  of.IsPlainText,
-                       },
-               }, nil
-       })
+                       pth = strings.TrimPrefix(pth, "/")
+                       ext := path.Ext(pth)
+                       pth = strings.TrimSuffix(pth, ext)
+                       ext = ext[1:]
+                       of, mt = s.resolveOutputFormatAndOrMediaType("", ext)
+               }
+       } else {
+               path := s.opts.PathParser.Parse(files.ComponentFolderLayouts, pth)
+               pth = path.PathNoIdentifier()
+               of, mt = s.resolveOutputFormatAndOrMediaType(path.OutputFormat(), path.Ext())
+       }
 
-       return d
+       return PathTemplateDescriptor{
+               Path: pth,
+               Desc: TemplateDescriptor{
+                       OutputFormat: of.Name,
+                       MediaType:    mt.Type,
+                       IsPlainText:  of.IsPlainText,
+               },
+       }
 }
 
 // resolveOutputFormatAndOrMediaType resolves the output format and/or media type
index 2f450e58dd39b9b35b1287fc172c583f1526040b..2f38495621ee75177f322c97e390335584ac6043 100644 (file)
@@ -8,6 +8,7 @@ import (
        qt "github.com/frankban/quicktest"
        "github.com/gohugoio/hugo/hugolib"
        "github.com/gohugoio/hugo/resources/kinds"
+       "github.com/gohugoio/hugo/resources/page"
        "github.com/gohugoio/hugo/tpl/tplimpl"
 )
 
@@ -849,7 +850,7 @@ func BenchmarkExecuteWithContext(b *testing.B) {
 disableKinds = ["taxonomy", "term", "home"]
 -- layouts/all.html --
 {{ .Title }}|
- {{ partial "p1.html" . }}
+{{ partial "p1.html" . }}
 -- layouts/_partials/p1.html --
  p1.
 {{ partial "p2.html" . }}
@@ -878,6 +879,82 @@ p3
        b.ResetTimer()
        for i := 0; i < b.N; i++ {
                err := store.ExecuteWithContext(context.Background(), ti, io.Discard, p)
-               bb.Assert(err, qt.IsNil)
+               if err != nil {
+                       b.Fatal(err)
+               }
+       }
+}
+
+func BenchmarkLookupPartial(b *testing.B) {
+       files := `
+-- hugo.toml --
+disableKinds = ["taxonomy", "term", "home"]
+-- layouts/all.html --
+{{ .Title }}|
+-- layouts/_partials/p1.html --
+-- layouts/_partials/p2.html --
+-- layouts/_partials/p2.json --
+-- layouts/_partials/p3.html --
+`
+       bb := hugolib.Test(b, files)
+
+       store := bb.H.TemplateStore
+
+       for i := 0; i < b.N; i++ {
+               fi := store.LookupPartial("p3.html")
+               if fi == nil {
+                       b.Fatal("not found")
+               }
        }
 }
+
+// Implemented by pageOutput.
+type getDescriptorProvider interface {
+       GetInternalTemplateBasePathAndDescriptor() (string, tplimpl.TemplateDescriptor)
+}
+
+func BenchmarkLookupShortcode(b *testing.B) {
+       files := `
+-- hugo.toml --
+disableKinds = ["taxonomy", "term", "home"]
+-- content/toplevelpage.md --
+-- content/a/b/c/nested.md --
+-- layouts/all.html --
+{{ .Title }}|
+-- layouts/_shortcodes/s.html --
+s1.
+-- layouts/_shortcodes/a/b/s.html --
+s2.
+
+`
+       bb := hugolib.Test(b, files)
+       store := bb.H.TemplateStore
+
+       runOne := func(p page.Page) {
+               pth, desc := p.(getDescriptorProvider).GetInternalTemplateBasePathAndDescriptor()
+               q := tplimpl.TemplateQuery{
+                       Path:     pth,
+                       Name:     "s",
+                       Category: tplimpl.CategoryShortcode,
+                       Desc:     desc,
+               }
+               v := store.LookupShortcode(q)
+               if v == nil {
+                       b.Fatal("not found")
+               }
+       }
+
+       b.Run("toplevelpage", func(b *testing.B) {
+               toplevelpage, _ := bb.H.Sites[0].GetPage("/toplevelpage")
+               for i := 0; i < b.N; i++ {
+                       runOne(toplevelpage)
+               }
+       })
+
+       b.Run("nestedpage", func(b *testing.B) {
+               toplevelpage, _ := bb.H.Sites[0].GetPage("/a/b/c/nested")
+               for i := 0; i < b.N; i++ {
+                       runOne(toplevelpage)
+               }
+       })
+}