testing: Revise usage of b.N and b.Loop() in benchmarks

diff --git a/common/hreflect/helpers_test.go b/common/hreflect/helpers_test.go
index bd578650d330a95ed4d631cc8b0866cf798db4b8..fb5a06d27f5d64dd55692c2a2f05d1097a8ea943 100644 (file)
--- a/common/hreflect/helpers_test.go
+++ b/common/hreflect/helpers_test.go
@@ -152,17 +152,18 @@ func TestIndirectInterface(t *testing.T) {
 }
 
 func BenchmarkIsContextType(b *testing.B) {
+       const size = 1000
        type k string
        b.Run("value", func(b *testing.B) {
                ctx := context.Background()
-               ctxs := make([]reflect.Type, b.N)
-               for i := 0; b.Loop(); i++ {
+               ctxs := make([]reflect.Type, size)
+               for i := range size {
                        ctxs[i] = reflect.TypeOf(context.WithValue(ctx, k("key"), i))
                }
 
-               b.ResetTimer()
-               for i := 0; i < b.N; i++ {
-                       if !IsContextType(ctxs[i]) {
+               for i := 0; b.Loop(); i++ {
+                       idx := i % size
+                       if !IsContextType(ctxs[idx]) {
                                b.Fatal("not context")
                        }
                }

diff --git a/common/hstrings/strings_test.go b/common/hstrings/strings_test.go
index 9eb3b04cb1d1148277c4849f5d53e5458b583ef6..3150ada075568e5b9d396a11737f58de976265ad 100644 (file)
--- a/common/hstrings/strings_test.go
+++ b/common/hstrings/strings_test.go
@@ -71,6 +71,7 @@ func TestUniqueStringsSorted(t *testing.T) {
        c.Assert(UniqueStringsSorted(nil), qt.IsNil)
 }
 
+// Note that these cannot use b.Loop() because of golang/go#27217.
 func BenchmarkUniqueStrings(b *testing.B) {
        input := []string{"a", "b", "d", "e", "d", "h", "a", "i"}
 
@@ -84,15 +85,14 @@ func BenchmarkUniqueStrings(b *testing.B) {
        })
 
        b.Run("Reuse slice", func(b *testing.B) {
-               b.StopTimer()
                inputs := make([][]string, b.N)
-               for i := 0; b.Loop(); i++ {
+               for i := 0; i < b.N; i++ {
                        inputc := make([]string, len(input))
                        copy(inputc, input)
                        inputs[i] = inputc
                }
-               b.StartTimer()
-               for i := 0; b.Loop(); i++ {
+               b.ResetTimer()
+               for i := 0; i < b.N; i++ {
                        inputc := inputs[i]
 
                        result := UniqueStringsReuse(inputc)
@@ -103,14 +103,13 @@ func BenchmarkUniqueStrings(b *testing.B) {
        })
 
        b.Run("Reuse slice sorted", func(b *testing.B) {
-               b.StopTimer()
                inputs := make([][]string, b.N)
                for i := 0; i < b.N; i++ {
                        inputc := make([]string, len(input))
                        copy(inputc, input)
                        inputs[i] = inputc
                }
-               b.StartTimer()
+               b.ResetTimer()
                for i := 0; i < b.N; i++ {
                        inputc := inputs[i]
 

diff --git a/compare/compare_strings_test.go b/compare/compare_strings_test.go
index 1a5bb0b1a115486013a2e61025856d0f7e468243..0ccfab243684d346f5c51a60475f528ef0f03391 100644 (file)
--- a/compare/compare_strings_test.go
+++ b/compare/compare_strings_test.go
@@ -63,6 +63,7 @@ func TestLexicographicSort(t *testing.T) {
        c.Assert(s, qt.DeepEquals, []string{"A", "b", "Ba", "ba", "ba", "Bz"})
 }
 
+// // Note that this cannot use b.Loop() because of golang/go#27217.
 func BenchmarkStringSort(b *testing.B) {
        prototype := []string{"b", "Bz", "zz", "ba", "αβδ αβδ αβδ", "A", "Ba", "ba", "nnnnasdfnnn", "AAgæåz", "αβδC"}
        b.Run("LessStrings", func(b *testing.B) {

diff --git a/hugolib/cascade_test.go b/hugolib/cascade_test.go
index b592ee0f85333c2ec452da685cd8871fe14a85a2..01db67c3b2db1fde9bdd0b8cccaef6c24b4f347d 100644 (file)
--- a/hugolib/cascade_test.go
+++ b/hugolib/cascade_test.go
@@ -34,15 +34,11 @@ func BenchmarkCascade(b *testing.B) {
                langs := allLangs[0:i]
                b.Run(fmt.Sprintf("langs-%d", len(langs)), func(b *testing.B) {
                        c := qt.New(b)
-                       b.StopTimer()
-                       builders := make([]*sitesBuilder, b.N)
-                       for i := 0; b.Loop(); i++ {
-                               builders[i] = newCascadeTestBuilder(b, langs)
-                       }
-                       b.StartTimer()
 
-                       for i := 0; b.Loop(); i++ {
-                               builder := builders[i]
+                       for b.Loop() {
+                               b.StopTimer()
+                               builder := newCascadeTestBuilder(b, langs)
+                               b.StartTimer()
                                err := builder.BuildE(BuildCfg{})
                                c.Assert(err, qt.IsNil)
                                first := builder.H.Sites[0]
@@ -75,16 +71,13 @@ kind = '{section,term}'
                        T:           b,
                        TxtarString: files,
                }
-               builders := make([]*IntegrationTestBuilder, b.N)
-
-               for i := range builders {
-                       builders[i] = NewIntegrationTestBuilder(cfg)
-               }
-
                b.ResetTimer()
 
-               for i := 0; i < b.N; i++ {
-                       builders[i].Build()
+               for b.Loop() {
+                       b.StopTimer()
+                       builder := NewIntegrationTestBuilder(cfg)
+                       b.StartTimer()
+                       builder.Build()
                }
        })
 }

diff --git a/hugolib/hugo_smoke_test.go b/hugolib/hugo_smoke_test.go
index f7c0562ed41fdaf6d67641518ad9372a4333334c..a32d01d443c5efb6874e6992298955e1a8fb74bb 100644 (file)
--- a/hugolib/hugo_smoke_test.go
+++ b/hugolib/hugo_smoke_test.go
@@ -743,14 +743,12 @@ func BenchmarkBaseline(b *testing.B) {
                T:           b,
                TxtarString: benchmarkBaselineFiles(false),
        }
-       builders := make([]*IntegrationTestBuilder, b.N)
 
-       for i := range builders {
-               builders[i] = NewIntegrationTestBuilder(cfg)
-       }
-
-       for i := 0; i < b.N; i++ {
-               builders[i].Build()
+       for b.Loop() {
+               b.StopTimer()
+               builder := NewIntegrationTestBuilder(cfg)
+               b.StartTimer()
+               builder.Build()
        }
 }
 

diff --git a/hugolib/page_permalink_test.go b/hugolib/page_permalink_test.go
index 03fe62cecc32e205d773de6d874538422c5a2bf0..52d1c9931ba0c4239cb76812f138f877995d734e 100644 (file)
--- a/hugolib/page_permalink_test.go
+++ b/hugolib/page_permalink_test.go
@@ -64,7 +64,6 @@ func TestPermalink(t *testing.T) {
        }
 
        for i, test := range tests {
-               test := test
                t.Run(fmt.Sprintf("%s-%d", test.file, i), func(t *testing.T) {
                        t.Parallel()
                        c := qt.New(t)

diff --git a/hugolib/pagecollections_test.go b/hugolib/pagecollections_test.go
index 94ae175b52f72caf9a8914ba16d525bbe599d0d1..74e5a7194089e43467f3a6a4c621219d9ca895f6 100644 (file)
--- a/hugolib/pagecollections_test.go
+++ b/hugolib/pagecollections_test.go
@@ -37,17 +37,16 @@ categories:
 `
 
 func BenchmarkGetPage(b *testing.B) {
-       var (
-               cfg, fs = newTestCfg()
-               r       = rand.New(rand.NewSource(time.Now().UnixNano()))
-       )
+       cfg, fs := newTestCfg()
 
        configs, err := loadTestConfigFromProvider(cfg)
        if err != nil {
                b.Fatal(err)
        }
 
-       for i := range 10 {
+       const size = 10
+
+       for i := range size {
                for j := range 100 {
                        writeSource(b, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", j)), "CONTENT")
                }
@@ -55,19 +54,18 @@ func BenchmarkGetPage(b *testing.B) {
 
        s := buildSingleSite(b, deps.DepsCfg{Fs: fs, Configs: configs}, BuildCfg{SkipRender: true})
 
-       pagePaths := make([]string, b.N)
-
-       for i := 0; b.Loop(); i++ {
-               pagePaths[i] = fmt.Sprintf("sect%d", r.Intn(10))
+       pagePaths := make([]string, size)
+       for i := range size {
+               pagePaths[i] = fmt.Sprintf("sect%d", i)
        }
 
-       for i := 0; i < b.N; i++ {
+       for i := 0; b.Loop(); i++ {
                home, _ := s.getPage(nil, "/")
                if home == nil {
                        b.Fatal("Home is nil")
                }
 
-               p, _ := s.getPage(nil, pagePaths[i])
+               p, _ := s.getPage(nil, pagePaths[i%size])
                if p == nil {
                        b.Fatal("Section is nil")
                }
@@ -113,20 +111,21 @@ func TestBenchmarkGetPageRegular(t *testing.T) {
 
 func BenchmarkGetPageRegular(b *testing.B) {
        r := rand.New(rand.NewSource(time.Now().UnixNano()))
+       const size = 100
 
        b.Run("From root", func(b *testing.B) {
                s := createGetPageRegularBenchmarkSite(b)
                c := qt.New(b)
 
-               pagePaths := make([]string, b.N)
+               pagePaths := make([]string, size)
 
-               for i := 0; i < b.N; i++ {
-                       pagePaths[i] = path.Join(fmt.Sprintf("/sect%d", r.Intn(10)), fmt.Sprintf("page%d.md", r.Intn(100)))
+               for i := range size {
+                       pagePaths[i] = path.Join(fmt.Sprintf("/sect%d", r.Intn(10)), fmt.Sprintf("page%d.md", i))
                }
 
                b.ResetTimer()
-               for i := 0; i < b.N; i++ {
-                       page, _ := s.getPage(nil, pagePaths[i])
+               for i := 0; b.Loop(); i++ {
+                       page, _ := s.getPage(nil, pagePaths[i%size])
                        c.Assert(page, qt.Not(qt.IsNil))
                }
        })
@@ -136,17 +135,15 @@ func BenchmarkGetPageRegular(b *testing.B) {
                c := qt.New(b)
                allPages := s.RegularPages()
 
-               pagePaths := make([]string, b.N)
-               pages := make([]page.Page, b.N)
+               pagePaths := make([]string, size)
+               pages := allPages[:size]
 
-               for i := 0; i < b.N; i++ {
-                       pagePaths[i] = fmt.Sprintf("page%d.md", r.Intn(100))
-                       pages[i] = allPages[r.Intn(len(allPages)/3)]
+               for i := range size {
+                       pagePaths[i] = fmt.Sprintf("page%d.md", i)
                }
 
-               b.ResetTimer()
-               for i := 0; i < b.N; i++ {
-                       page, _ := s.getPage(pages[i], pagePaths[i])
+               for i := 0; b.Loop(); i++ {
+                       page, _ := s.getPage(pages[i%size], pagePaths[i%size])
                        c.Assert(page, qt.Not(qt.IsNil))
                }
        })

diff --git a/hugolib/rebuild_test.go b/hugolib/rebuild_test.go
index 07317ae1c630f722fdc2f659e09b2f33ce813ccf..afb2d7909c0e8f75db9590e990cbf30b70571ecd 100644 (file)
--- a/hugolib/rebuild_test.go
+++ b/hugolib/rebuild_test.go
@@ -1787,22 +1787,15 @@ func BenchmarkRebuildContentFileChange(b *testing.B) {
                T:           b,
                TxtarString: files,
                Running:     true,
-               // Verbose:     true,
-               // LogLevel: logg.LevelInfo,
        }
-       builders := make([]*IntegrationTestBuilder, b.N)
 
-       for i := range builders {
-               builders[i] = NewIntegrationTestBuilder(cfg)
-               builders[i].Build()
-       }
-
-       for i := 0; i < b.N; i++ {
-               bb := builders[i]
+       for b.Loop() {
+               b.StopTimer()
+               bb := NewIntegrationTestBuilder(cfg).Build()
+               b.StartTimer()
                bb.EditFileReplaceFunc("content/mysect/p123/index.md", func(s string) string {
                        return s + "... Edited"
                }).Build()
-               // fmt.Println(bb.LogString())
        }
 }
 

diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go
index 560e0b4c0c2151001b47f210331d8314cb975043..0d6e72c1039d96bc8926b7d7251d6317c20afac1 100644 (file)
--- a/hugolib/shortcode_test.go
+++ b/hugolib/shortcode_test.go
@@ -242,6 +242,7 @@ CSV: {{< myShort >}}
        )
 }
 
+// Note that this cannot use b.Loop() because of golang/go#27217.
 func BenchmarkReplaceShortcodeTokens(b *testing.B) {
        type input struct {
                in           []byte
@@ -263,7 +264,7 @@ func BenchmarkReplaceShortcodeTokens(b *testing.B) {
 
        cnt := 0
        in := make([]input, b.N*len(data))
-       for b.Loop() {
+       for i := 0; i < b.N; i++ {
                for _, this := range data {
                        replacements := make(map[string]shortcodeRenderer)
                        for k, v := range this.replacements {
@@ -279,7 +280,8 @@ func BenchmarkReplaceShortcodeTokens(b *testing.B) {
 
        cnt = 0
        ctx := context.Background()
-       for i := 0; b.Loop(); i++ {
+       b.ResetTimer()
+       for i := 0; i < b.N; i++ {
                for j := range data {
                        currIn := in[cnt]
                        cnt++
@@ -335,14 +337,12 @@ title: "Markdown Shortcode"
                T:           b,
                TxtarString: files,
        }
-       builders := make([]*IntegrationTestBuilder, b.N)
 
-       for i := range builders {
-               builders[i] = NewIntegrationTestBuilder(cfg)
-       }
-
-       for i := 0; i < b.N; i++ {
-               builders[i].Build()
+       for b.Loop() {
+               b.StopTimer()
+               builder := NewIntegrationTestBuilder(cfg)
+               b.StartTimer()
+               builder.Build()
        }
 }
 

diff --git a/hugolib/site_benchmark_new_test.go b/hugolib/site_benchmark_new_test.go
deleted file mode 100644 (file)
index 85fff47..0000000
--- a/hugolib/site_benchmark_new_test.go
+++ /dev/null
@@ -1,560 +0,0 @@
-// Copyright 2019 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hugolib
-
-import (
-       "fmt"
-       "math/rand"
-       "path"
-       "path/filepath"
-       "strconv"
-       "strings"
-       "testing"
-
-       "github.com/gohugoio/hugo/resources/page"
-
-       qt "github.com/frankban/quicktest"
-)
-
-type siteBenchmarkTestcase struct {
-       name   string
-       create func(t testing.TB) *sitesBuilder
-       check  func(s *sitesBuilder)
-}
-
-func getBenchmarkSiteDeepContent(b testing.TB) *sitesBuilder {
-       pageContent := func(size int) string {
-               return getBenchmarkTestDataPageContentForMarkdown(size, false, "", benchmarkMarkdownSnippets)
-       }
-
-       sb := newTestSitesBuilder(b).WithConfigFile("toml", `
-baseURL = "https://example.com"
-
-[languages]
-[languages.en]
-weight=1
-contentDir="content/en"
-[languages.fr]
-weight=2
-contentDir="content/fr"
-[languages.no]
-weight=3
-contentDir="content/no"
-[languages.sv]
-weight=4
-contentDir="content/sv"
-                       
-`)
-
-       createContent := func(dir, name string) {
-               sb.WithContent(filepath.Join("content", dir, name), pageContent(1))
-       }
-
-       createBundledFiles := func(dir string) {
-               sb.WithContent(filepath.Join("content", dir, "data.json"), `{ "hello": "world" }`)
-               for i := 1; i <= 3; i++ {
-                       sb.WithContent(filepath.Join("content", dir, fmt.Sprintf("page%d.md", i)), pageContent(1))
-               }
-       }
-
-       for _, lang := range []string{"en", "fr", "no", "sv"} {
-               for level := 1; level <= 5; level++ {
-                       sectionDir := path.Join(lang, strings.Repeat("section/", level))
-                       createContent(sectionDir, "_index.md")
-                       createBundledFiles(sectionDir)
-                       for i := 1; i <= 3; i++ {
-                               leafBundleDir := path.Join(sectionDir, fmt.Sprintf("bundle%d", i))
-                               createContent(leafBundleDir, "index.md")
-                               createBundledFiles(path.Join(leafBundleDir, "assets1"))
-                               createBundledFiles(path.Join(leafBundleDir, "assets1", "assets2"))
-                       }
-               }
-       }
-
-       return sb
-}
-
-func getBenchmarkTestDataPageContentForMarkdown(size int, toml bool, category, markdown string) string {
-       base := `---
-title: "My Page"
-%s
----
-
-My page content.
-`
-       if toml {
-               base = `+++
-title="My Page"
-%s
-+++
-
-My page content.
-`
-       }
-
-       var categoryKey string
-       if category != "" {
-               categoryKey = fmt.Sprintf("categories: [%s]", category)
-               if toml {
-                       categoryKey = fmt.Sprintf("categories=[%s]", category)
-               }
-       }
-       base = fmt.Sprintf(base, categoryKey)
-
-       return base + strings.Repeat(markdown, size)
-}
-
-const benchmarkMarkdownSnippets = `
-
-## Links
-
-
-This is [an example](http://example.com/ "Title") inline link.
-
-[This link](http://example.net/) has no title attribute.
-
-This is [Relative](/all-is-relative).
-
-See my [About](/about/) page for details. 
-`
-
-func getBenchmarkSiteTestCases() []siteBenchmarkTestcase {
-       pageContentWithCategory := func(size int, category string) string {
-               return getBenchmarkTestDataPageContentForMarkdown(size, false, category, benchmarkMarkdownSnippets)
-       }
-
-       pageContent := func(size int) string {
-               return getBenchmarkTestDataPageContentForMarkdown(size, false, "", benchmarkMarkdownSnippets)
-       }
-
-       config := `
-baseURL = "https://example.com"
-`
-
-       benchmarks := []siteBenchmarkTestcase{
-               {
-                       "Bundle with image", func(b testing.TB) *sitesBuilder {
-                               sb := newTestSitesBuilder(b).WithConfigFile("toml", config)
-                               sb.WithContent("content/blog/mybundle/index.md", pageContent(1))
-                               sb.WithSunset("content/blog/mybundle/sunset1.jpg")
-
-                               return sb
-                       },
-                       func(s *sitesBuilder) {
-                               s.AssertFileContent("public/blog/mybundle/index.html", "/blog/mybundle/sunset1.jpg")
-                               s.CheckExists("public/blog/mybundle/sunset1.jpg")
-                       },
-               },
-               {
-                       "Bundle with JSON file", func(b testing.TB) *sitesBuilder {
-                               sb := newTestSitesBuilder(b).WithConfigFile("toml", config)
-                               sb.WithContent("content/blog/mybundle/index.md", pageContent(1))
-                               sb.WithContent("content/blog/mybundle/mydata.json", `{ "hello": "world" }`)
-
-                               return sb
-                       },
-                       func(s *sitesBuilder) {
-                               s.AssertFileContent("public/blog/mybundle/index.html", "Resources: application/json: /blog/mybundle/mydata.json")
-                               s.CheckExists("public/blog/mybundle/mydata.json")
-                       },
-               },
-               {
-                       "Tags and categories", func(b testing.TB) *sitesBuilder {
-                               sb := newTestSitesBuilder(b).WithConfigFile("toml", `
-title = "Tags and Cats"
-baseURL = "https://example.com"
-
-`)
-
-                               const pageTemplate = `
----
-title: "Some tags and cats"
-categories: ["caGR", "cbGR"]
-tags: ["taGR", "tbGR"]
----
-
-Some content.
-                       
-`
-                               for i := 1; i <= 100; i++ {
-                                       content := strings.Replace(pageTemplate, "GR", strconv.Itoa(i/3), -1)
-                                       sb.WithContent(fmt.Sprintf("content/page%d.md", i), content)
-                               }
-
-                               return sb
-                       },
-                       func(s *sitesBuilder) {
-                               s.AssertFileContent("public/page3/index.html", "/page3/|Permalink: https://example.com/page3/")
-                               s.AssertFileContent("public/tags/ta3/index.html", "a3")
-                       },
-               },
-               {
-                       "Canonify URLs", func(b testing.TB) *sitesBuilder {
-                               sb := newTestSitesBuilder(b).WithConfigFile("toml", `
-title = "Canon"
-baseURL = "https://example.com"
-canonifyURLs = true
-
-`)
-                               for i := 1; i <= 100; i++ {
-                                       sb.WithContent(fmt.Sprintf("content/page%d.md", i), pageContent(i))
-                               }
-
-                               return sb
-                       },
-                       func(s *sitesBuilder) {
-                               s.AssertFileContent("public/page8/index.html", "https://example.com/about/")
-                       },
-               },
-
-               {
-                       "Deep content tree", func(b testing.TB) *sitesBuilder {
-                               return getBenchmarkSiteDeepContent(b)
-                       },
-                       func(s *sitesBuilder) {
-                               s.CheckExists("public/blog/mybundle/index.html")
-                               s.Assert(len(s.H.Sites), qt.Equals, 4)
-                               s.Assert(len(s.H.Sites[0].RegularPages()), qt.Equals, len(s.H.Sites[1].RegularPages()))
-                               s.Assert(len(s.H.Sites[0].RegularPages()), qt.Equals, 30)
-                       },
-               },
-               {
-                       "TOML front matter", func(b testing.TB) *sitesBuilder {
-                               sb := newTestSitesBuilder(b).WithConfigFile("toml", config)
-                               for i := 1; i <= 200; i++ {
-                                       content := getBenchmarkTestDataPageContentForMarkdown(1, true, "\"a\", \"b\", \"c\"", benchmarkMarkdownSnippets)
-                                       sb.WithContent(fmt.Sprintf("content/p%d.md", i), content)
-                               }
-
-                               return sb
-                       },
-                       func(s *sitesBuilder) {
-                       },
-               },
-               {
-                       "Many HTML templates", func(b testing.TB) *sitesBuilder {
-                               pageTemplateTemplate := `
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8">
-    <title>{{ if not .IsPage }}{{ .Title }}{{ else }}{{ printf "Site: %s" site.Title }}{{ end }}</title>
-    <style>
-     body {
-       margin: 3rem;
-     }
-    </style>
-  </head>
-  <body>
-    <div class="page">{{ .Content }}</div>
-    <ul>
-    {{ with .Pages }}
-    {{ range . }}
-    <li><a href="{{ .RelPermalink }}">{{ .LinkTitle }} {{ if not .IsNode }} (Page){{ end }}</a></li>
-    {{ end }}
-    {{ end }}
-    </ul>
-  </body>
-</html>
-`
-
-                               sb := newTestSitesBuilder(b).WithConfigFile("toml", `
-baseURL = "https://example.com"
-
-[languages]
-[languages.en]
-weight=1
-contentDir="content/en"
-[languages.fr]
-weight=2
-contentDir="content/fr"
-[languages.no]
-weight=3
-contentDir="content/no"
-[languages.sv]
-weight=4
-contentDir="content/sv"
-                       
-`)
-
-                               createContent := func(dir, name string) {
-                                       sb.WithContent(filepath.Join("content", dir, name), pageContent(1))
-                               }
-
-                               for _, lang := range []string{"en", "fr", "no", "sv"} {
-                                       sb.WithTemplatesAdded(fmt.Sprintf("_default/single.%s.html", lang), pageTemplateTemplate)
-                                       sb.WithTemplatesAdded(fmt.Sprintf("_default/list.%s.html", lang), pageTemplateTemplate)
-
-                                       for level := 1; level <= 5; level++ {
-                                               sectionDir := path.Join(lang, strings.Repeat("section/", level))
-                                               createContent(sectionDir, "_index.md")
-                                               for i := 1; i <= 3; i++ {
-                                                       leafBundleDir := path.Join(sectionDir, fmt.Sprintf("bundle%d", i))
-                                                       createContent(leafBundleDir, "index.md")
-                                               }
-                                       }
-                               }
-
-                               return sb
-                       },
-                       func(s *sitesBuilder) {
-                               s.CheckExists("public/blog/mybundle/index.html")
-                               s.Assert(len(s.H.Sites), qt.Equals, 4)
-                               s.Assert(len(s.H.Sites[0].RegularPages()), qt.Equals, len(s.H.Sites[1].RegularPages()))
-                               s.Assert(len(s.H.Sites[0].RegularPages()), qt.Equals, 15)
-                       },
-               },
-               {
-                       "Page collections", func(b testing.TB) *sitesBuilder {
-                               pageTemplateTemplate := `
-{{ if .IsNode }}
-{{ len .Paginator.Pages }}
-{{ end }}
-{{ len .Sections }}
-{{ len .Pages }}
-{{ len .RegularPages }}
-{{ len .Resources }}
-{{ len site.RegularPages }}
-{{ len site.Pages }}
-{{ with .NextInSection }}Next in section: {{ .RelPermalink }}{{ end }}
-{{ with .PrevInSection }}Prev in section: {{ .RelPermalink }}{{ end }}
-{{ with .Next }}Next: {{ .RelPermalink }}{{ end }}
-{{ with .Prev }}Prev: {{ .RelPermalink }}{{ end }}
-`
-
-                               sb := newTestSitesBuilder(b).WithConfigFile("toml", `
-baseURL = "https://example.com"
-
-[languages]
-[languages.en]
-weight=1
-contentDir="content/en"
-[languages.fr]
-weight=2
-contentDir="content/fr"
-[languages.no]
-weight=3
-contentDir="content/no"
-[languages.sv]
-weight=4
-contentDir="content/sv"
-                       
-`)
-
-                               sb.WithTemplates("index.html", pageTemplateTemplate)
-                               sb.WithTemplates("_default/single.html", pageTemplateTemplate)
-                               sb.WithTemplates("_default/list.html", pageTemplateTemplate)
-
-                               r := rand.New(rand.NewSource(99))
-
-                               createContent := func(dir, name string) {
-                                       var content string
-                                       if strings.Contains(name, "_index") {
-                                               content = pageContent(1)
-                                       } else {
-                                               content = pageContentWithCategory(1, fmt.Sprintf("category%d", r.Intn(5)+1))
-                                       }
-
-                                       sb.WithContent(filepath.Join("content", dir, name), content)
-                               }
-
-                               createBundledFiles := func(dir string) {
-                                       sb.WithContent(filepath.Join("content", dir, "data.json"), `{ "hello": "world" }`)
-                                       for i := 1; i <= 3; i++ {
-                                               sb.WithContent(filepath.Join("content", dir, fmt.Sprintf("page%d.md", i)), pageContent(1))
-                                       }
-                               }
-
-                               for _, lang := range []string{"en", "fr", "no", "sv"} {
-                                       for level := 1; level <= r.Intn(5)+1; level++ {
-                                               sectionDir := path.Join(lang, strings.Repeat("section/", level))
-                                               createContent(sectionDir, "_index.md")
-                                               createBundledFiles(sectionDir)
-                                               for i := 1; i <= r.Intn(20)+1; i++ {
-                                                       leafBundleDir := path.Join(sectionDir, fmt.Sprintf("bundle%d", i))
-                                                       createContent(leafBundleDir, "index.md")
-                                                       createBundledFiles(path.Join(leafBundleDir, "assets1"))
-                                                       createBundledFiles(path.Join(leafBundleDir, "assets1", "assets2"))
-                                               }
-                                       }
-                               }
-
-                               return sb
-                       },
-                       func(s *sitesBuilder) {
-                               s.CheckExists("public/blog/mybundle/index.html")
-                               s.Assert(len(s.H.Sites), qt.Equals, 4)
-                               s.Assert(len(s.H.Sites[0].RegularPages()), qt.Equals, 26)
-                       },
-               },
-               {
-                       "List terms", func(b testing.TB) *sitesBuilder {
-                               pageTemplateTemplate := `
-<ul>
-    {{ range (.GetTerms "categories") }}
-        <li><a href="{{ .Permalink }}">{{ .LinkTitle }}</a></li>
-   {{ end }}
-</ul>
-`
-
-                               sb := newTestSitesBuilder(b).WithConfigFile("toml", `
-baseURL = "https://example.com"
-`)
-
-                               sb.WithTemplates("_default/single.html", pageTemplateTemplate)
-                               sb.WithTemplates("_default/list.html", "List")
-
-                               r := rand.New(rand.NewSource(99))
-
-                               createContent := func(dir, name string) {
-                                       var content string
-                                       if strings.Contains(name, "_index") {
-                                               // Empty
-                                       } else {
-                                               content = pageContentWithCategory(1, fmt.Sprintf("category%d", r.Intn(5)+1))
-                                       }
-                                       sb.WithContent(filepath.Join("content", dir, name), content)
-                               }
-
-                               for level := 1; level <= r.Intn(5)+1; level++ {
-                                       sectionDir := path.Join(strings.Repeat("section/", level))
-                                       createContent(sectionDir, "_index.md")
-                                       for i := 1; i <= r.Intn(33); i++ {
-                                               leafBundleDir := path.Join(sectionDir, fmt.Sprintf("bundle%d", i))
-                                               createContent(leafBundleDir, "index.md")
-                                       }
-                               }
-
-                               return sb
-                       },
-                       func(s *sitesBuilder) {
-                               s.AssertFileContent("public/section/bundle8/index.html", `<a href="https://example.com/categories/category1/">`)
-                               s.Assert(len(s.H.Sites), qt.Equals, 1)
-                               s.Assert(len(s.H.Sites[0].RegularPages()), qt.Equals, 35)
-                       },
-               },
-       }
-
-       return benchmarks
-}
-
-// Run the benchmarks below as tests. Mostly useful when adding new benchmark
-// variants.
-func TestBenchmarkSite(b *testing.T) {
-       benchmarks := getBenchmarkSiteTestCases()
-       for _, bm := range benchmarks {
-               if bm.name != "Deep content tree" {
-                       continue
-               }
-               b.Run(bm.name, func(b *testing.T) {
-                       s := bm.create(b)
-
-                       err := s.BuildE(BuildCfg{})
-                       if err != nil {
-                               b.Fatal(err)
-                       }
-                       bm.check(s)
-               })
-       }
-}
-
-func TestBenchmarkSiteDeepContentEdit(t *testing.T) {
-       b := getBenchmarkSiteDeepContent(t).Running()
-       b.Build(BuildCfg{})
-
-       p := b.H.Sites[0].RegularPages()[12]
-
-       b.EditFiles(p.File().Filename(), fmt.Sprintf(`---
-title: %s
----
-
-Edited!!`, p.Title()))
-
-       counters := &buildCounters{}
-
-       b.Build(BuildCfg{testCounters: counters})
-
-       // We currently rebuild all the language versions of the same content file.
-       // We could probably optimize that case, but it's not trivial.
-       b.Assert(int(counters.contentRenderCounter.Load()), qt.Equals, 4)
-       b.AssertFileContent("public"+p.RelPermalink()+"index.html", "Edited!!")
-}
-
-func BenchmarkSiteNew(b *testing.B) {
-       rnd := rand.New(rand.NewSource(32))
-       benchmarks := getBenchmarkSiteTestCases()
-       for _, edit := range []bool{true, false} {
-               for _, bm := range benchmarks {
-                       name := bm.name
-                       if edit {
-                               name = "Edit_" + name
-                       } else {
-                               name = "Regular_" + name
-                       }
-                       b.Run(name, func(b *testing.B) {
-                               sites := make([]*sitesBuilder, b.N)
-                               for i := 0; i < b.N; i++ {
-                                       sites[i] = bm.create(b)
-                                       if edit {
-                                               sites[i].Running()
-                                       }
-                               }
-
-                               b.ResetTimer()
-                               for i := 0; b.Loop(); i++ {
-                                       if edit {
-                                               b.StopTimer()
-                                       }
-                                       s := sites[i]
-                                       err := s.BuildE(BuildCfg{})
-                                       if err != nil {
-                                               b.Fatal(err)
-                                       }
-                                       bm.check(s)
-
-                                       if edit {
-                                               if edit {
-                                                       b.StartTimer()
-                                               }
-                                               // Edit a random page in a random language.
-                                               pages := s.H.Sites[rnd.Intn(len(s.H.Sites))].Pages()
-                                               var p page.Page
-                                               count := 0
-                                               for {
-                                                       count++
-                                                       if count > 100 {
-                                                               panic("infinite loop")
-                                                       }
-                                                       p = pages[rnd.Intn(len(pages))]
-                                                       if p.File() != nil {
-                                                               break
-                                                       }
-                                               }
-
-                                               s.EditFiles(p.File().Filename(), fmt.Sprintf(`---
-title: %s
----
-
-Edited!!`, p.Title()))
-
-                                               err := s.BuildE(BuildCfg{})
-                                               if err != nil {
-                                                       b.Fatal(err)
-                                               }
-                                       }
-                               }
-                       })
-               }
-       }
-}

diff --git a/hugolib/sitesmatrix/sitematrix_integration_test.go b/hugolib/sitesmatrix/sitematrix_integration_test.go
index 9408a701d82b40ff02d37b4d7cd215fedc63d1d9..bfe234db0aa6a492fee54ca0fca71f921abe1b97 100644 (file)
--- a/hugolib/sitesmatrix/sitematrix_integration_test.go
+++ b/hugolib/sitesmatrix/sitematrix_integration_test.go
@@ -1124,13 +1124,11 @@ func BenchmarkSitesMatrixContent(b *testing.B) {
        for _, numPages := range []int{10, 100} {
                for _, multipleDimensions := range []bool{false, true} {
                        b.Run(fmt.Sprintf("n%d/md%t", numPages, multipleDimensions), func(b *testing.B) {
-                               builders := make([]*hugolib.IntegrationTestBuilder, b.N)
-                               for i := 0; i < b.N; i++ {
-                                       builders[i] = newSitesMatrixContentBenchmarkBuilder(b, numPages, true, true)
-                               }
-                               b.ResetTimer()
-                               for i := 0; i < b.N; i++ {
-                                       builders[i].Build()
+                               for b.Loop() {
+                                       b.StopTimer()
+                                       bb := newSitesMatrixContentBenchmarkBuilder(b, numPages, true, multipleDimensions)
+                                       b.StartTimer()
+                                       bb.Build()
                                }
                        })
                }

diff --git a/hugolib/taxonomy_test.go b/hugolib/taxonomy_test.go
index 4b30108593edd3e255466e261f1739b68d45a965..108d327bb057681003914fab6c809f7e6f05a6d4 100644 (file)
--- a/hugolib/taxonomy_test.go
+++ b/hugolib/taxonomy_test.go
@@ -830,7 +830,9 @@ title: p1
 }
 
 func BenchmarkTaxonomiesGetTerms(b *testing.B) {
-       createBuilders := func(b *testing.B, numPages int) []*IntegrationTestBuilder {
+       createBuilder := func(b *testing.B, numPages int) *IntegrationTestBuilder {
+               b.StopTimer()
+
                files := `
 -- hugo.toml --
 baseURL = "https://example.com"
@@ -859,26 +861,23 @@ GetTerms.tags: {{ range .GetTerms "tags" }}{{ .Title }}|{{ end }}
                        tags := tagsVariants[i%len(tagsVariants)]
                        files += fmt.Sprintf("\n-- content/posts/p%d.md --\n---\n%s\n---", i+1, tags)
                }
+
                cfg := IntegrationTestConfig{
                        T:           b,
                        TxtarString: files,
                }
-               builders := make([]*IntegrationTestBuilder, b.N)
 
-               for i := range builders {
-                       builders[i] = NewIntegrationTestBuilder(cfg)
-               }
+               bb := NewIntegrationTestBuilder(cfg)
 
-               b.ResetTimer()
+               b.StartTimer()
 
-               return builders
+               return bb
        }
 
        for _, numPages := range []int{100, 1000, 10000, 20000} {
                b.Run(fmt.Sprintf("pages_%d", numPages), func(b *testing.B) {
-                       builders := createBuilders(b, numPages)
                        for i := 0; b.Loop(); i++ {
-                               builders[i].Build()
+                               createBuilder(b, numPages).Build()
                        }
                })
        }

diff --git a/identity/identity_test.go b/identity/identity_test.go
index 3b15b61b9b277e6dfc1a015e23d9a80818bca4c9..20617b8766b1f518fdbe40f91b18acff96d1d30d 100644 (file)
--- a/identity/identity_test.go
+++ b/identity/identity_test.go
@@ -41,25 +41,14 @@ func BenchmarkIdentityManager(b *testing.B) {
                }
        })
 
-       b.Run("Add unique", func(b *testing.B) {
-               ids := createIds(b.N)
+       b.Run("Add many", func(b *testing.B) {
+               const size = 1000
+               ids := createIds(size)
                im := identity.NewManager()
 
                b.ResetTimer()
-               for i := 0; i < b.N; i++ {
-                       im.AddIdentity(ids[i])
-               }
-
-               b.StopTimer()
-       })
-
-       b.Run("Add duplicates", func(b *testing.B) {
-               id := &testIdentity{base: "a", name: "b"}
-               im := identity.NewManager()
-
-               b.ResetTimer()
-               for b.Loop() {
-                       im.AddIdentity(id)
+               for i := 0; b.Loop(); i++ {
+                       im.AddIdentity(ids[i%size])
                }
 
                b.StopTimer()

diff --git a/markup/goldmark/goldmark_integration_test.go b/markup/goldmark/goldmark_integration_test.go
index 8ea9dbde2462e6e1ae3453737dcb8d7dfa7e960a..415d7d0b1d8565202a0992638f24133429b04825 100644 (file)
--- a/markup/goldmark/goldmark_integration_test.go
+++ b/markup/goldmark/goldmark_integration_test.go
@@ -278,14 +278,12 @@ D.
                T:           b,
                TxtarString: files,
        }
-       builders := make([]*hugolib.IntegrationTestBuilder, b.N)
 
-       for i := range builders {
-               builders[i] = hugolib.NewIntegrationTestBuilder(cfg)
-       }
-
-       for i := 0; i < b.N; i++ {
-               builders[i].Build()
+       for b.Loop() {
+               b.StopTimer()
+               bb := hugolib.NewIntegrationTestBuilder(cfg)
+               b.StartTimer()
+               bb.Build()
        }
 }
 
@@ -335,16 +333,12 @@ FENCE
                        T:           b,
                        TxtarString: files,
                }
-               builders := make([]*hugolib.IntegrationTestBuilder, b.N)
-
-               for i := range builders {
-                       builders[i] = hugolib.NewIntegrationTestBuilder(cfg)
-               }
-
-               b.ResetTimer()
 
-               for i := 0; i < b.N; i++ {
-                       builders[i].Build()
+               for b.Loop() {
+                       b.StopTimer()
+                       bb := hugolib.NewIntegrationTestBuilder(cfg)
+                       b.StartTimer()
+                       bb.Build()
                }
        }
 

diff --git a/markup/tableofcontents/tableofcontents_test.go b/markup/tableofcontents/tableofcontents_test.go
index 4b9b460a67013fa802ff37c79b805f87e702ebba..db50c91ab4f0ee4b07324acb0e794f0f50d95de3 100644 (file)
--- a/markup/tableofcontents/tableofcontents_test.go
+++ b/markup/tableofcontents/tableofcontents_test.go
@@ -193,6 +193,7 @@ func TestTocMisc(t *testing.T) {
        })
 }
 
+// Note that some of these cannot use b.Loop() because of golang/go#27217.
 func BenchmarkToc(b *testing.B) {
        newTocs := func(n int) []*Fragments {
                var tocs []*Fragments
@@ -204,22 +205,21 @@ func BenchmarkToc(b *testing.B) {
 
        b.Run("Build", func(b *testing.B) {
                var builders []Builder
-               for b.Loop() {
+               for i := 0; i < b.N; i++ {
                        builders = append(builders, newTestTocBuilder())
                }
                b.ResetTimer()
-
-               for i := 0; b.Loop(); i++ {
+               for i := 0; i < b.N; i++ {
                        b := builders[i]
                        b.Build()
                }
        })
 
        b.Run("ToHTML", func(b *testing.B) {
-               tocs := newTocs(b.N)
-               b.ResetTimer()
-               for i := 0; i < b.N; i++ {
-                       toc := tocs[i]
+               const size = 1000
+               tocs := newTocs(size)
+               for i := 0; b.Loop(); i++ {
+                       toc := tocs[i%size]
                        toc.ToHTML(1, -1, false)
                }
        })

diff --git a/related/related_integration_test.go b/related/related_integration_test.go
index 5d558368e55dc665de1a5c51ab5d984d61edd40e..de53b59d3a15540c004ad150fde54d9f4ad80d35 100644 (file)
--- a/related/related_integration_test.go
+++ b/related/related_integration_test.go
@@ -175,13 +175,11 @@ keywords: ['k%d']
                T:           b,
                TxtarString: files,
        }
-       builders := make([]*hugolib.IntegrationTestBuilder, b.N)
 
-       for i := range builders {
-               builders[i] = hugolib.NewIntegrationTestBuilder(cfg)
-       }
-
-       for i := 0; i < b.N; i++ {
-               builders[i].Build()
+       for b.Loop() {
+               b.StopTimer()
+               bb := hugolib.NewIntegrationTestBuilder(cfg)
+               b.StartTimer()
+               bb.Build()
        }
 }

diff --git a/resources/image_test.go b/resources/image_test.go
index e852d74013494bbd0c5747daa4694ac48437568f..19d2f5a8f99d0996b549117fba56ac7b5ed47064 100644 (file)
--- a/resources/image_test.go
+++ b/resources/image_test.go
@@ -466,13 +466,9 @@ func TestImageColorsLuminance(t *testing.T) {
 }
 
 func BenchmarkImageExif(b *testing.B) {
-       getImages := func(c *qt.C, b *testing.B, fs afero.Fs) []images.ImageResource {
+       getImage := func(i int, c *qt.C, b *testing.B, fs afero.Fs) images.ImageResource {
                spec := newTestResourceSpec(specDescriptor{fs: fs, c: c})
-               imgs := make([]images.ImageResource, b.N)
-               for i := 0; i < b.N; i++ {
-                       imgs[i] = fetchResourceForSpec(spec, c, "sunset.jpg", strconv.Itoa(i)).(images.ImageResource)
-               }
-               return imgs
+               return fetchResourceForSpec(spec, c, "sunset.jpg", strconv.Itoa(i)).(images.ImageResource)
        }
 
        getAndCheckExif := func(c *qt.C, image images.ImageResource) {
@@ -482,43 +478,45 @@ func BenchmarkImageExif(b *testing.B) {
        }
 
        b.Run("Cold cache", func(b *testing.B) {
-               b.StopTimer()
                c := qt.New(b)
-               images := getImages(c, b, afero.NewMemMapFs())
-
-               b.StartTimer()
+               fs := afero.NewMemMapFs()
                for i := 0; b.Loop(); i++ {
-                       getAndCheckExif(c, images[i])
+                       b.StopTimer()
+                       image := getImage(i, c, b, fs)
+                       b.StartTimer()
+                       getAndCheckExif(c, image)
                }
        })
 
        b.Run("Cold cache, 10", func(b *testing.B) {
-               b.StopTimer()
                c := qt.New(b)
-               images := getImages(c, b, afero.NewMemMapFs())
-
-               b.StartTimer()
+               fs := afero.NewMemMapFs()
                for i := 0; b.Loop(); i++ {
+                       b.StopTimer()
+                       image := getImage(i, c, b, fs)
+                       b.StartTimer()
                        for range 10 {
-                               getAndCheckExif(c, images[i])
+                               getAndCheckExif(c, image)
                        }
                }
        })
 
        b.Run("Warm cache", func(b *testing.B) {
-               b.StopTimer()
                c := qt.New(b)
                fs := afero.NewMemMapFs()
-               images := getImages(c, b, fs)
-               for i := 0; b.Loop(); i++ {
-                       getAndCheckExif(c, images[i])
+               // Prime the cache
+               for i := 0; i < b.N; i++ {
+                       image := getImage(i, c, b, fs)
+                       getAndCheckExif(c, image)
                }
 
-               images = getImages(c, b, fs)
-
-               b.StartTimer()
-               for i := 0; b.Loop(); i++ {
-                       getAndCheckExif(c, images[i])
+               // Start the real benchmark,
+               b.ResetTimer()
+               for i := 0; i < b.N; i++ {
+                       b.StopTimer()
+                       image := getImage(i, c, b, fs)
+                       b.StartTimer()
+                       getAndCheckExif(c, image)
                }
        })
 }

diff --git a/tpl/collections/collections_test.go b/tpl/collections/collections_test.go
index 8099ea91dd1490de08a620587416d353d1861895..75623d1fdaaca27419bed4027061302f5752c729 100644 (file)
--- a/tpl/collections/collections_test.go
+++ b/tpl/collections/collections_test.go
@@ -195,7 +195,6 @@ func TestDictionary(t *testing.T) {
                {[]any{5, "b"}, false},
                {[]any{"a", "b", "c"}, false},
        } {
-               test := test
                c.Run(fmt.Sprint(i), func(c *qt.C) {
                        c.Parallel()
                        errMsg := qt.Commentf("[%d] %v", i, test.values)

diff --git a/tpl/collections/merge_test.go b/tpl/collections/merge_test.go
index f6589b6d48f60b2de1523ae533b04a9fefcf08ab..3642e9060be68c526ea1063303e6c6b540870086 100644 (file)
--- a/tpl/collections/merge_test.go
+++ b/tpl/collections/merge_test.go
@@ -137,8 +137,6 @@ func TestMerge(t *testing.T) {
                {"all nil", []any{nil, nil}, nil, true},
        } {
 
-               i := i
-
                t.Run(test.name, func(t *testing.T) {
                        t.Parallel()
                        errMsg := qt.Commentf("[%d] %v", i, test)

diff --git a/tpl/partials/partials_integration_test.go b/tpl/partials/partials_integration_test.go
index 3337a0ec777b055d021a59556edd8275c2351f57..91ee56a5f4aa2c13421ec8835c9bff71448ce6ef 100644 (file)
--- a/tpl/partials/partials_integration_test.go
+++ b/tpl/partials/partials_integration_test.go
@@ -237,14 +237,12 @@ ABCDE
                T:           b,
                TxtarString: files,
        }
-       builders := make([]*hugolib.IntegrationTestBuilder, b.N)
 
-       for i := range builders {
-               builders[i] = hugolib.NewIntegrationTestBuilder(cfg)
-       }
-
-       for i := 0; i < b.N; i++ {
-               builders[i].Build()
+       for b.Loop() {
+               b.StopTimer()
+               bb := hugolib.NewIntegrationTestBuilder(cfg)
+               b.StartTimer()
+               bb.Build()
        }
 }