hugolib: Fix possible .Content cut

There have been one report of a site with truncated `.Content` after the Hugo `0.40.1` release.

This commit fixes this so that race should not be possible anymore. It also adds a stress test with focus on content rendering and multiple output formats.

Fixes #4706

diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go
index a5bb664c0905a3e24466ac338ba13449cf4cb7d8..e3defe974413d7c4786a3e4e3031548e26e2fef1 100644 (file)
--- a/hugolib/hugo_sites.go
+++ b/hugolib/hugo_sites.go
@@ -559,14 +559,22 @@ func (h *HugoSites) setupTranslations() {
        }
 }
 
-func (s *Site) preparePagesForRender(start bool) {
+func (s *Site) preparePagesForRender(start bool) error {
        for _, p := range s.Pages {
                p.setContentInit(start)
+               if err := p.initMainOutputFormat(); err != nil {
+                       return err
+               }
        }
 
        for _, p := range s.headlessPages {
                p.setContentInit(start)
+               if err := p.initMainOutputFormat(); err != nil {
+                       return err
+               }
        }
+
+       return nil
 }
 
 // Pages returns all pages for all sites.

diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go
index 0421c79255f893b322103ff2dd712a8004c2f39b..e1e4a60568390d1609f2527fa81d36b0f966e40f 100644 (file)
--- a/hugolib/hugo_sites_build.go
+++ b/hugolib/hugo_sites_build.go
@@ -234,7 +234,9 @@ func (h *HugoSites) render(config *BuildCfg) error {
 
                                isRenderingSite := s == s2
 
-                               s2.preparePagesForRender(isRenderingSite && i == 0)
+                               if err := s2.preparePagesForRender(isRenderingSite && i == 0); err != nil {
+                                       return err
+                               }
 
                        }
 

diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go
index 87eb2cb29c8f036b42284b2bc6e984a19b9bd947..4192580bd85ef5f834f68128c1662ef979baec65 100644 (file)
--- a/hugolib/hugo_sites_build_test.go
+++ b/hugolib/hugo_sites_build_test.go
@@ -734,6 +734,119 @@ func TestChangeDefaultLanguage(t *testing.T) {
        b.AssertFileContent("public/sect/doc2/index.html", "Single", "Hello")
 }
 
+// https://github.com/gohugoio/hugo/issues/4706
+func TestContentStressTest(t *testing.T) {
+       b := newTestSitesBuilder(t)
+
+       numPages := 500
+
+       contentTempl := `
+---
+%s
+title: %q
+weight: %d
+multioutput: %t
+---
+
+# Header
+
+CONTENT
+
+The End.
+`
+
+       contentTempl = strings.Replace(contentTempl, "CONTENT", strings.Repeat(`
+       
+## Another header
+
+Some text. Some more text.
+
+`, 100), -1)
+
+       var content []string
+       defaultOutputs := `outputs: ["html", "json", "rss" ]`
+
+       for i := 1; i <= numPages; i++ {
+               outputs := defaultOutputs
+               multioutput := true
+               if i%3 == 0 {
+                       outputs = `outputs: ["json"]`
+                       multioutput = false
+               }
+               section := "s1"
+               if i%10 == 0 {
+                       section = "s2"
+               }
+               content = append(content, []string{fmt.Sprintf("%s/page%d.md", section, i), fmt.Sprintf(contentTempl, outputs, fmt.Sprintf("Title %d", i), i, multioutput)}...)
+       }
+
+       content = append(content, []string{"_index.md", fmt.Sprintf(contentTempl, defaultOutputs, fmt.Sprintf("Home %d", 0), 0, true)}...)
+       content = append(content, []string{"s1/_index.md", fmt.Sprintf(contentTempl, defaultOutputs, fmt.Sprintf("S %d", 1), 1, true)}...)
+       content = append(content, []string{"s2/_index.md", fmt.Sprintf(contentTempl, defaultOutputs, fmt.Sprintf("S %d", 2), 2, true)}...)
+
+       b.WithSimpleConfigFile()
+       b.WithTemplates("layouts/_default/single.html", `Single: {{ .Content }}`)
+       b.WithTemplates("layouts/_default/myview.html", `View: {{ len .Content }}`)
+       b.WithTemplates("layouts/_default/single.json", `Single JSON: {{ .Content }}`)
+       b.WithTemplates("layouts/_default/list.html", `
+Page: {{ .Paginator.PageNumber }}
+P: {{ path.Join .Path }}
+List: {{ len .Paginator.Pages }}|List Content: {{ len .Content }}
+{{ $shuffled :=  where .Site.RegularPages "Params.multioutput" true | shuffle }}
+{{ $first5 := $shuffled | first 5 }}
+L1: {{ len .Site.RegularPages }} L2: {{ len $first5 }}
+{{ range $i, $e := $first5 }}
+Render {{ $i }}: {{ .Render "myview" }}
+{{ end }}
+END
+`)
+
+       b.WithContent(content...)
+
+       b.CreateSites().Build(BuildCfg{})
+
+       contentMatchers := []string{"<h2 id=\"another-header\">Another header</h2>", "<h2 id=\"another-header-99\">Another header</h2>", "<p>The End.</p>"}
+
+       for i := 1; i <= numPages; i++ {
+               if i%3 != 0 {
+                       section := "s1"
+                       if i%10 == 0 {
+                               section = "s2"
+                       }
+                       checkContent(b, fmt.Sprintf("public/%s/page%d/index.html", section, i), 8343, contentMatchers...)
+               }
+       }
+
+       for i := 1; i <= numPages; i++ {
+               section := "s1"
+               if i%10 == 0 {
+                       section = "s2"
+               }
+               checkContent(b, fmt.Sprintf("public/%s/page%d/index.json", section, i), 8348, contentMatchers...)
+       }
+
+       checkContent(b, "public/s1/index.html", 184, "P: s1/_index.md\nList: 10|List Content: 8335\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8335\n\nRender 1: View: 8335\n\nRender 2: View: 8335\n\nRender 3: View: 8335\n\nRender 4: View: 8335\n\nEND\n")
+       checkContent(b, "public/s2/index.html", 184, "P: s2/_index.md\nList: 10|List Content: 8335", "Render 4: View: 8335\n\nEND")
+       checkContent(b, "public/index.html", 181, "P: _index.md\nList: 10|List Content: 8335", "4: View: 8335\n\nEND")
+
+       // Chek paginated pages
+       for i := 2; i <= 9; i++ {
+               checkContent(b, fmt.Sprintf("public/page/%d/index.html", i), 181, fmt.Sprintf("Page: %d", i), "Content: 8335\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8335", "Render 4: View: 8335\n\nEND")
+       }
+}
+
+func checkContent(s *sitesBuilder, filename string, length int, matches ...string) {
+       content := readDestination(s.T, s.Fs, filename)
+       for _, match := range matches {
+               if !strings.Contains(content, match) {
+                       s.Fatalf("No match for %q in content for %s\n%q", match, filename, content)
+               }
+       }
+       if len(content) != length {
+               s.Fatalf("got %d expected %d", len(content), length)
+       }
+}
+
 func TestTableOfContentsInShortcodes(t *testing.T) {
        t.Parallel()
 

diff --git a/hugolib/page.go b/hugolib/page.go
index 2e008ece9b42d70e6202f47a09a56515ef67af2d..6e0f2b86f37544b4eb15c3ce83d5321778c0dc27 100644 (file)
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -416,7 +416,6 @@ type pageInit struct {
        languageInit        sync.Once
        pageMenusInit       sync.Once
        pageMetaInit        sync.Once
-       pageOutputInit      sync.Once
        renderingConfigInit sync.Once
        withoutContentInit  sync.Once
 }
@@ -1178,6 +1177,24 @@ func (p *Page) subResourceTargetPathFactory(base string) string {
        return path.Join(p.relTargetPathBase, base)
 }
 
+func (p *Page) initMainOutputFormat() error {
+       if p.mainPageOutput != nil {
+               return nil
+       }
+
+       outFormat := p.outputFormats[0]
+       pageOutput, err := newPageOutput(p, false, false, outFormat)
+
+       if err != nil {
+               return fmt.Errorf("Failed to create output page for type %q for page %q: %s", outFormat.Name, p.pathOrTitle(), err)
+       }
+
+       p.mainPageOutput = pageOutput
+
+       return nil
+
+}
+
 func (p *Page) setContentInit(start bool) error {
 
        if start {
@@ -1962,12 +1979,17 @@ func (p *Page) updatePageDates() {
 
 // copy creates a copy of this page with the lazy sync.Once vars reset
 // so they will be evaluated again, for word count calculations etc.
-func (p *Page) copy() *Page {
+func (p *Page) copy(initContent bool) *Page {
        p.contentInitMu.Lock()
        c := *p
        p.contentInitMu.Unlock()
        c.pageInit = &pageInit{}
-       c.pageContentInit = &pageContentInit{}
+       if initContent {
+               if len(p.outputFormats) < 2 {
+                       panic(fmt.Sprintf("programming error: page %q should not need to rebuild content as it has only %d outputs", p.Path(), len(p.outputFormats)))
+               }
+               c.pageContentInit = &pageContentInit{}
+       }
        return &c
 }
 

diff --git a/hugolib/page_output.go b/hugolib/page_output.go
index 0a51e72f757e19f8819e5360a21a57966e73ffc8..5893bc172680e245637bc2bcb239690eec363a78 100644 (file)
--- a/hugolib/page_output.go
+++ b/hugolib/page_output.go
@@ -55,7 +55,7 @@ func (p *PageOutput) targetPath(addends ...string) (string, error) {
        return tp, nil
 }
 
-func newPageOutput(p *Page, createCopy bool, f output.Format) (*PageOutput, error) {
+func newPageOutput(p *Page, createCopy, initContent bool, f output.Format) (*PageOutput, error) {
        // TODO(bep) This is only needed for tests and we should get rid of it.
        if p.targetPathDescriptorPrototype == nil {
                if err := p.initPaths(); err != nil {
@@ -64,7 +64,7 @@ func newPageOutput(p *Page, createCopy bool, f output.Format) (*PageOutput, erro
        }
 
        if createCopy {
-               p = p.copy()
+               p = p.copy(initContent)
        }
 
        td, err := p.createTargetPathDescriptor(f)
@@ -82,8 +82,8 @@ func newPageOutput(p *Page, createCopy bool, f output.Format) (*PageOutput, erro
 
 // copy creates a copy of this PageOutput with the lazy sync.Once vars reset
 // so they will be evaluated again, for word count calculations etc.
-func (p *PageOutput) copyWithFormat(f output.Format) (*PageOutput, error) {
-       c, err := newPageOutput(p.Page, true, f)
+func (p *PageOutput) copyWithFormat(f output.Format, initContent bool) (*PageOutput, error) {
+       c, err := newPageOutput(p.Page, true, initContent, f)
        if err != nil {
                return nil, err
        }
@@ -92,7 +92,7 @@ func (p *PageOutput) copyWithFormat(f output.Format) (*PageOutput, error) {
 }
 
 func (p *PageOutput) copy() (*PageOutput, error) {
-       return p.copyWithFormat(p.outputFormat)
+       return p.copyWithFormat(p.outputFormat, false)
 }
 
 func (p *PageOutput) layouts(layouts ...string) ([]string, error) {
@@ -142,24 +142,6 @@ func (p *PageOutput) Render(layout ...string) template.HTML {
 }
 
 func (p *Page) Render(layout ...string) template.HTML {
-       p.pageOutputInit.Do(func() {
-               if p.mainPageOutput != nil {
-                       return
-               }
-               // If Render is called in a range loop, the page output isn't available.
-               // So, create one.
-               outFormat := p.outputFormats[0]
-               pageOutput, err := newPageOutput(p, true, outFormat)
-
-               if err != nil {
-                       p.s.Log.ERROR.Printf("Failed to create output page for type %q for page %q: %s", outFormat.Name, p.pathOrTitle(), err)
-                       return
-               }
-
-               p.mainPageOutput = pageOutput
-
-       })
-
        return p.mainPageOutput.Render(layout...)
 }
 

diff --git a/hugolib/pagination_test.go b/hugolib/pagination_test.go
index edfac3f3e86fcf1c3ed51d24b0b7c6a17a2e39fc..61668c3dfc82d3f8e71b0e85f88069e5f4fd674e 100644 (file)
--- a/hugolib/pagination_test.go
+++ b/hugolib/pagination_test.go
@@ -279,8 +279,8 @@ func doTestPaginator(t *testing.T, useViper bool) {
        require.NoError(t, err)
 
        pages := createTestPages(s, 12)
-       n1, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
-       n2, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
+       n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
+       n2, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
        n1.Data["Pages"] = pages
 
        var paginator1 *Pager
@@ -306,7 +306,7 @@ func doTestPaginator(t *testing.T, useViper bool) {
        require.Equal(t, paginator1, samePaginator)
 
        pp, _ := s.NewPage("test")
-       p, _ := newPageOutput(pp, false, output.HTMLFormat)
+       p, _ := newPageOutput(pp, false, false, output.HTMLFormat)
 
        _, err = p.Paginator()
        require.NotNil(t, err)
@@ -315,7 +315,7 @@ func doTestPaginator(t *testing.T, useViper bool) {
 func TestPaginatorWithNegativePaginate(t *testing.T) {
        t.Parallel()
        s := newTestSite(t, "paginate", -1)
-       n1, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
+       n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
        _, err := n1.Paginator()
        require.Error(t, err)
 }
@@ -378,8 +378,8 @@ func doTestPaginate(t *testing.T, useViper bool) {
        }
 
        pages := createTestPages(s, 6)
-       n1, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
-       n2, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
+       n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
+       n2, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
 
        var paginator1, paginator2 *Pager
 
@@ -404,7 +404,7 @@ func doTestPaginate(t *testing.T, useViper bool) {
        require.Equal(t, paginator2, paginator1.Next())
 
        pp, err := s.NewPage("test")
-       p, _ := newPageOutput(pp, false, output.HTMLFormat)
+       p, _ := newPageOutput(pp, false, false, output.HTMLFormat)
 
        _, err = p.Paginate(pages)
        require.NotNil(t, err)
@@ -413,7 +413,7 @@ func doTestPaginate(t *testing.T, useViper bool) {
 func TestInvalidOptions(t *testing.T) {
        t.Parallel()
        s := newTestSite(t)
-       n1, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
+       n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
 
        _, err := n1.Paginate(createTestPages(s, 1), 1, 2)
        require.NotNil(t, err)
@@ -431,7 +431,7 @@ func TestPaginateWithNegativePaginate(t *testing.T) {
        s, err := NewSiteForCfg(deps.DepsCfg{Cfg: cfg, Fs: fs})
        require.NoError(t, err)
 
-       n, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
+       n, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
 
        _, err = n.Paginate(createTestPages(s, 2))
        require.NotNil(t, err)
@@ -458,8 +458,8 @@ func TestPaginatePages(t *testing.T) {
 func TestPaginatorFollowedByPaginateShouldFail(t *testing.T) {
        t.Parallel()
        s := newTestSite(t, "paginate", 10)
-       n1, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
-       n2, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
+       n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
+       n2, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
 
        _, err := n1.Paginator()
        require.Nil(t, err)
@@ -475,8 +475,8 @@ func TestPaginateFollowedByDifferentPaginateShouldFail(t *testing.T) {
        t.Parallel()
        s := newTestSite(t, "paginate", 10)
 
-       n1, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
-       n2, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
+       n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
+       n2, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
 
        p1 := createTestPages(s, 2)
        p2 := createTestPages(s, 10)

diff --git a/hugolib/site_render.go b/hugolib/site_render.go
index e9d2f01833449bf31f674e0702a0a7cd489128e0..e837d9f0b314db81de782c0cd4cede953701979a 100644 (file)
--- a/hugolib/site_render.go
+++ b/hugolib/site_render.go
@@ -73,7 +73,7 @@ func headlessPagesPublisher(s *Site, wg *sync.WaitGroup) {
                        // Avoid double work.
                        continue
                }
-               pageOutput, err := newPageOutput(page, false, outFormat)
+               pageOutput, err := newPageOutput(page, false, false, outFormat)
                if err == nil {
                        page.mainPageOutput = pageOutput
                        err = pageOutput.renderResources()
@@ -92,28 +92,29 @@ func pageRenderer(s *Site, pages <-chan *Page, results chan<- error, wg *sync.Wa
 
                for i, outFormat := range page.outputFormats {
 
+                       if outFormat != page.s.rc.Format {
+                               // Will be rendered  ... later.
+                               continue
+                       }
+
                        var (
                                pageOutput *PageOutput
                                err        error
                        )
 
                        if i == 0 {
-                               pageOutput, err = newPageOutput(page, false, outFormat)
-                               page.mainPageOutput = pageOutput
+                               pageOutput = page.mainPageOutput
+                       } else {
+                               pageOutput, err = page.mainPageOutput.copyWithFormat(outFormat, true)
                        }
 
-                       if outFormat != page.s.rc.Format {
-                               // Will be rendered  ... later.
+                       if err != nil {
+                               s.Log.ERROR.Printf("Failed to create output page for type %q for page %q: %s", outFormat.Name, page, err)
                                continue
                        }
 
                        if pageOutput == nil {
-                               pageOutput, err = page.mainPageOutput.copyWithFormat(outFormat)
-                       }
-
-                       if err != nil {
-                               s.Log.ERROR.Printf("Failed to create output page for type %q for page %q: %s", outFormat.Name, page, err)
-                               continue
+                               panic("no pageOutput")
                        }
 
                        // We only need to re-publish the resources if the output format is different
@@ -291,7 +292,7 @@ func (s *Site) render404() error {
        htmlOut := output.HTMLFormat
        htmlOut.BaseName = "404"
 
-       pageOutput, err := newPageOutput(p, false, htmlOut)
+       pageOutput, err := newPageOutput(p, false, false, htmlOut)
        if err != nil {
                return err
        }
@@ -373,7 +374,7 @@ func (s *Site) renderRobotsTXT() error {
 
        rLayouts := []string{"robots.txt", "_default/robots.txt", "_internal/_default/robots.txt"}
 
-       pageOutput, err := newPageOutput(p, false, output.RobotsTxtFormat)
+       pageOutput, err := newPageOutput(p, false, false, output.RobotsTxtFormat)
        if err != nil {
                return err
        }

diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go
index 5300eee2216154a354c47ffc54004d14e094b095..62fe3860a7a7c96216373e20914098af47493846 100644 (file)
--- a/hugolib/testhelpers_test.go
+++ b/hugolib/testhelpers_test.go
@@ -39,6 +39,8 @@ type sitesBuilder struct {
        Fs  *hugofs.Fs
        T   testing.TB
 
+       logger *jww.Notepad
+
        dumper litter.Options
 
        // Aka the Hugo server mode.
@@ -88,6 +90,11 @@ func (s *sitesBuilder) Running() *sitesBuilder {
        return s
 }
 
+func (s *sitesBuilder) WithLogger(logger *jww.Notepad) *sitesBuilder {
+       s.logger = logger
+       return s
+}
+
 func (s *sitesBuilder) WithWorkingDir(dir string) *sitesBuilder {
        s.workingDir = dir
        return s
@@ -282,7 +289,7 @@ func (s *sitesBuilder) CreateSites() *sitesBuilder {
                s.Cfg = cfg
        }
 
-       sites, err := NewHugoSites(deps.DepsCfg{Fs: s.Fs, Cfg: s.Cfg, Running: s.running})
+       sites, err := NewHugoSites(deps.DepsCfg{Fs: s.Fs, Cfg: s.Cfg, Logger: s.logger, Running: s.running})
        if err != nil {
                s.Fatalf("Failed to create sites: %s", err)
        }