This commit also refines the partial rebuild logic, to make sure we do not do more work than needed.
Updates #2309
import (
"bytes"
+ "fmt"
"github.com/spf13/hugo/helpers"
"github.com/spf13/hugo/source"
func (h htmlHandler) Extensions() []string { return []string{"html", "htm"} }
func (h htmlHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
+ if p.rendered {
+ panic(fmt.Sprintf("Page %q already rendered, does not need conversion", p.BaseFileName()))
+ }
+
p.ProcessShortcodes(t)
return HandledResult{err: nil}
}
func commonConvert(p *Page, t tpl.Template) HandledResult {
+
+ if p.rendered {
+ panic(fmt.Sprintf("Page %q already rendered, does not need conversion", p.BaseFileName()))
+ }
+
p.ProcessShortcodes(t)
// TODO(bep) these page handlers need to be re-evaluated, as it is hard to
}
}
- if err := h.preRender(); err != nil {
+ if err := h.preRender(config, whatChanged{source: true, other: true}); err != nil {
return err
}
return errors.New("Rebuild does not support 'ResetState'. Use Build.")
}
+ if !config.Watching {
+ return errors.New("Rebuild called when not in watch mode")
+ }
+
h.runMode.Watching = config.Watching
firstSite := h.Sites[0]
s.resetBuildState()
}
- sourceChanged, err := firstSite.reBuild(events)
+ changed, err := firstSite.reBuild(events)
if err != nil {
return err
// Assign pages to sites per translation.
h.setupTranslations(firstSite)
- if sourceChanged {
+ if changed.source {
for _, s := range h.Sites {
if err := s.postProcess(); err != nil {
return err
}
}
- if err := h.preRender(); err != nil {
+ if err := h.preRender(config, changed); err != nil {
return err
}
// preRender performs build tasks that need to be done as late as possible.
// Shortcode handling is the main task in here.
// TODO(bep) We need to look at the whole handler-chain construct with he below in mind.
-func (h *HugoSites) preRender() error {
+func (h *HugoSites) preRender(cfg BuildCfg, changed whatChanged) error {
for _, s := range h.Sites {
if err := s.setCurrentLanguageConfig(); err != nil {
if err := s.setCurrentLanguageConfig(); err != nil {
return err
}
- renderShortcodesForSite(s)
+ s.preparePagesForRender(cfg, changed)
}
return nil
}
-func renderShortcodesForSite(s *Site) {
+func (s *Site) preparePagesForRender(cfg BuildCfg, changed whatChanged) {
pageChan := make(chan *Page)
wg := &sync.WaitGroup{}
go func(pages <-chan *Page, wg *sync.WaitGroup) {
defer wg.Done()
for p := range pages {
+
+ if !changed.other && p.rendered {
+ // No need to process it again.
+ continue
+ }
+
+ // If we got this far it means that this is either a new Page pointer
+ // or a template or similar has changed so wee need to do a rerendering
+ // of the shortcodes etc.
+
+ // Mark it as rendered
+ p.rendered = true
+
+ // If in watch mode, we need to keep the original so we can
+ // repeat this process on rebuild.
+ if cfg.Watching {
+ p.rawContentCopy = make([]byte, len(p.rawContent))
+ copy(p.rawContentCopy, p.rawContent)
+ } else {
+ // Just reuse the same slice.
+ p.rawContentCopy = p.rawContent
+ }
+
if err := handleShortcodes(p, s.owner.tmpl); err != nil {
jww.ERROR.Printf("Failed to handle shortcodes for page %s: %s", p.BaseFileName(), err)
}
if p.Markup == "markdown" {
- tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.rawContent)
+ tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.rawContentCopy)
p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
- p.rawContent = tmpContent
+ p.rawContentCopy = tmpContent
}
if p.Markup != "html" {
if err != nil {
jww.ERROR.Printf("Failed to set use defined summary: %s", err)
} else if summaryContent != nil {
- p.rawContent = summaryContent.content
+ p.rawContentCopy = summaryContent.content
}
- p.Content = helpers.BytesToHTML(p.rawContent)
- p.rendered = true
+ p.Content = helpers.BytesToHTML(p.rawContentCopy)
if summaryContent == nil {
p.setAutoSummary()
}
+
+ } else {
+ p.Content = helpers.BytesToHTML(p.rawContentCopy)
}
+ // no need for this anymore
+ p.rawContentCopy = nil
+
//analyze for raw stats
p.analyzePage()
+
}
}(pageChan, wg)
}
return err
}
- p.rawContent, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, shortcodes)
+ p.rawContentCopy, err = replaceShortcodeTokens(p.rawContentCopy, shortcodePlaceholderPrefix, shortcodes)
if err != nil {
jww.FATAL.Printf("Failed to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())
testCommonResetState()
viper.Set("DefaultContentLanguageInSubdir", defaultInSubDir)
- sites := createMultiTestSites(t, multiSiteTomlConfig)
+ sites := createMultiTestSites(t, multiSiteTOMLConfig)
err := sites.Build(BuildCfg{})
content string
suffix string
}{
- {multiSiteTomlConfig, "toml"},
+ {multiSiteTOMLConfig, "toml"},
{multiSiteYAMLConfig, "yml"},
{multiSiteJSONConfig, "json"},
} {
func TestMultiSitesRebuild(t *testing.T) {
testCommonResetState()
- sites := createMultiTestSites(t, multiSiteTomlConfig)
- cfg := BuildCfg{}
+ sites := createMultiTestSites(t, multiSiteTOMLConfig)
+ cfg := BuildCfg{Watching: true}
err := sites.Build(cfg)
docFr := readDestination(t, "public/fr/sect/doc1/index.html")
assert.True(t, strings.Contains(docFr, "Bonjour"), "No Bonjour")
+ // check single page content
+ assertFileContent(t, "public/fr/sect/doc1/index.html", true, "Single", "Shortcode: Bonjour")
+ assertFileContent(t, "public/en/sect/doc1-slug/index.html", true, "Single", "Shortcode: Hello")
+
for i, this := range []struct {
preFunc func(t *testing.T)
events []fsnotify.Event
},
},
+ // Change a shortcode
+ {
+ func(t *testing.T) {
+ writeSource(t, "layouts/shortcodes/shortcode.html", "Modified Shortcode: {{ i18n \"hello\" }}")
+ },
+ []fsnotify.Event{
+ {Name: "layouts/shortcodes/shortcode.html", Op: fsnotify.Write},
+ },
+ func(t *testing.T) {
+ assert.Len(t, enSite.Pages, 4)
+ assert.Len(t, enSite.AllPages, 10)
+ assert.Len(t, frSite.Pages, 4)
+ assertFileContent(t, "public/fr/sect/doc1/index.html", true, "Single", "Modified Shortcode: Salut")
+ assertFileContent(t, "public/en/sect/doc1-slug/index.html", true, "Single", "Modified Shortcode: Hello")
+ },
+ },
} {
if this.preFunc != nil {
func TestAddNewLanguage(t *testing.T) {
testCommonResetState()
- sites := createMultiTestSites(t, multiSiteTomlConfig)
+ sites := createMultiTestSites(t, multiSiteTOMLConfig)
cfg := BuildCfg{}
err := sites.Build(cfg)
t.Fatalf("Failed to build sites: %s", err)
}
- newConfig := multiSiteTomlConfig + `
+ newConfig := multiSiteTOMLConfig + `
[Languages.sv]
weight = 15
}
-var multiSiteTomlConfig = `
+var multiSiteTOMLConfig = `
DefaultExtension = "html"
baseurl = "http://example.com/blog"
DisableSitemap = false
)
type Page struct {
- Params map[string]interface{}
- Content template.HTML
- Summary template.HTML
- Aliases []string
- Status string
- Images []Image
- Videos []Video
- TableOfContents template.HTML
- Truncated bool
- Draft bool
- PublishDate time.Time
- ExpiryDate time.Time
- Markup string
- translations Pages
- extension string
- contentType string
- renderable bool
- Layout string
- layoutsCalculated []string
- linkTitle string
- frontmatter []byte
- rawContent []byte
+ Params map[string]interface{}
+ Content template.HTML
+ Summary template.HTML
+ Aliases []string
+ Status string
+ Images []Image
+ Videos []Video
+ TableOfContents template.HTML
+ Truncated bool
+ Draft bool
+ PublishDate time.Time
+ ExpiryDate time.Time
+ Markup string
+ translations Pages
+ extension string
+ contentType string
+ renderable bool
+ Layout string
+ layoutsCalculated []string
+ linkTitle string
+ frontmatter []byte
+
+ // rawContent isn't "raw" as in the same as in the content file.
+ // Hugo cares about memory consumption, so we make changes to it to do
+ // markdown rendering etc., but it is "raw enough" so we can do rebuilds
+ // when shortcode changes etc.
+ rawContent []byte
+
+ // When running Hugo in watch mode, we do partial rebuilds and have to make
+ // a copy of the rawContent to be prepared for rebuilds when shortcodes etc.
+ // have changed.
+ rawContentCopy []byte
+
+ // state telling if this is a "new page" or if we have rendered it previously.
+ rendered bool
+
contentShortCodes map[string]func() (string, error)
shortcodes map[string]shortcode
plain string // TODO should be []byte
Source
Position `json:"-"`
Node
- rendered bool
}
type Source struct {
// Returns the page as summary and main if a user defined split is provided.
func (p *Page) setUserDefinedSummaryIfProvided() (*summaryContent, error) {
- sc := splitUserDefinedSummaryAndContent(p.Markup, p.rawContent)
+ sc := splitUserDefinedSummaryAndContent(p.Markup, p.rawContentCopy)
if sc == nil {
// No divider found
}
func (p *Page) ProcessShortcodes(t tpl.Template) {
-
- // these short codes aren't used until after Page render,
- // but processed here to avoid coupling
- // TODO(bep) Move this and remove p.contentShortCodes
- if !p.rendered {
- tmpContent, tmpContentShortCodes, _ := extractAndRenderShortcodes(string(p.rawContent), p, t)
- p.rawContent = []byte(tmpContent)
- p.contentShortCodes = tmpContentShortCodes
- } else {
- // shortcode template may have changed, rerender
- p.contentShortCodes = renderShortcodes(p.shortcodes, p, t)
- }
-
+ tmpContent, tmpContentShortCodes, _ := extractAndRenderShortcodes(string(p.rawContent), p, t)
+ p.rawContent = []byte(tmpContent)
+ p.contentShortCodes = tmpContentShortCodes
}
func (p *Page) FullFilePath() string {
func extractAndRenderShortcodes(stringToParse string, p *Page, t tpl.Template) (string, map[string]func() (string, error), error) {
- if p.rendered {
- panic("Illegal state: Page already marked as rendered, please reuse the shortcodes")
- }
-
content, shortcodes, err := extractShortcodes(stringToParse, p, t)
if err != nil {
s.timer.Step(step)
}
+type whatChanged struct {
+ source bool
+ other bool
+}
+
// reBuild partially rebuilds a site given the filesystem events.
// It returns whetever the content source was changed.
-func (s *Site) reBuild(events []fsnotify.Event) (bool, error) {
+func (s *Site) reBuild(events []fsnotify.Event) (whatChanged, error) {
jww.DEBUG.Printf("Rebuild for events %q", events)
}
if len(i18nChanged) > 0 {
- // TODO(bep ml
s.readI18nSources()
}
go incrementalReadCollator(s, readResults, pageChan, fileConvChan, coordinator, errs)
go converterCollator(s, convertResults, errs)
- if len(tmplChanged) > 0 || len(dataChanged) > 0 {
- // Do not need to read the files again, but they need conversion
- // for shortocde re-rendering.
- for _, p := range s.rawAllPages {
- if p.shouldBuild() {
- pageChan <- p
- }
- }
- }
-
for _, ev := range sourceReallyChanged {
file, err := s.reReadFile(ev.Name)
s.timerStep("read & convert pages from source")
- return len(sourceChanged) > 0, nil
+ changed := whatChanged{
+ source: len(sourceChanged) > 0,
+ other: len(tmplChanged) > 0 || len(i18nChanged) > 0 || len(dataChanged) > 0,
+ }
+
+ return changed, nil
}
projects / brevno-suite / hugo / commitdiff