tpl/tplimpl: Handle late transformation of templates

Fixes #5865

diff --git a/tpl/template.go b/tpl/template.go
index 01f79c407b2ff728ff0f8d38b152488e3e32e25f..93577136407634250696ee1327247aa07fcb2eee 100644 (file)
--- a/tpl/template.go
+++ b/tpl/template.go
@@ -52,7 +52,7 @@ type TemplateHandler interface {
 
        NewTextTemplate() TemplateParseFinder
 
-       MarkReady()
+       MarkReady() error
        RebuildClone()
 }
 

diff --git a/tpl/tplimpl/ace.go b/tpl/tplimpl/ace.go
index 6fedcb583e01c991b67da6fc27cba0094058cd62..bdbc7105992487a2b3a23188a2fefd69a6605dc5 100644 (file)
--- a/tpl/tplimpl/ace.go
+++ b/tpl/tplimpl/ace.go
@@ -53,15 +53,15 @@ func (t *templateHandler) addAceTemplate(name, basePath, innerPath string, baseC
 
        typ := resolveTemplateType(name)
 
-       info, err := applyTemplateTransformersToHMLTTemplate(typ, templ)
+       c, err := applyTemplateTransformersToHMLTTemplate(typ, templ)
        if err != nil {
                return err
        }
 
        if typ == templateShortcode {
-               t.addShortcodeVariant(name, info, templ)
+               t.addShortcodeVariant(name, c.Info, templ)
        } else {
-               t.templateInfo[name] = info
+               t.templateInfo[name] = c.Info
        }
 
        return nil

diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go
index 8fcaa8d646296f00d60c66e012995526cd7ed8bf..c1fad30687c7a1b707968368e0fac2f1ac3caca7 100644 (file)
--- a/tpl/tplimpl/template.go
+++ b/tpl/tplimpl/template.go
@@ -18,6 +18,7 @@ import (
        "html/template"
        "strings"
        texttemplate "text/template"
+       "text/template/parse"
 
        "github.com/gohugoio/hugo/hugofs"
        "github.com/gohugoio/hugo/tpl/tplimpl/embedded"
@@ -51,7 +52,6 @@ var (
        _ tpl.TemplateFinder        = (*textTemplates)(nil)
        _ templateLoader            = (*htmlTemplates)(nil)
        _ templateLoader            = (*textTemplates)(nil)
-       _ templateLoader            = (*templateHandler)(nil)
        _ templateFuncsterTemplater = (*htmlTemplates)(nil)
        _ templateFuncsterTemplater = (*textTemplates)(nil)
 )
@@ -66,7 +66,7 @@ type templateErr struct {
 
 type templateLoader interface {
        handleMaster(name, overlayFilename, masterFilename string, onMissing func(filename string) (templateInfo, error)) error
-       addTemplate(name, tpl string) error
+       addTemplate(name, tpl string) (*templateContext, error)
        addLateTemplate(name, tpl string) error
 }
 
@@ -329,6 +329,7 @@ func (t *templateHandler) clone(d *deps.Deps) *templateHandler {
 func newTemplateAdapter(deps *deps.Deps) *templateHandler {
        common := &templatesCommon{
                nameBaseTemplateName: make(map[string]string),
+               transformNotFound:    make(map[string]bool),
        }
 
        htmlT := &htmlTemplates{
@@ -364,6 +365,10 @@ type templatesCommon struct {
 
        // Used to get proper filenames in errors
        nameBaseTemplateName map[string]string
+
+       // Holds names of the templates not found during the first AST transformation
+       // pass.
+       transformNotFound map[string]bool
 }
 type htmlTemplates struct {
        mu sync.RWMutex
@@ -491,37 +496,42 @@ func (t *templateHandler) LoadTemplates(prefix string) error {
 
 }
 
-func (t *htmlTemplates) addTemplateIn(tt *template.Template, name, tpl string) error {
+func (t *htmlTemplates) addTemplateIn(tt *template.Template, name, tpl string) (*templateContext, error) {
        t.mu.Lock()
        defer t.mu.Unlock()
 
        templ, err := tt.New(name).Parse(tpl)
        if err != nil {
-               return err
+               return nil, err
        }
 
        typ := resolveTemplateType(name)
 
-       info, err := applyTemplateTransformersToHMLTTemplate(typ, templ)
+       c, err := applyTemplateTransformersToHMLTTemplate(typ, templ)
        if err != nil {
-               return err
+               return nil, err
+       }
+
+       for k, _ := range c.notFound {
+               t.transformNotFound[k] = true
        }
 
        if typ == templateShortcode {
-               t.handler.addShortcodeVariant(name, info, templ)
+               t.handler.addShortcodeVariant(name, c.Info, templ)
        } else {
-               t.handler.templateInfo[name] = info
+               t.handler.templateInfo[name] = c.Info
        }
 
-       return nil
+       return c, nil
 }
 
-func (t *htmlTemplates) addTemplate(name, tpl string) error {
+func (t *htmlTemplates) addTemplate(name, tpl string) (*templateContext, error) {
        return t.addTemplateIn(t.t, name, tpl)
 }
 
 func (t *htmlTemplates) addLateTemplate(name, tpl string) error {
-       return t.addTemplateIn(t.clone, name, tpl)
+       _, err := t.addTemplateIn(t.clone, name, tpl)
+       return err
 }
 
 type textTemplate struct {
@@ -556,41 +566,91 @@ func (t *textTemplate) parseIn(tt *texttemplate.Template, name, tpl string) (*te
        return templ, nil
 }
 
-func (t *textTemplates) addTemplateIn(tt *texttemplate.Template, name, tpl string) error {
+func (t *textTemplates) addTemplateIn(tt *texttemplate.Template, name, tpl string) (*templateContext, error) {
        name = strings.TrimPrefix(name, textTmplNamePrefix)
        templ, err := t.parseIn(tt, name, tpl)
        if err != nil {
-               return err
+               return nil, err
        }
 
        typ := resolveTemplateType(name)
 
-       info, err := applyTemplateTransformersToTextTemplate(typ, templ)
+       c, err := applyTemplateTransformersToTextTemplate(typ, templ)
        if err != nil {
-               return err
+               return nil, err
+       }
+
+       for k, _ := range c.notFound {
+               t.transformNotFound[k] = true
        }
 
        if typ == templateShortcode {
-               t.handler.addShortcodeVariant(name, info, templ)
+               t.handler.addShortcodeVariant(name, c.Info, templ)
        } else {
-               t.handler.templateInfo[name] = info
+               t.handler.templateInfo[name] = c.Info
        }
 
-       return nil
+       return c, nil
 }
 
-func (t *textTemplates) addTemplate(name, tpl string) error {
+func (t *textTemplates) addTemplate(name, tpl string) (*templateContext, error) {
        return t.addTemplateIn(t.t, name, tpl)
 }
 
 func (t *textTemplates) addLateTemplate(name, tpl string) error {
-       return t.addTemplateIn(t.clone, name, tpl)
+       _, err := t.addTemplateIn(t.clone, name, tpl)
+       return err
 }
 
 func (t *templateHandler) addTemplate(name, tpl string) error {
        return t.AddTemplate(name, tpl)
 }
 
+func (t *templateHandler) postTransform() error {
+       if len(t.html.transformNotFound) == 0 && len(t.text.transformNotFound) == 0 {
+               return nil
+       }
+
+       defer func() {
+               t.text.transformNotFound = make(map[string]bool)
+               t.html.transformNotFound = make(map[string]bool)
+       }()
+
+       for _, s := range []struct {
+               lookup            func(name string) *parse.Tree
+               transformNotFound map[string]bool
+       }{
+               // html templates
+               {func(name string) *parse.Tree {
+                       templ := t.html.lookup(name)
+                       if templ == nil {
+                               return nil
+                       }
+                       return templ.Tree
+               }, t.html.transformNotFound},
+               // text templates
+               {func(name string) *parse.Tree {
+                       templT := t.text.lookup(name)
+                       if templT == nil {
+                               return nil
+                       }
+                       return templT.Tree
+               }, t.text.transformNotFound},
+       } {
+               for name, _ := range s.transformNotFound {
+                       templ := s.lookup(name)
+                       if templ != nil {
+                               _, err := applyTemplateTransformers(templateUndefined, templ, s.lookup)
+                               if err != nil {
+                                       return err
+                               }
+                       }
+               }
+       }
+
+       return nil
+}
+
 func (t *templateHandler) addLateTemplate(name, tpl string) error {
        return t.AddLateTemplate(name, tpl)
 }
@@ -608,9 +668,11 @@ func (t *templateHandler) AddLateTemplate(name, tpl string) error {
 // AddTemplate parses and adds a template to the collection.
 // Templates with name prefixed with "_text" will be handled as plain
 // text templates.
+// TODO(bep) clean up these addTemplate variants
 func (t *templateHandler) AddTemplate(name, tpl string) error {
        h := t.getTemplateHandler(name)
-       if err := h.addTemplate(name, tpl); err != nil {
+       _, err := h.addTemplate(name, tpl)
+       if err != nil {
                return err
        }
        return nil
@@ -620,7 +682,11 @@ func (t *templateHandler) AddTemplate(name, tpl string) error {
 // after this is set.
 // TODO(bep) if this proves to be resource heavy, we could detect
 // earlier if we really need this, or make it lazy.
-func (t *templateHandler) MarkReady() {
+func (t *templateHandler) MarkReady() error {
+       if err := t.postTransform(); err != nil {
+               return err
+       }
+
        if t.html.clone == nil {
                t.html.clone = template.Must(t.html.t.Clone())
                t.html.cloneClone = template.Must(t.html.clone.Clone())
@@ -629,6 +695,8 @@ func (t *templateHandler) MarkReady() {
                t.text.clone = texttemplate.Must(t.text.t.Clone())
                t.text.cloneClone = texttemplate.Must(t.text.clone.Clone())
        }
+
+       return nil
 }
 
 // RebuildClone rebuilds the cloned templates. Used for live-reloads.
@@ -890,15 +958,15 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e
 
                typ := resolveTemplateType(name)
 
-               info, err := applyTemplateTransformersToHMLTTemplate(typ, templ)
+               c, err := applyTemplateTransformersToHMLTTemplate(typ, templ)
                if err != nil {
                        return err
                }
 
                if typ == templateShortcode {
-                       t.addShortcodeVariant(templateName, info, templ)
+                       t.addShortcodeVariant(templateName, c.Info, templ)
                } else {
-                       t.templateInfo[name] = info
+                       t.templateInfo[name] = c.Info
                }
 
                return nil

diff --git a/tpl/tplimpl/templateProvider.go b/tpl/tplimpl/templateProvider.go
index 3a803f2da9cc194a6cb04c573cd1c54aaa787e03..9322223442aec179533fb117756235695bb0217e 100644 (file)
--- a/tpl/tplimpl/templateProvider.go
+++ b/tpl/tplimpl/templateProvider.go
@@ -46,9 +46,7 @@ func (*TemplateProvider) Update(deps *deps.Deps) error {
 
        }
 
-       newTmpl.MarkReady()
-
-       return nil
+       return newTmpl.MarkReady()
 
 }
 
@@ -60,7 +58,6 @@ func (*TemplateProvider) Clone(d *deps.Deps) error {
 
        d.Tmpl = clone
 
-       clone.MarkReady()
+       return clone.MarkReady()
 
-       return nil
 }

diff --git a/tpl/tplimpl/template_ast_transformers.go b/tpl/tplimpl/template_ast_transformers.go
index 57fafcd88f65b06870382ed2a748419a0581d92e..fb0728a63a783621064d6e42c399deba754327ab 100644 (file)
--- a/tpl/tplimpl/template_ast_transformers.go
+++ b/tpl/tplimpl/template_ast_transformers.go
@@ -50,6 +50,7 @@ const (
 type templateContext struct {
        decl     decl
        visited  map[string]bool
+       notFound map[string]bool
        lookupFn func(name string) *parse.Tree
 
        // The last error encountered.
@@ -72,7 +73,15 @@ func (c templateContext) getIfNotVisited(name string) *parse.Tree {
                return nil
        }
        c.visited[name] = true
-       return c.lookupFn(name)
+       templ := c.lookupFn(name)
+       if templ == nil {
+               // This may be a inline template defined outside of this file
+               // and not yet parsed. Unusual, but it happens.
+               // Store the name to try again later.
+               c.notFound[name] = true
+       }
+
+       return templ
 }
 
 func newTemplateContext(lookupFn func(name string) *parse.Tree) *templateContext {
@@ -80,8 +89,8 @@ func newTemplateContext(lookupFn func(name string) *parse.Tree) *templateContext
                Info:     tpl.Info{Config: tpl.DefaultConfig},
                lookupFn: lookupFn,
                decl:     make(map[string]string),
-               visited:  make(map[string]bool)}
-
+               visited:  make(map[string]bool),
+               notFound: make(map[string]bool)}
 }
 
 func createParseTreeLookup(templ *template.Template) func(nn string) *parse.Tree {
@@ -94,11 +103,11 @@ func createParseTreeLookup(templ *template.Template) func(nn string) *parse.Tree
        }
 }
 
-func applyTemplateTransformersToHMLTTemplate(typ templateType, templ *template.Template) (tpl.Info, error) {
+func applyTemplateTransformersToHMLTTemplate(typ templateType, templ *template.Template) (*templateContext, error) {
        return applyTemplateTransformers(typ, templ.Tree, createParseTreeLookup(templ))
 }
 
-func applyTemplateTransformersToTextTemplate(typ templateType, templ *texttemplate.Template) (tpl.Info, error) {
+func applyTemplateTransformersToTextTemplate(typ templateType, templ *texttemplate.Template) (*templateContext, error) {
        return applyTemplateTransformers(typ, templ.Tree,
                func(nn string) *parse.Tree {
                        tt := templ.Lookup(nn)
@@ -109,9 +118,9 @@ func applyTemplateTransformersToTextTemplate(typ templateType, templ *texttempla
                })
 }
 
-func applyTemplateTransformers(typ templateType, templ *parse.Tree, lookupFn func(name string) *parse.Tree) (tpl.Info, error) {
+func applyTemplateTransformers(typ templateType, templ *parse.Tree, lookupFn func(name string) *parse.Tree) (*templateContext, error) {
        if templ == nil {
-               return tpl.Info{}, errors.New("expected template, but none provided")
+               return nil, errors.New("expected template, but none provided")
        }
 
        c := newTemplateContext(lookupFn)
@@ -125,7 +134,7 @@ func applyTemplateTransformers(typ templateType, templ *parse.Tree, lookupFn fun
                templ.Root = c.wrapInPartialReturnWrapper(templ.Root)
        }
 
-       return c.Info, err
+       return c, err
 }
 
 const (

diff --git a/tpl/tplimpl/template_ast_transformers_test.go b/tpl/tplimpl/template_ast_transformers_test.go
index 9ed29d27f4b2889766652766b286ff63ccc78f63..be1efc522c858c98d786aee6b8a102c35e3edb77 100644 (file)
--- a/tpl/tplimpl/template_ast_transformers_test.go
+++ b/tpl/tplimpl/template_ast_transformers_test.go
@@ -26,10 +26,6 @@ import (
        "github.com/stretchr/testify/require"
 )
 
-type handler interface {
-       addTemplate(name, tpl string) error
-}
-
 var (
        testFuncs = map[string]interface{}{
                "getif":  func(v interface{}) interface{} { return v },
@@ -413,7 +409,7 @@ func TestInsertIsZeroFunc(t *testing.T) {
                        "T":        &T{NonEmptyInterfaceTypedNil: (*T)(nil)},
                }
 
-               templ = `
+               templ1 = `
 {{ if .True }}.True: TRUE{{ else }}.True: FALSE{{ end }}
 {{ if .TimeZero }}.TimeZero1: TRUE{{ else }}.TimeZero1: FALSE{{ end }}
 {{ if (.TimeZero) }}.TimeZero2: TRUE{{ else }}.TimeZero2: FALSE{{ end }}
@@ -423,31 +419,49 @@ func TestInsertIsZeroFunc(t *testing.T) {
 {{ template "mytemplate" . }}
 {{ if .T.NonEmptyInterfaceTypedNil }}.NonEmptyInterfaceTypedNil: TRUE{{ else }}.NonEmptyInterfaceTypedNil: FALSE{{ end }}
 
+{{ template "other-file-template" . }}
 
 {{ define "mytemplate" }}
 {{ if .TimeZero }}.TimeZero1: mytemplate: TRUE{{ else }}.TimeZero1: mytemplate: FALSE{{ end }}
 {{ end }}
 
+`
+
+               // https://github.com/gohugoio/hugo/issues/5865
+               templ2 = `{{ define "other-file-template" }}
+{{ if .TimeZero }}.TimeZero1: other-file-template: TRUE{{ else }}.TimeZero1: other-file-template: FALSE{{ end }}
+{{ end }}              
 `
        )
 
        d := newD(assert)
-       h := d.Tmpl.(handler)
-
-       assert.NoError(h.addTemplate("mytemplate.html", templ))
-
-       tt, _ := d.Tmpl.Lookup("mytemplate.html")
-       result, err := tt.(tpl.TemplateExecutor).ExecuteToString(ctx)
-       assert.NoError(err)
-
-       assert.Contains(result, ".True: TRUE")
-       assert.Contains(result, ".TimeZero1: FALSE")
-       assert.Contains(result, ".TimeZero2: FALSE")
-       assert.Contains(result, ".TimeZero3: TRUE")
-       assert.Contains(result, ".Now: TRUE")
-       assert.Contains(result, "TimeZero1 with: FALSE")
-       assert.Contains(result, ".TimeZero1: mytemplate: FALSE")
-       assert.Contains(result, ".NonEmptyInterfaceTypedNil: FALSE")
+       h := d.Tmpl.(tpl.TemplateHandler)
+
+       // HTML templates
+       assert.NoError(h.AddTemplate("mytemplate.html", templ1))
+       assert.NoError(h.AddTemplate("othertemplate.html", templ2))
+
+       // Text templates
+       assert.NoError(h.AddTemplate("_text/mytexttemplate.txt", templ1))
+       assert.NoError(h.AddTemplate("_text/myothertexttemplate.txt", templ2))
+
+       assert.NoError(h.MarkReady())
+
+       for _, name := range []string{"mytemplate.html", "mytexttemplate.txt"} {
+               tt, _ := d.Tmpl.Lookup(name)
+               result, err := tt.(tpl.TemplateExecutor).ExecuteToString(ctx)
+               assert.NoError(err)
+
+               assert.Contains(result, ".True: TRUE")
+               assert.Contains(result, ".TimeZero1: FALSE")
+               assert.Contains(result, ".TimeZero2: FALSE")
+               assert.Contains(result, ".TimeZero3: TRUE")
+               assert.Contains(result, ".Now: TRUE")
+               assert.Contains(result, "TimeZero1 with: FALSE")
+               assert.Contains(result, ".TimeZero1: mytemplate: FALSE")
+               assert.Contains(result, ".TimeZero1: other-file-template: FALSE")
+               assert.Contains(result, ".NonEmptyInterfaceTypedNil: FALSE")
+       }
 
 }
 

diff --git a/tpl/tplimpl/template_info_test.go b/tpl/tplimpl/template_info_test.go
index 0ebaa6da3c208520a10adc6f5903f2c1e53315b5..be9d7e2f1a38c65dcbeee27b500a36b389e63e8a 100644 (file)
--- a/tpl/tplimpl/template_info_test.go
+++ b/tpl/tplimpl/template_info_test.go
@@ -24,9 +24,9 @@ import (
 func TestTemplateInfoShortcode(t *testing.T) {
        assert := require.New(t)
        d := newD(assert)
-       h := d.Tmpl.(handler)
+       h := d.Tmpl.(tpl.TemplateHandler)
 
-       assert.NoError(h.addTemplate("shortcodes/mytemplate.html", `
+       assert.NoError(h.AddTemplate("shortcodes/mytemplate.html", `
 {{ .Inner }}
 `))
        tt, found, _ := d.Tmpl.LookupVariant("mytemplate", tpl.TemplateVariants{})