Use a regular expression in replaceShortcodeTokens

This fixes a bug where a shortcode needs to be expanded multiple times,
which can arise in practice when using reference links.

diff --git a/hugolib/handler_page.go b/hugolib/handler_page.go
index 6c912ded3377e7788458abd8cb71009295b850c2..b3b236344c24dfc74a70a9458af090fd9ca6df1d 100644 (file)
--- a/hugolib/handler_page.go
+++ b/hugolib/handler_page.go
@@ -60,7 +60,7 @@ func (h markdownHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
        tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)))
 
        if len(p.contentShortCodes) > 0 {
-               tmpContentWithTokensReplaced, err := replaceShortcodeTokens(tmpContent, shortcodePlaceholderPrefix, -1, true, p.contentShortCodes)
+               tmpContentWithTokensReplaced, err := replaceShortcodeTokens(tmpContent, shortcodePlaceholderPrefix, true, p.contentShortCodes)
 
                if err != nil {
                        jww.FATAL.Printf("Fail to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())
@@ -113,7 +113,7 @@ func (h rstHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
        tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)))
 
        if len(p.contentShortCodes) > 0 {
-               tmpContentWithTokensReplaced, err := replaceShortcodeTokens(tmpContent, shortcodePlaceholderPrefix, -1, true, p.contentShortCodes)
+               tmpContentWithTokensReplaced, err := replaceShortcodeTokens(tmpContent, shortcodePlaceholderPrefix, true, p.contentShortCodes)
 
                if err != nil {
                        jww.FATAL.Printf("Fail to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())

diff --git a/hugolib/page.go b/hugolib/page.go
index ffbe7772b23e9b64db6ab47cc5d01986c11e6ab9..9a96b8360ce7b2414f52972fd51610267263d378 100644 (file)
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -162,10 +162,9 @@ func (p *Page) setSummary() {
                p.Truncated = true // by definition
                header := bytes.Split(p.rawContent, helpers.SummaryDivider)[0]
                renderedHeader := p.renderBytes(header)
-               numShortcodesInHeader := bytes.Count(header, []byte(shortcodePlaceholderPrefix))
                if len(p.contentShortCodes) > 0 {
                        tmpContentWithTokensReplaced, err :=
-                               replaceShortcodeTokens(renderedHeader, shortcodePlaceholderPrefix, numShortcodesInHeader, true, p.contentShortCodes)
+                               replaceShortcodeTokens(renderedHeader, shortcodePlaceholderPrefix, true, p.contentShortCodes)
                        if err != nil {
                                jww.FATAL.Printf("Failed to replace short code tokens in Summary for %s:\n%s", p.BaseFileName(), err.Error())
                        } else {

diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go
index 03cd7d4a7dc1aa0493ed4d693866ee246f159a34..9458b39612d741f7c882f7097f47c59e10650b8f 100644 (file)
--- a/hugolib/shortcode.go
+++ b/hugolib/shortcode.go
@@ -20,7 +20,6 @@ import (
        "reflect"
        "regexp"
        "sort"
-       "strconv"
        "strings"
        "sync"
 
@@ -132,7 +131,7 @@ func ShortcodesHandle(stringToParse string, page *Page, t tpl.Template) string {
        tmpContent, tmpShortcodes := extractAndRenderShortcodes(stringToParse, page, t)
 
        if len(tmpShortcodes) > 0 {
-               tmpContentWithTokensReplaced, err := replaceShortcodeTokens([]byte(tmpContent), shortcodePlaceholderPrefix, -1, true, tmpShortcodes)
+               tmpContentWithTokensReplaced, err := replaceShortcodeTokens([]byte(tmpContent), shortcodePlaceholderPrefix, true, tmpShortcodes)
 
                if err != nil {
                        jww.ERROR.Printf("Fail to replace short code tokens in %s:\n%s", page.BaseFileName(), err.Error())
@@ -428,60 +427,44 @@ Loop:
 }
 
 // Replace prefixed shortcode tokens (HUGOSHORTCODE-1, HUGOSHORTCODE-2) with the real content.
-// This assumes that all tokens exist in the input string and that they are in order.
-// numReplacements = -1 will do len(replacements), and it will always start from the beginning (1)
 // wrapped = true means that the token has been wrapped in {@{@/@}@}
-func replaceShortcodeTokens(source []byte, prefix string, numReplacements int, wrapped bool, replacements map[string]string) ([]byte, error) {
+func replaceShortcodeTokens(source []byte, prefix string, wrapped bool, replacements map[string]string) (b []byte, err error) {
+       var re *regexp.Regexp
 
-       if numReplacements < 0 {
-               numReplacements = len(replacements)
-       }
-
-       if numReplacements == 0 {
-               return source, nil
-       }
-
-       newLen := len(source)
-
-       for i := 1; i <= numReplacements; i++ {
-               key := prefix + "-" + strconv.Itoa(i)
-
-               if wrapped {
-                       key = "{@{@" + key + "@}@}"
+       if wrapped {
+               re, err = regexp.Compile(`\{@\{@` + regexp.QuoteMeta(prefix) + `-\d+@\}@\}`)
+               if err != nil {
+                       return nil, err
+               }
+       } else {
+               re, err = regexp.Compile(regexp.QuoteMeta(prefix) + `-(\d+)`)
+               if err != nil {
+                       return nil, err
                }
-               val := []byte(replacements[key])
-
-               newLen += (len(val) - len(key))
        }
 
-       buff := make([]byte, newLen)
-
-       width := 0
-       start := 0
-
-       for i := 0; i < numReplacements; i++ {
-               tokenNum := i + 1
-               oldVal := prefix + "-" + strconv.Itoa(tokenNum)
-               if wrapped {
-                       oldVal = "{@{@" + oldVal + "@}@}"
+       // use panic/recover for reporting if an unknown
+       defer func() {
+               if r := recover(); r != nil {
+                       var ok bool
+                       b = nil
+                       err, ok = r.(error)
+                       if !ok {
+                               err = fmt.Errorf("unexpected panic during replaceShortcodeTokens: %v", r)
+                       }
                }
-               newVal := []byte(replacements[oldVal])
-               j := start
-
-               k := bytes.Index(source[start:], []byte(oldVal))
+       }()
+       b = re.ReplaceAllFunc(source, func(m []byte) []byte {
+               key := string(m)
 
-               if k < 0 {
-                       // this should never happen, but let the caller decide to panic or not
-                       return nil, fmt.Errorf("illegal state in content; shortcode token #%d is missing or out of order (%q)", tokenNum, source)
+               if val, ok := replacements[key]; ok {
+                       return []byte(val)
+               } else {
+                       panic(fmt.Errorf("unknown shortcode token %q", key))
                }
-               j += k
+       })
 
-               width += copy(buff[width:], source[start:j])
-               width += copy(buff[width:], newVal)
-               start = j + len(oldVal)
-       }
-       width += copy(buff[width:], source[start:])
-       return buff[0:width], nil
+       return b, err
 }
 
 func GetTemplate(name string, t tpl.Template) *template.Template {

diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go
index 06f36aebd67b9264ebda3ad9dba75935a2c11c65..ad966b286871a17d542645eeb682408d97c52f10 100644 (file)
--- a/hugolib/shortcode_test.go
+++ b/hugolib/shortcode_test.go
@@ -281,23 +281,24 @@ func collectAndShortShortcodes(shortcodes map[string]shortcode) []string {
 
 func TestReplaceShortcodeTokens(t *testing.T) {
        for i, this := range []struct {
-               input           []byte
-               prefix          string
-               replacements    map[string]string
-               numReplacements int
-               wrappedInDiv    bool
-               expect          interface{}
+               input        []byte
+               prefix       string
+               replacements map[string]string
+               wrappedInDiv bool
+               expect       interface{}
        }{
-               {[]byte("Hello PREFIX-1."), "PREFIX", map[string]string{"PREFIX-1": "World"}, -1, false, []byte("Hello World.")},
-               {[]byte("A {@{@A-1@}@} asdf {@{@A-2@}@}."), "A", map[string]string{"{@{@A-1@}@}": "v1", "{@{@A-2@}@}": "v2"}, -1, true, []byte("A v1 asdf v2.")},
-               {[]byte("Hello PREFIX2-1. Go PREFIX2-2, Go, Go PREFIX2-3 Go Go!."), "PREFIX2", map[string]string{"PREFIX2-1": "Europe", "PREFIX2-2": "Jonny", "PREFIX2-3": "Johnny"}, -1, false, []byte("Hello Europe. Go Jonny, Go, Go Johnny Go Go!.")},
-               {[]byte("A PREFIX-2 PREFIX-1."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B"}, -1, false, false},
-               {[]byte("A PREFIX-1 PREFIX-2"), "PREFIX", map[string]string{"PREFIX-1": "A"}, -1, false, []byte("A A PREFIX-2")},
-               {[]byte("A PREFIX-1 but not the second."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B"}, -1, false, false},
-               {[]byte("An PREFIX-1."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B"}, 1, false, []byte("An A.")},
-               {[]byte("An PREFIX-1 PREFIX-2."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B"}, 1, false, []byte("An A PREFIX-2.")},
+               {[]byte("Hello PREFIX-1."), "PREFIX", map[string]string{"PREFIX-1": "World"}, false, []byte("Hello World.")},
+               {[]byte("A {@{@A-1@}@} asdf {@{@A-2@}@}."), "A", map[string]string{"{@{@A-1@}@}": "v1", "{@{@A-2@}@}": "v2"}, true, []byte("A v1 asdf v2.")},
+               {[]byte("Hello PREFIX2-1. Go PREFIX2-2, Go, Go PREFIX2-3 Go Go!."), "PREFIX2", map[string]string{"PREFIX2-1": "Europe", "PREFIX2-2": "Jonny", "PREFIX2-3": "Johnny"}, false, []byte("Hello Europe. Go Jonny, Go, Go Johnny Go Go!.")},
+               {[]byte("A PREFIX-2 PREFIX-1."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B"}, false, []byte("A B A.")},
+               {[]byte("A PREFIX-1 PREFIX-2"), "PREFIX", map[string]string{"PREFIX-1": "A"}, false, false},
+               {[]byte("A PREFIX-1 but not the second."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B"}, false, []byte("A A but not the second.")},
+               {[]byte("An PREFIX-1."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B"}, false, []byte("An A.")},
+               {[]byte("An PREFIX-1 PREFIX-2."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B"}, false, []byte("An A B.")},
+               {[]byte("A PREFIX-1 PREFIX-2 PREFIX-3 PREFIX-1 PREFIX-3."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B", "PREFIX-3": "C"}, false, []byte("A A B C A C.")},
+               {[]byte("A {@{@PREFIX-1@}@} {@{@PREFIX-2@}@} {@{@PREFIX-3@}@} {@{@PREFIX-1@}@} {@{@PREFIX-3@}@}."), "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "A", "{@{@PREFIX-2@}@}": "B", "{@{@PREFIX-3@}@}": "C"}, true, []byte("A A B C A C.")},
        } {
-               results, err := replaceShortcodeTokens(this.input, this.prefix, this.numReplacements, this.wrappedInDiv, this.replacements)
+               results, err := replaceShortcodeTokens(this.input, this.prefix, this.wrappedInDiv, this.replacements)
 
                if b, ok := this.expect.(bool); ok && !b {
                        if err == nil {