Make replaceShortcodeTokens rewrite the input slice

Currently a `[]byte` copy is returned. In most cases this is the safe thing to do, but we should just modify/grow the slice as needed.

This is faster and consumes less memory:

```
benchmark                             old ns/op     new ns/op     delta
BenchmarkReplaceShortcodeTokens-4     7350          4419          -39.88%

benchmark                             old allocs     new allocs     delta
BenchmarkReplaceShortcodeTokens-4     5              1              -80.00%

benchmark                             old bytes     new bytes     delta
BenchmarkReplaceShortcodeTokens-4     4816          1152          -76.08%
```

This commit is aso a small spring cleaning of duplicated code in the different `PageConvert` methods.

Fixes #1516

diff --git a/hugolib/handler_page.go b/hugolib/handler_page.go
index f925976f718ff888848676af67d9dce46d0a8311..0f0e3ca607966690cc6d2ede50e3fd8419d33922 100644 (file)
--- a/hugolib/handler_page.go
+++ b/hugolib/handler_page.go
@@ -56,25 +56,7 @@ type markdownHandler struct {
 
 func (h markdownHandler) Extensions() []string { return []string{"mdown", "markdown", "md"} }
 func (h markdownHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
-       p.ProcessShortcodes(t)
-
-       tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)))
-
-       if len(p.contentShortCodes) > 0 {
-               replaced, err := replaceShortcodeTokensInsources(shortcodePlaceholderPrefix, p.contentShortCodes,
-                       tmpContent, tmpTableOfContents)
-               if err != nil {
-                       jww.FATAL.Printf("Fail to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())
-                       return HandledResult{err: err}
-               }
-               tmpContent = replaced[0]
-               tmpTableOfContents = replaced[1]
-       }
-
-       p.Content = helpers.BytesToHTML(tmpContent)
-       p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
-
-       return HandledResult{err: nil}
+       return commonConvert(p, t)
 }
 
 type htmlHandler struct {
@@ -84,21 +66,18 @@ type htmlHandler struct {
 func (h htmlHandler) Extensions() []string { return []string{"html", "htm"} }
 func (h htmlHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
        p.ProcessShortcodes(t)
-       var content []byte
        var err error
 
        if len(p.contentShortCodes) > 0 {
-               content, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, p.contentShortCodes)
+               p.rawContent, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, p.contentShortCodes)
 
                if err != nil {
-                       jww.FATAL.Printf("Fail to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())
+                       jww.FATAL.Printf("Failed to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())
                        return HandledResult{err: err}
                }
-       } else {
-               content = p.rawContent
        }
 
-       p.Content = helpers.BytesToHTML(content)
+       p.Content = helpers.BytesToHTML(p.rawContent)
        return HandledResult{err: nil}
 }
 
@@ -108,27 +87,7 @@ type asciidocHandler struct {
 
 func (h asciidocHandler) Extensions() []string { return []string{"asciidoc", "adoc", "ad"} }
 func (h asciidocHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
-       p.ProcessShortcodes(t)
-
-       // TODO(spf13) Add/Refactor AsciiDoc Logic here
-       tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)))
-
-       if len(p.contentShortCodes) > 0 {
-               replaced, err := replaceShortcodeTokensInsources(shortcodePlaceholderPrefix, p.contentShortCodes,
-                       tmpContent, tmpTableOfContents)
-               if err != nil {
-                       jww.FATAL.Printf("Fail to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())
-                       return HandledResult{err: err}
-               }
-               tmpContent = replaced[0]
-               tmpTableOfContents = replaced[1]
-       }
-
-       p.Content = helpers.BytesToHTML(tmpContent)
-       p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
-
-       //err := p.Convert()
-       return HandledResult{page: p, err: nil}
+       return commonConvert(p, t)
 }
 
 type rstHandler struct {
@@ -137,25 +96,7 @@ type rstHandler struct {
 
 func (h rstHandler) Extensions() []string { return []string{"rest", "rst"} }
 func (h rstHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
-       p.ProcessShortcodes(t)
-
-       tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)))
-
-       if len(p.contentShortCodes) > 0 {
-               replaced, err := replaceShortcodeTokensInsources(shortcodePlaceholderPrefix, p.contentShortCodes,
-                       tmpContent, tmpTableOfContents)
-               if err != nil {
-                       jww.FATAL.Printf("Fail to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())
-                       return HandledResult{err: err}
-               }
-               tmpContent = replaced[0]
-               tmpTableOfContents = replaced[1]
-       }
-
-       p.Content = helpers.BytesToHTML(tmpContent)
-       p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
-
-       return HandledResult{err: nil}
+       return commonConvert(p, t)
 }
 
 type mmarkHandler struct {
@@ -164,21 +105,27 @@ type mmarkHandler struct {
 
 func (h mmarkHandler) Extensions() []string { return []string{"mmark"} }
 func (h mmarkHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
+       return commonConvert(p, t)
+}
+
+func commonConvert(p *Page, t tpl.Template) HandledResult {
        p.ProcessShortcodes(t)
 
-       tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)))
+       var err error
+
+       renderedContent := p.renderContent(helpers.RemoveSummaryDivider(p.rawContent))
 
        if len(p.contentShortCodes) > 0 {
-               tmpContentWithTokensReplaced, err := replaceShortcodeTokens(tmpContent, shortcodePlaceholderPrefix, p.contentShortCodes)
+               renderedContent, err = replaceShortcodeTokens(renderedContent, shortcodePlaceholderPrefix, p.contentShortCodes)
 
                if err != nil {
-                       jww.FATAL.Printf("Fail to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())
+                       jww.FATAL.Printf("Failed to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())
                        return HandledResult{err: err}
-               } else {
-                       tmpContent = tmpContentWithTokensReplaced
                }
        }
 
+       tmpContent, tmpTableOfContents := helpers.ExtractTOC(renderedContent)
+
        p.Content = helpers.BytesToHTML(tmpContent)
        p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
 

diff --git a/hugolib/page.go b/hugolib/page.go
index a384c35e7d75d2b4f95b33375ea26710dc57f2f1..bbcca535ace8240fd958eb5274589cea73b23f0c 100644 (file)
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -204,6 +204,7 @@ func (p *Page) setSummary() {
                        p.Truncated = len(bytes.Trim(sections[1], " \n\r")) > 0
                }
 
+               // TODO(bep) consider doing this once only
                renderedHeader := p.renderBytes(header)
                if len(p.contentShortCodes) > 0 {
                        tmpContentWithTokensReplaced, err :=

diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go
index 3fa136173e2649ee92017ffdf848e6c8bec6b944..5fb9b7cc8cfaa1cd70c677f2e8b378b33ecd2b85 100644 (file)
--- a/hugolib/shortcode.go
+++ b/hugolib/shortcode.go
@@ -453,30 +453,14 @@ Loop:
 
 }
 
-// replaceShortcodeTokensInsources calls replaceShortcodeTokens for every source given.
-func replaceShortcodeTokensInsources(prefix string, replacements map[string]string, sources ...[]byte) (b [][]byte, err error) {
-       result := make([][]byte, len(sources))
-       for i, s := range sources {
-               b, err := replaceShortcodeTokens(s, prefix, replacements)
-
-               if err != nil {
-                       return nil, err
-               }
-               result[i] = b
-       }
-       return result, nil
-}
-
 // Replace prefixed shortcode tokens (HUGOSHORTCODE-1, HUGOSHORTCODE-2) with the real content.
+// Note: This function will rewrite the input slice.
 func replaceShortcodeTokens(source []byte, prefix string, replacements map[string]string) ([]byte, error) {
 
        if len(replacements) == 0 {
                return source, nil
        }
 
-       buff := bp.GetBuffer()
-       defer bp.PutBuffer(buff)
-
        sourceLen := len(source)
        start := 0
 
@@ -507,28 +491,14 @@ func replaceShortcodeTokens(source []byte, prefix string, replacements map[strin
                        }
                }
 
-               oldVal := source[j:end]
-               _, err := buff.Write(source[start:j])
-               if err != nil {
-                       return nil, errors.New("buff write failed")
-               }
-               _, err = buff.Write(newVal)
-               if err != nil {
-                       return nil, errors.New("buff write failed")
-               }
-               start = j + len(oldVal)
+               // This and other cool slice tricks: https://github.com/golang/go/wiki/SliceTricks
+               source = append(source[:j], append(newVal, source[end:]...)...)
 
                k = bytes.Index(source[start:], pre)
-       }
-       _, err := buff.Write(source[start:])
-       if err != nil {
-               return nil, errors.New("buff write failed")
-       }
 
-       bc := make([]byte, buff.Len(), buff.Len())
-       copy(bc, buff.Bytes())
+       }
 
-       return bc, nil
+       return source, nil
 }
 
 func getShortcodeTemplate(name string, t tpl.Template) *template.Template {

diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go
index c3917a00dbd900c0cff27561747aae1016f892ec..3527bbdf1db6647c12e9726e73cdf3afe24c215d 100644 (file)
--- a/hugolib/shortcode_test.go
+++ b/hugolib/shortcode_test.go
@@ -405,6 +405,14 @@ func TestReplaceShortcodeTokens(t *testing.T) {
                expect       interface{}
        }{
                {"Hello {@{@PREFIX-1@}@}.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "Hello World."},
+               {"Hello {@{@PREFIX-1@}@.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, false},
+               {"{@{@PREFIX2-1@}@}", "PREFIX2", map[string]string{"{@{@PREFIX2-1@}@}": "World"}, "World"},
+               {"Hello World!", "PREFIX2", map[string]string{}, "Hello World!"},
+               {"!{@{@PREFIX-1@}@}", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "!World"},
+               {"{@{@PREFIX-1@}@}!", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "World!"},
+               {"!{@{@PREFIX-1@}@}!", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "!World!"},
+               {"@{@PREFIX-1@}@}", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "@{@PREFIX-1@}@}"},
+               {"Hello {@{@PREFIX-1@}@}.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "To You My Old Friend Who Told Me This Fantastic Story"}, "Hello To You My Old Friend Who Told Me This Fantastic Story."},
                {"A {@{@A-1@}@} asdf {@{@A-2@}@}.", "A", map[string]string{"{@{@A-1@}@}": "v1", "{@{@A-2@}@}": "v2"}, "A v1 asdf v2."},
                {"Hello {@{@PREFIX2-1@}@}. Go {@{@PREFIX2-2@}@}, Go, Go {@{@PREFIX2-3@}@} Go Go!.", "PREFIX2", map[string]string{"{@{@PREFIX2-1@}@}": "Europe", "{@{@PREFIX2-2@}@}": "Jonny", "{@{@PREFIX2-3@}@}": "Johnny"}, "Hello Europe. Go Jonny, Go, Go Johnny Go Go!."},
                {"A {@{@PREFIX-2@}@} {@{@PREFIX-1@}@}.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "A", "{@{@PREFIX-2@}@}": "B"}, "A B A."},
@@ -420,11 +428,11 @@ func TestReplaceShortcodeTokens(t *testing.T) {
                {"Hello <p>{@{@PREFIX-1@}@}. END</p>.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "Hello <p>World. END</p>."},
                {"<p>Hello {@{@PREFIX-1@}@}</p>. END.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "<p>Hello World</p>. END."},
                {"Hello <p>{@{@PREFIX-1@}@}12", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "Hello <p>World12"},
-               // Make sure the buffering expands as needed
                {"Hello {@{@P-1@}@}. {@{@P-1@}@}-{@{@P-1@}@} {@{@P-1@}@} {@{@P-1@}@} {@{@P-1@}@} END", "P", map[string]string{"{@{@P-1@}@}": strings.Repeat("BC", 100)},
                        fmt.Sprintf("Hello %s. %s-%s %s %s %s END",
                                strings.Repeat("BC", 100), strings.Repeat("BC", 100), strings.Repeat("BC", 100), strings.Repeat("BC", 100), strings.Repeat("BC", 100), strings.Repeat("BC", 100))},
        } {
+
                results, err := replaceShortcodeTokens([]byte(this.input), this.prefix, this.replacements)
 
                if b, ok := this.expect.(bool); ok && !b {