hugolib: Use []byte in shortcode parsing

See #5324

diff --git a/hugolib/page.go b/hugolib/page.go
index 0359769e3dfb237afb0abc5d729147317ecce15c..e867dd525607d2cdb0409b9eec220a0ac8f26017 100644 (file)
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -1871,7 +1871,7 @@ func (p *Page) SaveSource() error {
 // TODO(bep) lazy consolidate
 func (p *Page) processShortcodes() error {
        p.shortcodeState = newShortcodeHandler(p)
-       tmpContent, err := p.shortcodeState.extractShortcodes(string(p.workContent), p.withoutContent())
+       tmpContent, err := p.shortcodeState.extractShortcodes(p.workContent, p.withoutContent())
        if err != nil {
                return err
        }

diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go
index f7141031d2cd860b85b57cd2336915b36c7e0acd..a21a10ad242cff9655315b347d80ef0218f0ec43 100644 (file)
--- a/hugolib/shortcode.go
+++ b/hugolib/shortcode.go
@@ -553,9 +553,9 @@ Loop:
 
                        return sc, nil
                case currItem.IsText():
-                       sc.inner = append(sc.inner, currItem.Val)
+                       sc.inner = append(sc.inner, currItem.ValStr())
                case currItem.IsShortcodeName():
-                       sc.name = currItem.Val
+                       sc.name = currItem.ValStr()
                        // We pick the first template for an arbitrary output format
                        // if more than one. It is "all inner or no inner".
                        tmpl := getShortcodeTemplateForTemplateKey(scKey{}, sc.name, p.s.Tmpl)
@@ -576,11 +576,11 @@ Loop:
                                // named params
                                if sc.params == nil {
                                        params := make(map[string]string)
-                                       params[currItem.Val] = pt.Next().Val
+                                       params[currItem.ValStr()] = pt.Next().ValStr()
                                        sc.params = params
                                } else {
                                        if params, ok := sc.params.(map[string]string); ok {
-                                               params[currItem.Val] = pt.Next().Val
+                                               params[currItem.ValStr()] = pt.Next().ValStr()
                                        } else {
                                                return sc, errShortCodeIllegalState
                                        }
@@ -590,11 +590,11 @@ Loop:
                                // positional params
                                if sc.params == nil {
                                        var params []string
-                                       params = append(params, currItem.Val)
+                                       params = append(params, currItem.ValStr())
                                        sc.params = params
                                } else {
                                        if params, ok := sc.params.([]string); ok {
-                                               params = append(params, currItem.Val)
+                                               params = append(params, currItem.ValStr())
                                                sc.params = params
                                        } else {
                                                return sc, errShortCodeIllegalState
@@ -613,19 +613,21 @@ Loop:
        return sc, nil
 }
 
-func (s *shortcodeHandler) extractShortcodes(stringToParse string, p *PageWithoutContent) (string, error) {
+var shortCodeStart = []byte("{{")
 
-       startIdx := strings.Index(stringToParse, "{{")
+func (s *shortcodeHandler) extractShortcodes(input []byte, p *PageWithoutContent) (string, error) {
+
+       startIdx := bytes.Index(input, shortCodeStart)
 
        // short cut for docs with no shortcodes
        if startIdx < 0 {
-               return stringToParse, nil
+               return string(input), nil
        }
 
        // the parser takes a string;
        // since this is an internal API, it could make sense to use the mutable []byte all the way, but
        // it seems that the time isn't really spent in the byte copy operations, and the impl. gets a lot cleaner
-       pt := pageparser.ParseFrom(stringToParse, startIdx)
+       pt := pageparser.ParseFrom(input, startIdx)
 
        result := bp.GetBuffer()
        defer bp.PutBuffer(result)
@@ -642,7 +644,7 @@ Loop:
 
                switch {
                case currItem.IsText():
-                       result.WriteString(currItem.Val)
+                       result.WriteString(currItem.ValStr())
                case currItem.IsLeftShortcodeDelim():
                        // let extractShortcode handle left delim (will do so recursively)
                        pt.Backup()

diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go
index 3385d31f0a550959c03eb683ae5b094cc3be2536..f8837810c91d1e1158a07e89cbe587393f7fea89 100644 (file)
--- a/hugolib/shortcode_test.go
+++ b/hugolib/shortcode_test.go
@@ -424,7 +424,7 @@ func TestExtractShortcodes(t *testing.T) {
                        return fmt.Sprintf("HAHA%s-%dHBHB", shortcodePlaceholderPrefix, counter)
                }
 
-               content, err := s.extractShortcodes(this.input, p.withoutContent())
+               content, err := s.extractShortcodes([]byte(this.input), p.withoutContent())
 
                if b, ok := this.expect.(bool); ok && !b {
                        if err == nil {

diff --git a/parser/pageparser/item.go b/parser/pageparser/item.go
index 35bc8e2687db01fe77c84f539cade37b48b80fea..6e93bb696d455833f08b24408f807ba17ad7d9a6 100644 (file)
--- a/parser/pageparser/item.go
+++ b/parser/pageparser/item.go
@@ -21,6 +21,10 @@ type Item struct {
        Val []byte
 }
 
+func (i Item) ValStr() string {
+       return string(i.Val)
+}
+
 func (i Item) IsText() bool {
        return i.typ == tText
 }

diff --git a/parser/pageparser/pagelexer.go b/parser/pageparser/pagelexer.go
index 3bdfb6c336c6cef56b8657152df72116995aa1e1..c15e977ca31a98405c54745bcc108c36d1161499 100644 (file)
--- a/parser/pageparser/pagelexer.go
+++ b/parser/pageparser/pagelexer.go
@@ -60,17 +60,6 @@ type pageLexer struct {
        items []Item
 }
 
-func Parse(s string) *Tokens {
-       return ParseFrom(s, 0)
-}
-
-func ParseFrom(s string, from int) *Tokens {
-       input := []byte(s)
-       lexer := newPageLexer(input, pos(from), lexMainSection) // TODO(bep) 2errors
-       lexer.run()
-       return &Tokens{lexer: lexer}
-}
-
 // note: the input position here is normally 0 (start), but
 // can be set if position of first shortcode is known
 // TODO(bep) 2errors byte

diff --git a/parser/pageparser/pageparser.go b/parser/pageparser/pageparser.go
index 5534ee64b31ae15aced13b57f432a9b53f0145fc..948c05edf280e29f39a71f01a8b3eec008c6c436 100644 (file)
--- a/parser/pageparser/pageparser.go
+++ b/parser/pageparser/pageparser.go
@@ -17,7 +17,15 @@
 // See slides here: http://cuddle.googlecode.com/hg/talk/lex.html
 package pageparser
 
-// The lexical scanning below
+func Parse(input []byte) *Tokens {
+       return ParseFrom(input, 0)
+}
+
+func ParseFrom(input []byte, from int) *Tokens {
+       lexer := newPageLexer(input, pos(from), lexMainSection) // TODO(bep) 2errors
+       lexer.run()
+       return &Tokens{lexer: lexer}
+}
 
 type Tokens struct {
        lexer     *pageLexer