parser/pageparser: Fix handling of commented out front matter

When the page parser was rewritten in 0.51, this was interpreted literally, but commented out front matter is used in the wild to "hide it from GitHub", e.g:

```
<!--
+++
title = "hello"
+++
-->
```

Fixes #5478

diff --git a/go.sum b/go.sum
index 9f5b5bea3069e940f0d10c931d054a9ec61058dc..951e4c6f924e2341f7a36e07e544e6fd98489d81 100644 (file)
--- a/go.sum
+++ b/go.sum
@@ -68,6 +68,7 @@ github.com/magefile/mage v1.4.0 h1:RI7B1CgnPAuu2O9lWszwya61RLmfL0KCdo+QyyI/Bhk=
 github.com/magefile/mage v1.4.0/go.mod h1:IUDi13rsHje59lecXokTfGX0QIzO45uVPlXnJYsXepA=
 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6 h1:LZhVjIISSbj8qLf2qDPP0D8z0uvOWAW5C85ly5mJW6c=
 github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6/go.mod h1:oTeZL2KHA7CUX6X+fovmK9OvIOFuqu0TwdQrZjLTh88=
 github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs=
 github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=

diff --git a/hugolib/page_content.go b/hugolib/page_content.go
index 75124a1df9e6bfa4bbbd96b4011f3cdee9529ef2..a363249d5fd748d6f5f62e5dad8943ce77523163 100644 (file)
--- a/hugolib/page_content.go
+++ b/hugolib/page_content.go
@@ -83,8 +83,6 @@ Loop:
 
                switch {
                case it.Type == pageparser.TypeIgnore:
-               case it.Type == pageparser.TypeHTMLComment:
-                       // Ignore. This is only a leading Front matter comment.
                case it.Type == pageparser.TypeHTMLStart:
                        // This is HTML without front matter. It can still have shortcodes.
                        p.renderable = false

diff --git a/hugolib/page_test.go b/hugolib/page_test.go
index f9c26754b0627c01f7983ec272a1e0633e87057d..cd8bd8ffc53e27ad5c7a277c51475148784aa96f 100644 (file)
--- a/hugolib/page_test.go
+++ b/hugolib/page_test.go
@@ -1586,6 +1586,33 @@ CONTENT:{{ .Content }}
        )
 }
 
+// https://github.com/gohugoio/hugo/issues/5478
+func TestPageWithCommentedOutFrontMatter(t *testing.T) {
+       b := newTestSitesBuilder(t)
+       b.WithSimpleConfigFile()
+
+       b.WithContent("page.md", `<!--
++++
+title = "hello"
++++
+-->
+This is the content.
+`)
+
+       b.WithTemplatesAdded("layouts/_default/single.html", `
+Title: {{ .Title }}
+Content:{{ .Content }}
+`)
+
+       b.CreateSites().Build(BuildCfg{})
+
+       b.AssertFileContent("public/page/index.html",
+               "Title: hello",
+               "Content:<p>This is the content.</p>",
+       )
+
+}
+
 // TODO(bep) this may be useful for other tests.
 func compareObjects(a interface{}, b interface{}) bool {
        aStr := strings.Split(fmt.Sprintf("%v", a), "")

diff --git a/parser/pageparser/item.go b/parser/pageparser/item.go
index 644c20e873273ca44d1f43263efea20c42129a79..b9d47b16e25711f0c7b0efabd11d9a3c3b599620 100644 (file)
--- a/parser/pageparser/item.go
+++ b/parser/pageparser/item.go
@@ -108,7 +108,6 @@ const (
 
        // page items
        TypeHTMLStart          // document starting with < as first non-whitespace
-       TypeHTMLComment        // We ignore leading comments
        TypeLeadSummaryDivider // <!--more-->,  # more
        TypeFrontMatterYAML
        TypeFrontMatterTOML

diff --git a/parser/pageparser/pagelexer.go b/parser/pageparser/pagelexer.go
index 94c1ff26bc9db2a082ab786ac6471dc468ecef44..5802c318bb95e841ddbc17aff20f94fd5a890ed7 100644 (file)
--- a/parser/pageparser/pagelexer.go
+++ b/parser/pageparser/pagelexer.go
@@ -53,6 +53,8 @@ type pageLexer struct {
        summaryDivider []byte
        // Set when we have parsed any summary divider
        summaryDividerChecked bool
+       // Whether we're in a HTML comment.
+       isInHTMLComment bool
 
        lexerShortcodeState
 
@@ -120,7 +122,7 @@ var (
        delimYAML         = []byte("---")
        delimOrg          = []byte("#+")
        htmlCommentStart  = []byte("<!--")
-       htmlCOmmentEnd    = []byte("-->")
+       htmlCommentEnd    = []byte("-->")
 )
 
 func (l *pageLexer) next() rune {
@@ -195,6 +197,15 @@ func (l *pageLexer) consumeCRLF() bool {
        return consumed
 }
 
+func (l *pageLexer) consumeToNextLine() {
+       for {
+               r := l.next()
+               if r == eof || isEndOfLine(r) {
+                       return
+               }
+       }
+}
+
 func (l *pageLexer) consumeSpace() {
        for {
                r := l.next()
@@ -206,6 +217,10 @@ func (l *pageLexer) consumeSpace() {
 }
 
 func lexMainSection(l *pageLexer) stateFunc {
+       if l.isInHTMLComment {
+               return lexEndFromtMatterHTMLComment
+       }
+
        // Fast forward as far as possible.
        var l1, l2 int
 
@@ -312,16 +327,15 @@ LOOP:
                case r == byteOrderMark:
                        l.emit(TypeIgnore)
                case !isSpace(r) && !isEndOfLine(r):
-                       // No front matter.
                        if r == '<' {
                                l.backup()
                                if l.hasPrefix(htmlCommentStart) {
-                                       right := l.index(htmlCOmmentEnd)
-                                       if right == -1 {
-                                               return l.errorf("starting HTML comment with no end")
-                                       }
-                                       l.pos += right + len(htmlCOmmentEnd)
-                                       l.emit(TypeHTMLComment)
+                                       // This may be commented out front mattter, which should
+                                       // still be read.
+                                       l.consumeToNextLine()
+                                       l.isInHTMLComment = true
+                                       l.emit(TypeIgnore)
+                                       continue LOOP
                                } else {
                                        if l.pos > l.start {
                                                l.emit(tText)
@@ -341,6 +355,19 @@ LOOP:
        return lexMainSection
 }
 
+func lexEndFromtMatterHTMLComment(l *pageLexer) stateFunc {
+       l.isInHTMLComment = false
+       right := l.index(htmlCommentEnd)
+       if right == -1 {
+               return l.errorf("starting HTML comment with no end")
+       }
+       l.pos += right + len(htmlCommentEnd)
+       l.emit(TypeIgnore)
+
+       // Now move on to the shortcodes.
+       return lexMainSection
+}
+
 func lexDone(l *pageLexer) stateFunc {
 
        // Done!

diff --git a/parser/pageparser/pageparser_intro_test.go b/parser/pageparser/pageparser_intro_test.go
index ba48a3ee3f35c1f848bcd1a7da4e31c7350cd57d..901b216c8aa86a20e0ca739363edb7bc8e9b4f9a 100644 (file)
--- a/parser/pageparser/pageparser_intro_test.go
+++ b/parser/pageparser/pageparser_intro_test.go
@@ -60,7 +60,8 @@ var frontMatterTests = []lexerTest{
        {"No front matter", "\nSome text.\n", []Item{tstSomeText, tstEOF}},
        {"YAML front matter", "---\nfoo: \"bar\"\n---\n\nSome text.\n", []Item{tstFrontMatterYAML, tstSomeText, tstEOF}},
        {"YAML empty front matter", "---\n---\n\nSome text.\n", []Item{nti(TypeFrontMatterYAML, ""), tstSomeText, tstEOF}},
-       {"YAML commented out front matter", "<!--\n---\nfoo: \"bar\"\n---\n-->\nSome text.\n", []Item{nti(TypeHTMLComment, "<!--\n---\nfoo: \"bar\"\n---\n-->"), tstSomeText, tstEOF}},
+       {"YAML commented out front matter", "<!--\n---\nfoo: \"bar\"\n---\n-->\nSome text.\n", []Item{nti(TypeIgnore, "<!--\n"), tstFrontMatterYAML, nti(TypeIgnore, "-->"), tstSomeText, tstEOF}},
+       {"YAML commented out front matter, no end", "<!--\n---\nfoo: \"bar\"\n---\nSome text.\n", []Item{nti(TypeIgnore, "<!--\n"), tstFrontMatterYAML, nti(tError, "starting HTML comment with no end")}},
        // Note that we keep all bytes as they are, but we need to handle CRLF
        {"YAML front matter CRLF", "---\r\nfoo: \"bar\"\r\n---\n\nSome text.\n", []Item{tstFrontMatterYAMLCRLF, tstSomeText, tstEOF}},
        {"TOML front matter", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, tstEOF}},
@@ -78,7 +79,6 @@ var frontMatterTests = []lexerTest{
 func TestFrontMatter(t *testing.T) {
        t.Parallel()
        for i, test := range frontMatterTests {
-
                items := collect([]byte(test.input), false, lexIntroSection)
                if !equal(items, test.items) {
                        got := crLfReplacer.Replace(fmt.Sprint(items))