Chomp Unicode BOM if present

Useful if using or sharing files with users that use editors that
append a unicode byte order marker header (like Windows notepad).

This will still assume files are UTF-8 encoded.

Closes #2075

diff --git a/hugolib/page_test.go b/hugolib/page_test.go
index c4e442a414f2d73596bc61015e4fa6026f6ccdd4..876a74caf7bd73d2a957470edc293aa1ea7eb2fd 100644 (file)
--- a/hugolib/page_test.go
+++ b/hugolib/page_test.go
@@ -1052,6 +1052,19 @@ func TestPageSimpleMethods(t *testing.T) {
        }
 }
 
+func TestChompBOM(t *testing.T) {
+       p, _ := NewPage("simple.md")
+       const utf8BOM = "\xef\xbb\xbf"
+       _, err := p.ReadFrom(strings.NewReader(utf8BOM + simplePage))
+       p.Convert()
+
+       if err != nil {
+               t.Fatalf("Unable to create a page with BOM prefixed frontmatter and body content: %s", err)
+       }
+
+       checkPageTitle(t, p, "Simple")
+}
+
 func listEqual(left, right []string) bool {
        if len(left) != len(right) {
                return false

diff --git a/parser/page.go b/parser/page.go
index 949bfdac6489ed1f7cc309eb8b487523b21c98c9..5092caddcc63b446e4f133465cbd8f72f0b1e307 100644 (file)
--- a/parser/page.go
+++ b/parser/page.go
@@ -50,6 +50,8 @@ const (
        HTMLCommentStart = "<!--"
        // HTMLCommentEnd identifies the end of HTML comment.
        HTMLCommentEnd = "-->"
+       // BOM Unicode byte order marker
+       BOM = '\ufeff'
 )
 
 var (
@@ -101,6 +103,10 @@ func (p *page) Metadata() (meta interface{}, err error) {
 func ReadFrom(r io.Reader) (p Page, err error) {
        reader := bufio.NewReader(r)
 
+       // chomp BOM and assume UTF-8
+       if err = chompBOM(reader); err != nil && err != io.EOF {
+               return
+       }
        if err = chompWhitespace(reader); err != nil && err != io.EOF {
                return
        }
@@ -135,6 +141,19 @@ func ReadFrom(r io.Reader) (p Page, err error) {
        return newp, nil
 }
 
+func chompBOM(r io.RuneScanner) (err error) {
+       for {
+               c, _, err := r.ReadRune()
+               if err != nil {
+                       return err
+               }
+               if c != BOM {
+                       r.UnreadRune()
+                       return nil
+               }
+       }
+}
+
 func chompWhitespace(r io.RuneScanner) (err error) {
        for {
                c, _, err := r.ReadRune()