markup: Reimplement pygmentsCodefencesGuessSyntax

Fixes #6565

diff --git a/go.mod b/go.mod
index 3e4628f571e6ae8c6a46e48108b555ee530ecb95..787bb27d5111f07dc2f658b6d0c56280bc5b43ec 100644 (file)
--- a/go.mod
+++ b/go.mod
@@ -54,8 +54,8 @@ require (
        github.com/spf13/viper v1.4.0
        github.com/tdewolff/minify/v2 v2.6.1
        github.com/yosssi/ace v0.0.5
-       github.com/yuin/goldmark v1.1.10
-       github.com/yuin/goldmark-highlighting v0.0.0-20191124122839-ede94e40cc3a
+       github.com/yuin/goldmark v1.1.11
+       github.com/yuin/goldmark-highlighting v0.0.0-20191202084645-78f32c8dd6d5
        go.opencensus.io v0.22.0 // indirect
        gocloud.dev v0.15.0
        golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff

diff --git a/go.sum b/go.sum
index c682bef3d70c613e2432b364897f72dbd04ac17d..f6d64a4e733f9da1d44434f7cb756ca2b676d122 100644 (file)
--- a/go.sum
+++ b/go.sum
@@ -359,8 +359,14 @@ github.com/yuin/goldmark v1.1.8 h1:d0m8Ac9JaetYjPZLC4P4W32ac7I0lpJpQbvxZtFqBoM=
 github.com/yuin/goldmark v1.1.8/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.10 h1:bg3TC1aj4DbjGdhvjSSffGfAgVUdBEIpccuCozwOYWo=
 github.com/yuin/goldmark v1.1.10/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.11 h1:OO08ilczi3F4swaYWPB99s08WRxP9DdLBemiLFQ6vCo=
+github.com/yuin/goldmark v1.1.11/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark-highlighting v0.0.0-20191124122839-ede94e40cc3a h1:L7FTUnbc0WEBqGWgjbx4sPNAOX1/q5W/3KCD6g8XkKo=
 github.com/yuin/goldmark-highlighting v0.0.0-20191124122839-ede94e40cc3a/go.mod h1:1gshkGdH4gcrIH5MGSScGH42rOOCO+4Ks6acjAkA9C0=
+github.com/yuin/goldmark-highlighting v0.0.0-20191126180129-d7a4bf4d7ea4 h1:vI4Jv29V1cMPqetuLPMW1CMB9xNgxsHVBo8Mid6bwH8=
+github.com/yuin/goldmark-highlighting v0.0.0-20191126180129-d7a4bf4d7ea4/go.mod h1:4QGn5rJFOASBa2uK4Q2h3BRTyJqRfsAucPFIipSTcaM=
+github.com/yuin/goldmark-highlighting v0.0.0-20191202084645-78f32c8dd6d5 h1:QbH7ca1qtgZHrzvcVAEoiJIwBqrXxMOfHYfwZIniIK0=
+github.com/yuin/goldmark-highlighting v0.0.0-20191202084645-78f32c8dd6d5/go.mod h1:4QGn5rJFOASBa2uK4Q2h3BRTyJqRfsAucPFIipSTcaM=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.mongodb.org/mongo-driver v1.0.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=

diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go
index e022cb2d3111ff1e4246072a7cd5d683fbae56c2..feee859105eb1348bd31eaa79d28d8f403a37575 100644 (file)
--- a/hugolib/hugo_sites_build_test.go
+++ b/hugolib/hugo_sites_build_test.go
@@ -671,7 +671,7 @@ END
 
        b.CreateSites().Build(BuildCfg{})
 
-       contentMatchers := []string{"<h2 id=\"another-header\">Another header</h2>", "<h2 id=\"another-header99\">Another header</h2>", "<p>The End.</p>"}
+       contentMatchers := []string{"<h2 id=\"another-header\">Another header</h2>", "<h2 id=\"another-header-99\">Another header</h2>", "<p>The End.</p>"}
 
        for i := 1; i <= numPages; i++ {
                if i%3 != 0 {
@@ -691,13 +691,13 @@ END
                checkContent(b, fmt.Sprintf("public/%s/page%d/index.json", section, i), contentMatchers...)
        }
 
-       checkContent(b, "public/s1/index.html", "P: s1/_index.md\nList: 10|List Content: 8033\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8033\n\nRender 1: View: 8033\n\nRender 2: View: 8033\n\nRender 3: View: 8033\n\nRender 4: View: 8033\n\nEND\n")
-       checkContent(b, "public/s2/index.html", "P: s2/_index.md\nList: 10|List Content: 8033", "Render 4: View: 8033\n\nEND")
-       checkContent(b, "public/index.html", "P: _index.md\nList: 10|List Content: 8033", "4: View: 8033\n\nEND")
+       checkContent(b, "public/s1/index.html", "P: s1/_index.md\nList: 10|List Content: 8132\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8132\n\nRender 1: View: 8132\n\nRender 2: View: 8132\n\nRender 3: View: 8132\n\nRender 4: View: 8132\n\nEND\n")
+       checkContent(b, "public/s2/index.html", "P: s2/_index.md\nList: 10|List Content: 8132", "Render 4: View: 8132\n\nEND")
+       checkContent(b, "public/index.html", "P: _index.md\nList: 10|List Content: 8132", "4: View: 8132\n\nEND")
 
        // Check paginated pages
        for i := 2; i <= 9; i++ {
-               checkContent(b, fmt.Sprintf("public/page/%d/index.html", i), fmt.Sprintf("Page: %d", i), "Content: 8033\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8033", "Render 4: View: 8033\n\nEND")
+               checkContent(b, fmt.Sprintf("public/page/%d/index.html", i), fmt.Sprintf("Page: %d", i), "Content: 8132\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8132", "Render 4: View: 8132\n\nEND")
        }
 }
 

diff --git a/hugolib/page_test.go b/hugolib/page_test.go
index 58433fdc524186c17ccdc93280efbbc55f3a7e99..dc8bc821c15f55bd511feb2ac7fc881ea0bd1a78 100644 (file)
--- a/hugolib/page_test.go
+++ b/hugolib/page_test.go
@@ -1658,7 +1658,7 @@ $$$
 
        b.AssertFileContent("public/page/index.html",
                `<nav id="TableOfContents">`,
-               `<li><a href="#shortcode-tshort-in-header">Shortcode T-SHORT in header</a></li>`,
+               `<li><a href="#shortcode-t-short-in-header">Shortcode T-SHORT in header</a></li>`,
                `<code class="language-bash" data-lang="bash"><span class="hl">SHORT`,
                `<code class="language-bash" data-lang="bash"><span class="hl">MARKDOWN`)
 }

diff --git a/markup/goldmark/convert.go b/markup/goldmark/convert.go
index 16728683110bcd9519be13946d68b661110df8a1..15b0f0d77c8129a6015198f21e0b73fde3fc8ac4 100644 (file)
--- a/markup/goldmark/convert.go
+++ b/markup/goldmark/convert.go
@@ -198,6 +198,7 @@ func newHighlighting(cfg highlight.Config) goldmark.Extender {
 
        e := hl.NewHighlighting(
                hl.WithStyle(cfg.Style),
+               hl.WithGuessLanguage(cfg.GuessSyntax),
                hl.WithCodeBlockOptions(highlight.GetCodeBlockOptions()),
                hl.WithFormatOptions(
                        cfg.ToHTMLOptions()...,

diff --git a/markup/goldmark/convert_test.go b/markup/goldmark/convert_test.go
index a23976fd665ad379d1e967be74ea3a1b8cc40ed7..b6816d2e54a7037168c70b9f97efb66e2d655784 100644 (file)
--- a/markup/goldmark/convert_test.go
+++ b/markup/goldmark/convert_test.go
@@ -224,4 +224,25 @@ LINE5
                result = convertForConfig(c, cfg, lines, "bash {linenos=table}")
                c.Assert(result, qt.Contains, "<span class=\"lnt\">1\n</span>")
        })
+
+       c.Run("No language", func(c *qt.C) {
+               cfg := highlight.DefaultConfig
+               cfg.NoClasses = false
+               cfg.LineNos = true
+               cfg.LineNumbersInTable = false
+
+               result := convertForConfig(c, cfg, lines, "")
+               c.Assert(result, qt.Contains, "<pre><code>LINE1\n")
+       })
+
+       c.Run("No language, guess syntax", func(c *qt.C) {
+               cfg := highlight.DefaultConfig
+               cfg.NoClasses = false
+               cfg.GuessSyntax = true
+               cfg.LineNos = true
+               cfg.LineNumbersInTable = false
+
+               result := convertForConfig(c, cfg, lines, "")
+               c.Assert(result, qt.Contains, "<span class=\"ln\">2</span>LINE2\n<")
+       })
 }

diff --git a/markup/highlight/config.go b/markup/highlight/config.go
index 56e38fd85e93c5c7692719d4fa93aa14d2fed42c..3f31e65eab6fdf9873775599c7002679cea395f0 100644 (file)
--- a/markup/highlight/config.go
+++ b/markup/highlight/config.go
@@ -58,6 +58,8 @@ type Config struct {
 
        // TabWidth sets the number of characters for a tab. Defaults to 4.
        TabWidth int
+
+       GuessSyntax bool
 }
 
 func (cfg Config) ToHTMLOptions() []html.Option {
@@ -104,6 +106,10 @@ func ApplyLegacyConfig(cfg config.Provider, conf *Config) error {
                conf.CodeFences = cfg.GetBool("pygmentsCodeFences")
        }
 
+       if conf.GuessSyntax == DefaultConfig.GuessSyntax && cfg.IsSet("pygmentsCodefencesGuessSyntax") {
+               conf.GuessSyntax = cfg.GetBool("pygmentsCodefencesGuessSyntax")
+       }
+
        if cfg.IsSet("pygmentsOptions") {
                if err := applyOptionsFromString(cfg.GetString("pygmentsOptions"), conf); err != nil {
                        return err

diff --git a/markup/highlight/highlight.go b/markup/highlight/highlight.go
index 322bde1ef52d9f90c058af03ce34edd1efe35f19..9e26aaf844e05a72e209a6fd839471e550695729 100644 (file)
--- a/markup/highlight/highlight.go
+++ b/markup/highlight/highlight.go
@@ -52,6 +52,14 @@ func highlight(code, lang string, cfg Config) (string, error) {
                lexer = lexers.Get(lang)
        }
 
+       if lexer == nil && cfg.GuessSyntax {
+               lexer = lexers.Analyse(code)
+               if lexer == nil {
+                       lexer = lexers.Fallback
+               }
+               lang = strings.ToLower(lexer.Config().Name)
+       }
+
        if lexer == nil {
                wrapper := getPreWrapper(lang)
                fmt.Fprint(w, wrapper.Start(true, ""))

diff --git a/markup/highlight/highlight_test.go b/markup/highlight/highlight_test.go
index 58bd9c119f1dbeb6a820e0dcbc977b2ed3a4b82b..6da2924896db334f46044eda36f7f5cc3c2e2594 100644 (file)
--- a/markup/highlight/highlight_test.go
+++ b/markup/highlight/highlight_test.go
@@ -84,4 +84,26 @@ LINE5
                result, _ = h.Highlight(lines, "bash", "linenos=table")
                c.Assert(result, qt.Contains, "<span class=\"lnt\">1\n</span>")
        })
+
+       c.Run("No language", func(c *qt.C) {
+               cfg := DefaultConfig
+               cfg.NoClasses = false
+               cfg.LineNos = true
+               h := New(cfg)
+
+               result, _ := h.Highlight(lines, "", "")
+               c.Assert(result, qt.Equals, "<pre><code>LINE1\nLINE2\nLINE3\nLINE4\nLINE5\n</code></pre>")
+       })
+
+       c.Run("No language, guess syntax", func(c *qt.C) {
+               cfg := DefaultConfig
+               cfg.NoClasses = false
+               cfg.GuessSyntax = true
+               cfg.LineNos = true
+               cfg.LineNumbersInTable = false
+               h := New(cfg)
+
+               result, _ := h.Highlight(lines, "", "")
+               c.Assert(result, qt.Contains, "<span class=\"ln\">2</span>LINE2\n<")
+       })
 }

diff --git a/markup/markup_config/config_test.go b/markup/markup_config/config_test.go
index 726d1146b04fdd2c6b343f89f7db8a2488773c4c..22f0ab1d483278992861414cdb66882bf7b57334 100644 (file)
--- a/markup/markup_config/config_test.go
+++ b/markup/markup_config/config_test.go
@@ -55,7 +55,7 @@ func TestConfig(t *testing.T) {
                v.Set("footnoteAnchorPrefix", "myprefix")
                v.Set("footnoteReturnLinkContents", "myreturn")
                v.Set("pygmentsStyle", "hugo")
-
+               v.Set("pygmentsCodefencesGuessSyntax", true)
                conf, err := Decode(v)
 
                c.Assert(err, qt.IsNil)
@@ -64,6 +64,7 @@ func TestConfig(t *testing.T) {
                c.Assert(conf.BlackFriday.FootnoteReturnLinkContents, qt.Equals, "myreturn")
                c.Assert(conf.Highlight.Style, qt.Equals, "hugo")
                c.Assert(conf.Highlight.CodeFences, qt.Equals, true)
+               c.Assert(conf.Highlight.GuessSyntax, qt.Equals, true)
        })
 
 }