Add transform.HTMLToMarkdown

Fixes #13946

diff --git a/go.mod b/go.mod
index b2ab752c38aeef2251f60e787ee1d7ca2f6cb4e4..a916bf939863944413b0fc84bc3231f293de5a8e 100644 (file)
--- a/go.mod
+++ b/go.mod
@@ -105,6 +105,8 @@ require (
        github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect
        github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect
        github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect
+       github.com/JohannesKaufmann/dom v0.2.0 // indirect
+       github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0 // indirect
        github.com/aws/aws-sdk-go v1.55.7 // indirect
        github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 // indirect
        github.com/aws/aws-sdk-go-v2/config v1.29.17 // indirect
@@ -162,7 +164,7 @@ require (
        github.com/perimeterx/marshmallow v1.1.5 // indirect
        github.com/pkg/errors v0.9.1 // indirect
        github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
-       github.com/rivo/uniseg v0.2.0 // indirect
+       github.com/rivo/uniseg v0.4.7 // indirect
        github.com/russross/blackfriday/v2 v2.1.0 // indirect
        github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
        github.com/woodsbury/decimal128 v1.3.0 // indirect

diff --git a/go.sum b/go.sum
index 463815e4c6612ebd612ab604bddf3c74e6986d07..8e0a8591060dec17554e56a3473fa3e9b9345bca 100644 (file)
--- a/go.sum
+++ b/go.sum
@@ -91,6 +91,10 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo=
+github.com/JohannesKaufmann/dom v0.2.0 h1:1bragmEb19K8lHAqgFgqCpiPCFEZMTXzOIEjuxkUfLQ=
+github.com/JohannesKaufmann/dom v0.2.0/go.mod h1:57iSUl5RKric4bUkgos4zu6Xt5LMHUnw3TF1l5CbGZo=
+github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0 h1:C0/TerKdQX9Y9pbYi1EsLr5LDNANsqunyI/btpyfCg8=
+github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0/go.mod h1:OLaKh+giepO8j7teevrNwiy/fwf8LXgoc9g7rwaE1jk=
 github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
 github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
 github.com/alecthomas/chroma/v2 v2.20.0 h1:sfIHpxPyR07/Oylvmcai3X/exDlE8+FA820NTz+9sGw=
@@ -465,6 +469,8 @@ github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhi
 github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=

diff --git a/tpl/transform/transform.go b/tpl/transform/transform.go
index 380ea252bda1b6a061fc10176978559a6686b875..b5a2d3b4c8c8bb96bd558eb19199e9c3a0569ba0 100644 (file)
--- a/tpl/transform/transform.go
+++ b/tpl/transform/transform.go
@@ -27,6 +27,11 @@ import (
        "strings"
        "sync/atomic"
 
+       htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2/converter"
+       "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base"
+       "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark"
+       "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/table"
+
        bp "github.com/gohugoio/hugo/bufferpool"
 
        "github.com/bep/goportabletext"
@@ -324,3 +329,32 @@ func (ns *Namespace) ToMath(ctx context.Context, args ...any) (template.HTML, er
 func (ns *Namespace) Reset() {
        ns.cacheUnmarshal.Clear()
 }
+
+// This was added in Hugo v0.151.0 and should be considered experimental for now.
+// We need to test this out in the wild for a while before committing to this API,
+// and there will eventually be more options here.
+func (ns *Namespace) HTMLToMarkdown(ctx context.Context, args ...any) (string, error) {
+       if len(args) < 1 {
+               return "", errors.New("must provide at least one argument")
+       }
+       input, err := cast.ToStringE(args[0])
+       if err != nil {
+               return "", err
+       }
+
+       plugins := []htmltomarkdown.Plugin{
+               base.NewBasePlugin(),
+               commonmark.NewCommonmarkPlugin(),
+               table.NewTablePlugin(),
+       }
+
+       conv := htmltomarkdown.NewConverter(
+               htmltomarkdown.WithPlugins(plugins...),
+       )
+
+       markdown, err := conv.ConvertString(input)
+       if err != nil {
+               return "", err
+       }
+       return markdown, nil
+}

diff --git a/tpl/transform/transform_integration_test.go b/tpl/transform/transform_integration_test.go
index 8197b3e3d9ac0524789706dad7b7b66dd142d94e..51c2f5fb1d04db7d83ec30451c78633bc2107c49 100644 (file)
--- a/tpl/transform/transform_integration_test.go
+++ b/tpl/transform/transform_integration_test.go
@@ -535,3 +535,59 @@ disableKinds = ['page','rss','section','sitemap','taxonomy','term']
        b, err = hugolib.TestE(t, f)
        b.Assert(err.Error(), qt.Contains, "invalid strict mode")
 }
+
+func TestHTMLToMarkdown(t *testing.T) {
+       t.Parallel()
+
+       markdown := `
+# Heading
+
+Some **bold** text.
+
+A [link](https://example.com).
+
+An image:
+
+![alt text](https://example.com/image.jpg "Image Title")
+
+A list:
+
+- Item 1
+- Item 2
+  - Item 2a
+  - Item 2b
+
+A table:
+
+| Header 1 | Header 2 |
+|----------|----------|
+| Cell 1   | Cell 2   |
+| Cell 3   | Cell 4   |
+
+A blockquote:
+
+> This is a quote.
+       `
+       files := `
+-- hugo.toml --
+disableKinds = ['rss','section','sitemap','taxonomy','term']
+-- layouts/all.html --
+All html.
+-- layouts/all.markdown --
+{{ .Content | transform.HTMLToMarkdown | safeHTML }}
+-- content/p1.md --
+---
+title: p1
+outputs: ["html", "markdown"]
+---
+`
+
+       files += markdown
+
+       b := hugolib.Test(t, files)
+
+       b.AssertFileContent("public/p1/index.html", `All html.`)
+
+       // There are some white space differences, so we cannot do an exact match.
+       b.AssertFileContent("public/p1/index.md", markdown)
+}