Remove accents in URLs

So the taxonomy `Gérard Depardieu` gives paths on the form `gerard-depardieu`.

Unfortunately this introduces two imports from the `golang.org/`, but Unicode-normalization isn't something we'd want to write from scratch.

See https://blog.golang.org/normalization

See #1180

diff --git a/helpers/path.go b/helpers/path.go
index 9f47549d267b3f5ba170be7784602675130d3c3a..5e5e3fa897cb0b73850afcb649f265cedc8b1a2d 100644 (file)
--- a/helpers/path.go
+++ b/helpers/path.go
@@ -19,6 +19,8 @@ import (
        "github.com/spf13/afero"
        jww "github.com/spf13/jwalterweatherman"
        "github.com/spf13/viper"
+       "golang.org/x/text/transform"
+       "golang.org/x/text/unicode/norm"
        "io"
        "os"
        "path/filepath"
@@ -97,9 +99,18 @@ func UnicodeSanitize(s string) string {
                }
        }
 
+       // remove accents - see https://blog.golang.org/normalization
+       t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC)
+       result, _, _ := transform.String(t, string(target))
+       return result
+
        return string(target)
 }
 
+func isMn(r rune) bool {
+       return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks
+}
+
 // ReplaceExtension takes a path and an extension, strips the old extension
 // and returns the path with the new extension.
 func ReplaceExtension(path string, newExt string) string {

diff --git a/helpers/path_test.go b/helpers/path_test.go
index cb87142668b8e9f551dc6ef8e8c0ad242d1622a9..3fdb94a4f166b0eedbf835d41405b940188e2872 100644 (file)
--- a/helpers/path_test.go
+++ b/helpers/path_test.go
@@ -27,7 +27,7 @@ func TestMakePath(t *testing.T) {
                {"FOo/BaR.html", "FOo/BaR.html"},
                {"трям/трям", "трям/трям"},
                {"은행", "은행"},
-               {"Ð\91анковÑ\81кий каÑ\81Ñ\81иÑ\80", "Ð\91анковÑ\81кий-кассир"},
+               {"Ð\91анковÑ\81кий каÑ\81Ñ\81иÑ\80", "Ð\91анковÑ\81кии-кассир"},
        }
 
        for _, test := range tests {
@@ -717,7 +717,7 @@ func TestGetTempDir(t *testing.T) {
                {testDir + "FOo/BaR.html", dir + testDir + "FOo/BaR.html" + FilePathSeparator},
                {testDir + "трям/трям", dir + testDir + "трям/трям" + FilePathSeparator},
                {testDir + "은행", dir + testDir + "은행" + FilePathSeparator},
-               {testDir + "Ð\91анковÑ\81кий каÑ\81Ñ\81иÑ\80", dir + testDir + "Ð\91анковÑ\81кий-кассир" + FilePathSeparator},
+               {testDir + "Ð\91анковÑ\81кий каÑ\81Ñ\81иÑ\80", dir + testDir + "Ð\91анковÑ\81кии-кассир" + FilePathSeparator},
        }
 
        for _, test := range tests {