Add unicode support for aliases, indexes, urlize template filter.

Now aliases and indexes are not restricted ASCII letters and can include
any unicode letters.

diff --git a/helpers/templates.go b/helpers/templates.go
index 793450b43bea9493d6af568cee3252493f672964..94d6993ce4ffe8b908d468775400168ad8c705ba 100644 (file)
--- a/helpers/templates.go
+++ b/helpers/templates.go
@@ -14,16 +14,43 @@
 package helpers
 
 import (
+       "net/url"
        "regexp"
        "strings"
+       "unicode"
 )
 
 var sanitizeRegexp = regexp.MustCompile("[^a-zA-Z0-9./_-]")
 
-func Urlize(url string) string {
-       return Sanitize(strings.ToLower(strings.Replace(strings.TrimSpace(url), " ", "-", -1)))
+func MakePath(s string) string {
+       return unicodeSanitize(strings.ToLower(strings.Replace(strings.TrimSpace(s), " ", "-", -1)))
+}
+
+func Urlize(uri string) string {
+       sanitized := MakePath(uri)
+
+       // escape unicode letters
+       parsedUri, err := url.Parse(sanitized)
+       if err != nil {
+               // if net/url can not parse URL it's meaning Sanitize works incorrect
+               panic(err)
+       }
+       return parsedUri.String()
 }
 
 func Sanitize(s string) string {
        return sanitizeRegexp.ReplaceAllString(s, "")
 }
+
+func unicodeSanitize(s string) string {
+       source := []rune(s)
+       target := make([]rune, 0, len(source))
+
+       for _, r := range source {
+               if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '.' || r == '/' || r == '_' || r == '-' {
+                       target = append(target, r)
+               }
+       }
+
+       return string(target)
+}

diff --git a/helpers/templates_test.go b/helpers/templates_test.go
new file mode 100644 (file)
index 0000000..7252c2d
--- /dev/null
+++ b/helpers/templates_test.go
@@ -0,0 +1,45 @@
+package helpers
+
+import (
+       "testing"
+)
+
+func TestMakePath(t *testing.T) {
+       tests := []struct {
+               input    string
+               expected string
+       }{
+               {"  foo bar  ", "foo-bar"},
+               {"foo.bar/foo_bar-foo", "foo.bar/foo_bar-foo"},
+               {"foo,bar:foo%bar", "foobarfoobar"},
+               {"foo/bar.html", "foo/bar.html"},
+               {"трям/трям", "трям/трям"},
+       }
+
+       for _, test := range tests {
+               output := MakePath(test.input)
+               if output != test.expected {
+                       t.Errorf("Expected %#v, got %#v\n", test.expected, output)
+               }
+       }
+}
+
+func TestUrlize(t *testing.T) {
+       tests := []struct {
+               input    string
+               expected string
+       }{
+               {"  foo bar  ", "foo-bar"},
+               {"foo.bar/foo_bar-foo", "foo.bar/foo_bar-foo"},
+               {"foo,bar:foo%bar", "foobarfoobar"},
+               {"foo/bar.html", "foo/bar.html"},
+               {"трям/трям", "%D1%82%D1%80%D1%8F%D0%BC/%D1%82%D1%80%D1%8F%D0%BC"},
+       }
+
+       for _, test := range tests {
+               output := Urlize(test.input)
+               if output != test.expected {
+                       t.Errorf("Expected %#v, got %#v\n", test.expected, output)
+               }
+       }
+}

diff --git a/hugolib/index.go b/hugolib/index.go
index 0189eccb369af93b17268e5dc345073533003d93..30cb8f941f114aa3180a18fc7cfb9807affade17 100644 (file)
--- a/hugolib/index.go
+++ b/hugolib/index.go
@@ -59,7 +59,7 @@ type OrderedIndexEntry struct {
 
 // KeyPrep... Indexes should be case insensitive. Can make it easily conditional later.
 func kp(in string) string {
-       return helpers.Urlize(in)
+       return helpers.MakePath(in)
 }
 
 func (i Index) Get(key string) WeightedPages { return i[kp(key)] }

diff --git a/target/alias_test.go b/target/alias_test.go
index 7f5db79af2b5898adb61b4349512f070930c3876..d19349f230f63c7053a6f5245db1b82c4baaea50 100644 (file)
--- a/target/alias_test.go
+++ b/target/alias_test.go
@@ -20,6 +20,7 @@ func TestHTMLRedirectAlias(t *testing.T) {
                {"alias 3.html", "alias-3.html"},
                {"alias4.html", "alias4.html"},
                {"/alias 5.html", "/alias-5.html"},
+               {"/трям.html", "/трям.html"},
        }
 
        for _, test := range tests {

diff --git a/target/htmlredirect.go b/target/htmlredirect.go
index 53e900f9163207aa086a5f8179b3c7f96610bad4..55f4896e1b5d5d9597c48497ed19f698c27d118b 100644 (file)
--- a/target/htmlredirect.go
+++ b/target/htmlredirect.go
@@ -39,7 +39,7 @@ func (h *HTMLRedirectAlias) Translate(alias string) (aliasPath string, err error
        } else if !strings.HasSuffix(alias, ".html") {
                alias = alias + "/index.html"
        }
-       return path.Join(h.PublishDir, helpers.Urlize(alias)), nil
+       return path.Join(h.PublishDir, helpers.MakePath(alias)), nil
 }
 
 type AliasNode struct {