Always use content to resolve content type in resources.GetRemote

This is a security hardening measure; don't trust the URL extension or any `Content-Type`/`Content-Disposition` header on its own, always look at the file content using Go's `http.DetectContentType`.

This commit also adds ttf and otf media type definitions to Hugo.

Fixes #9302
Fixes #9301

diff --git a/media/mediaType.go b/media/mediaType.go
index eec7a27a832b75a7819d3579e03fc2458424fa33..0bdeb6db787207818d4eea02aae1ab66525e89a3 100644 (file)
--- a/media/mediaType.go
+++ b/media/mediaType.go
@@ -17,6 +17,7 @@ import (
        "encoding/json"
        "errors"
        "fmt"
+       "net/http"
        "sort"
        "strings"
 
@@ -60,6 +61,42 @@ type SuffixInfo struct {
        FullSuffix string `json:"fullSuffix"`
 }
 
+// FromContent resolve the Type primarily using http.DetectContentType.
+// If http.DetectContentType resolves to application/octet-stream, a zero Type is returned.
+// If http.DetectContentType  resolves to text/plain or application/xml, we try to get more specific using types and ext.
+func FromContent(types Types, ext string, content []byte) Type {
+       ext = strings.TrimPrefix(ext, ".")
+       t := strings.Split(http.DetectContentType(content), ";")[0]
+       var m Type
+       if t == "application/octet-stream" {
+               return m
+       }
+
+       var found bool
+       m, found = types.GetByType(t)
+       if !found {
+               if t == "text/xml" {
+                       // This is how it's configured in Hugo by default.
+                       m, found = types.GetByType("application/xml")
+               }
+       }
+
+       if !found || ext == "" {
+               return m
+       }
+
+       if m.Type() == "text/plain" || m.Type() == "application/xml" {
+               // http.DetectContentType isn't brilliant when it comes to common text formats, so we need to do better.
+               // For now we say that if it's detected to be a text format and the extension/content type in header reports
+               // it to be a text format, then we use that.
+               mm, _, found := types.GetFirstBySuffix(ext)
+               if found && mm.IsText() {
+                       return mm
+               }
+       }
+       return m
+}
+
 // FromStringAndExt creates a Type from a MIME string and a given extension.
 func FromStringAndExt(t, ext string) (Type, error) {
        tp, err := fromString(t)
@@ -122,6 +159,21 @@ func (m Type) Suffixes() []string {
        return strings.Split(m.suffixesCSV, ",")
 }
 
+// IsText returns whether this Type is a text format.
+// Note that this may currently return false negatives.
+// TODO(bep) improve
+func (m Type) IsText() bool {
+       if m.MainType == "text" {
+               return true
+       }
+       switch m.SubType {
+       case "javascript", "json", "rss", "xml", "svg", TOMLType.SubType, YAMLType.SubType:
+               return true
+
+       }
+       return false
+}
+
 func (m *Type) init() {
        m.FirstSuffix.FullSuffix = ""
        m.FirstSuffix.Suffix = ""
@@ -183,6 +235,10 @@ var (
        BMPType  = newMediaType("image", "bmp", []string{"bmp"})
        WEBPType = newMediaType("image", "webp", []string{"webp"})
 
+       // Common font types
+       TrueTypeFontType = newMediaType("font", "ttf", []string{"ttf"})
+       OpenTypeFontType = newMediaType("font", "otf", []string{"otf"})
+
        // Common video types
        AVIType  = newMediaType("video", "x-msvideo", []string{"avi"})
        MPEGType = newMediaType("video", "mpeg", []string{"mpg", "mpeg"})
@@ -224,6 +280,8 @@ var DefaultTypes = Types{
        OGGType,
        WEBMType,
        GPPType,
+       OpenTypeFontType,
+       TrueTypeFontType,
 }
 
 func init() {

diff --git a/media/mediaType_test.go b/media/mediaType_test.go
index b33ca174cf2b61b2ef541e53064106e5e110e5e7..f3a06e8ed0c3201304ed8461ea185e4cec01ee09 100644 (file)
--- a/media/mediaType_test.go
+++ b/media/mediaType_test.go
@@ -15,10 +15,14 @@ package media
 
 import (
        "encoding/json"
+       "io/ioutil"
+       "path/filepath"
        "sort"
+       "strings"
        "testing"
 
        qt "github.com/frankban/quicktest"
+       "github.com/gohugoio/hugo/common/paths"
 )
 
 func TestDefaultTypes(t *testing.T) {
@@ -47,6 +51,8 @@ func TestDefaultTypes(t *testing.T) {
                {XMLType, "application", "xml", "xml", "application/xml", "application/xml"},
                {TOMLType, "application", "toml", "toml", "application/toml", "application/toml"},
                {YAMLType, "application", "yaml", "yaml", "application/yaml", "application/yaml"},
+               {TrueTypeFontType, "font", "ttf", "ttf", "font/ttf", "font/ttf"},
+               {OpenTypeFontType, "font", "otf", "otf", "font/otf", "font/otf"},
        } {
                c.Assert(test.tp.MainType, qt.Equals, test.expectedMainType)
                c.Assert(test.tp.SubType, qt.Equals, test.expectedSubType)
@@ -56,7 +62,7 @@ func TestDefaultTypes(t *testing.T) {
 
        }
 
-       c.Assert(len(DefaultTypes), qt.Equals, 28)
+       c.Assert(len(DefaultTypes), qt.Equals, 30)
 }
 
 func TestGetByType(t *testing.T) {
@@ -175,6 +181,26 @@ func TestFromExtensionMultipleSuffixes(t *testing.T) {
 
 }
 
+func TestFromContent(t *testing.T) {
+       c := qt.New(t)
+
+       files, err := filepath.Glob("./testdata/resource.*")
+       c.Assert(err, qt.IsNil)
+       mtypes := DefaultTypes
+
+       for _, filename := range files {
+               c.Run(filepath.Base(filename), func(c *qt.C) {
+                       content, err := ioutil.ReadFile(filename)
+                       c.Assert(err, qt.IsNil)
+                       ext := strings.TrimPrefix(paths.Ext(filename), ".")
+                       expected, _, found := mtypes.GetFirstBySuffix(ext)
+                       c.Assert(found, qt.IsTrue)
+                       got := FromContent(mtypes, ext, content)
+                       c.Assert(got, qt.Equals, expected)
+               })
+       }
+}
+
 func TestDecodeTypes(t *testing.T) {
        c := qt.New(t)
 

diff --git a/media/testdata/reosurce.otf b/media/testdata/reosurce.otf
new file mode 100644 (file)
index 0000000..99034a2
Binary files /dev/null and b/media/testdata/reosurce.otf differ

diff --git a/media/testdata/resource.css b/media/testdata/resource.css
new file mode 100644 (file)
index 0000000..a267873
--- /dev/null
+++ b/media/testdata/resource.css
@@ -0,0 +1,8 @@
+body {
+    background-color: lightblue;
+  }
+  
+  h1 {
+    color: navy;
+    margin-left: 20px;
+  }
\ No newline at end of file

diff --git a/media/testdata/resource.csv b/media/testdata/resource.csv
new file mode 100644 (file)
index 0000000..ee6b058
--- /dev/null
+++ b/media/testdata/resource.csv
@@ -0,0 +1,130 @@
+"LatD", "LatM", "LatS", "NS", "LonD", "LonM", "LonS", "EW", "City", "State"
+   41,    5,   59, "N",     80,   39,    0, "W", "Youngstown", OH
+   42,   52,   48, "N",     97,   23,   23, "W", "Yankton", SD
+   46,   35,   59, "N",    120,   30,   36, "W", "Yakima", WA
+   42,   16,   12, "N",     71,   48,    0, "W", "Worcester", MA
+   43,   37,   48, "N",     89,   46,   11, "W", "Wisconsin Dells", WI
+   36,    5,   59, "N",     80,   15,    0, "W", "Winston-Salem", NC
+   49,   52,   48, "N",     97,    9,    0, "W", "Winnipeg", MB
+   39,   11,   23, "N",     78,    9,   36, "W", "Winchester", VA
+   34,   14,   24, "N",     77,   55,   11, "W", "Wilmington", NC
+   39,   45,    0, "N",     75,   33,    0, "W", "Wilmington", DE
+   48,    9,    0, "N",    103,   37,   12, "W", "Williston", ND
+   41,   15,    0, "N",     77,    0,    0, "W", "Williamsport", PA
+   37,   40,   48, "N",     82,   16,   47, "W", "Williamson", WV
+   33,   54,    0, "N",     98,   29,   23, "W", "Wichita Falls", TX
+   37,   41,   23, "N",     97,   20,   23, "W", "Wichita", KS
+   40,    4,   11, "N",     80,   43,   12, "W", "Wheeling", WV
+   26,   43,   11, "N",     80,    3,    0, "W", "West Palm Beach", FL
+   47,   25,   11, "N",    120,   19,   11, "W", "Wenatchee", WA
+   41,   25,   11, "N",    122,   23,   23, "W", "Weed", CA
+   31,   13,   11, "N",     82,   20,   59, "W", "Waycross", GA
+   44,   57,   35, "N",     89,   38,   23, "W", "Wausau", WI
+   42,   21,   36, "N",     87,   49,   48, "W", "Waukegan", IL
+   44,   54,    0, "N",     97,    6,   36, "W", "Watertown", SD
+   43,   58,   47, "N",     75,   55,   11, "W", "Watertown", NY
+   42,   30,    0, "N",     92,   20,   23, "W", "Waterloo", IA
+   41,   32,   59, "N",     73,    3,    0, "W", "Waterbury", CT
+   38,   53,   23, "N",     77,    1,   47, "W", "Washington", DC
+   41,   50,   59, "N",     79,    8,   23, "W", "Warren", PA
+   46,    4,   11, "N",    118,   19,   48, "W", "Walla Walla", WA
+   31,   32,   59, "N",     97,    8,   23, "W", "Waco", TX
+   38,   40,   48, "N",     87,   31,   47, "W", "Vincennes", IN
+   28,   48,   35, "N",     97,    0,   36, "W", "Victoria", TX
+   32,   20,   59, "N",     90,   52,   47, "W", "Vicksburg", MS
+   49,   16,   12, "N",    123,    7,   12, "W", "Vancouver", BC
+   46,   55,   11, "N",     98,    0,   36, "W", "Valley City", ND
+   30,   49,   47, "N",     83,   16,   47, "W", "Valdosta", GA
+   43,    6,   36, "N",     75,   13,   48, "W", "Utica", NY
+   39,   54,    0, "N",     79,   43,   48, "W", "Uniontown", PA
+   32,   20,   59, "N",     95,   18,    0, "W", "Tyler", TX
+   42,   33,   36, "N",    114,   28,   12, "W", "Twin Falls", ID
+   33,   12,   35, "N",     87,   34,   11, "W", "Tuscaloosa", AL
+   34,   15,   35, "N",     88,   42,   35, "W", "Tupelo", MS
+   36,    9,   35, "N",     95,   54,   36, "W", "Tulsa", OK
+   32,   13,   12, "N",    110,   58,   12, "W", "Tucson", AZ
+   37,   10,   11, "N",    104,   30,   36, "W", "Trinidad", CO
+   40,   13,   47, "N",     74,   46,   11, "W", "Trenton", NJ
+   44,   45,   35, "N",     85,   37,   47, "W", "Traverse City", MI
+   43,   39,    0, "N",     79,   22,   47, "W", "Toronto", ON
+   39,    2,   59, "N",     95,   40,   11, "W", "Topeka", KS
+   41,   39,    0, "N",     83,   32,   24, "W", "Toledo", OH
+   33,   25,   48, "N",     94,    3,    0, "W", "Texarkana", TX
+   39,   28,   12, "N",     87,   24,   36, "W", "Terre Haute", IN
+   27,   57,    0, "N",     82,   26,   59, "W", "Tampa", FL
+   30,   27,    0, "N",     84,   16,   47, "W", "Tallahassee", FL
+   47,   14,   24, "N",    122,   25,   48, "W", "Tacoma", WA
+   43,    2,   59, "N",     76,    9,    0, "W", "Syracuse", NY
+   32,   35,   59, "N",     82,   20,   23, "W", "Swainsboro", GA
+   33,   55,   11, "N",     80,   20,   59, "W", "Sumter", SC
+   40,   59,   24, "N",     75,   11,   24, "W", "Stroudsburg", PA
+   37,   57,   35, "N",    121,   17,   24, "W", "Stockton", CA
+   44,   31,   12, "N",     89,   34,   11, "W", "Stevens Point", WI
+   40,   21,   36, "N",     80,   37,   12, "W", "Steubenville", OH
+   40,   37,   11, "N",    103,   13,   12, "W", "Sterling", CO
+   38,    9,    0, "N",     79,    4,   11, "W", "Staunton", VA
+   39,   55,   11, "N",     83,   48,   35, "W", "Springfield", OH
+   37,   13,   12, "N",     93,   17,   24, "W", "Springfield", MO
+   42,    5,   59, "N",     72,   35,   23, "W", "Springfield", MA
+   39,   47,   59, "N",     89,   39,    0, "W", "Springfield", IL
+   47,   40,   11, "N",    117,   24,   36, "W", "Spokane", WA
+   41,   40,   48, "N",     86,   15,    0, "W", "South Bend", IN
+   43,   32,   24, "N",     96,   43,   48, "W", "Sioux Falls", SD
+   42,   29,   24, "N",     96,   23,   23, "W", "Sioux City", IA
+   32,   30,   35, "N",     93,   45,    0, "W", "Shreveport", LA
+   33,   38,   23, "N",     96,   36,   36, "W", "Sherman", TX
+   44,   47,   59, "N",    106,   57,   35, "W", "Sheridan", WY
+   35,   13,   47, "N",     96,   40,   48, "W", "Seminole", OK
+   32,   25,   11, "N",     87,    1,   11, "W", "Selma", AL
+   38,   42,   35, "N",     93,   13,   48, "W", "Sedalia", MO
+   47,   35,   59, "N",    122,   19,   48, "W", "Seattle", WA
+   41,   24,   35, "N",     75,   40,   11, "W", "Scranton", PA
+   41,   52,   11, "N",    103,   39,   36, "W", "Scottsbluff", NB
+   42,   49,   11, "N",     73,   56,   59, "W", "Schenectady", NY
+   32,    4,   48, "N",     81,    5,   23, "W", "Savannah", GA
+   46,   29,   24, "N",     84,   20,   59, "W", "Sault Sainte Marie", MI
+   27,   20,   24, "N",     82,   31,   47, "W", "Sarasota", FL
+   38,   26,   23, "N",    122,   43,   12, "W", "Santa Rosa", CA
+   35,   40,   48, "N",    105,   56,   59, "W", "Santa Fe", NM
+   34,   25,   11, "N",    119,   41,   59, "W", "Santa Barbara", CA
+   33,   45,   35, "N",    117,   52,   12, "W", "Santa Ana", CA
+   37,   20,   24, "N",    121,   52,   47, "W", "San Jose", CA
+   37,   46,   47, "N",    122,   25,   11, "W", "San Francisco", CA
+   41,   27,    0, "N",     82,   42,   35, "W", "Sandusky", OH
+   32,   42,   35, "N",    117,    9,    0, "W", "San Diego", CA
+   34,    6,   36, "N",    117,   18,   35, "W", "San Bernardino", CA
+   29,   25,   12, "N",     98,   30,    0, "W", "San Antonio", TX
+   31,   27,   35, "N",    100,   26,   24, "W", "San Angelo", TX
+   40,   45,   35, "N",    111,   52,   47, "W", "Salt Lake City", UT
+   38,   22,   11, "N",     75,   35,   59, "W", "Salisbury", MD
+   36,   40,   11, "N",    121,   39,    0, "W", "Salinas", CA
+   38,   50,   24, "N",     97,   36,   36, "W", "Salina", KS
+   38,   31,   47, "N",    106,    0,    0, "W", "Salida", CO
+   44,   56,   23, "N",    123,    1,   47, "W", "Salem", OR
+   44,   57,    0, "N",     93,    5,   59, "W", "Saint Paul", MN
+   38,   37,   11, "N",     90,   11,   24, "W", "Saint Louis", MO
+   39,   46,   12, "N",     94,   50,   23, "W", "Saint Joseph", MO
+   42,    5,   59, "N",     86,   28,   48, "W", "Saint Joseph", MI
+   44,   25,   11, "N",     72,    1,   11, "W", "Saint Johnsbury", VT
+   45,   34,   11, "N",     94,   10,   11, "W", "Saint Cloud", MN
+   29,   53,   23, "N",     81,   19,   11, "W", "Saint Augustine", FL
+   43,   25,   48, "N",     83,   56,   24, "W", "Saginaw", MI
+   38,   35,   24, "N",    121,   29,   23, "W", "Sacramento", CA
+   43,   36,   36, "N",     72,   58,   12, "W", "Rutland", VT
+   33,   24,    0, "N",    104,   31,   47, "W", "Roswell", NM
+   35,   56,   23, "N",     77,   48,    0, "W", "Rocky Mount", NC
+   41,   35,   24, "N",    109,   13,   48, "W", "Rock Springs", WY
+   42,   16,   12, "N",     89,    5,   59, "W", "Rockford", IL
+   43,    9,   35, "N",     77,   36,   36, "W", "Rochester", NY
+   44,    1,   12, "N",     92,   27,   35, "W", "Rochester", MN
+   37,   16,   12, "N",     79,   56,   24, "W", "Roanoke", VA
+   37,   32,   24, "N",     77,   26,   59, "W", "Richmond", VA
+   39,   49,   48, "N",     84,   53,   23, "W", "Richmond", IN
+   38,   46,   12, "N",    112,    5,   23, "W", "Richfield", UT
+   45,   38,   23, "N",     89,   25,   11, "W", "Rhinelander", WI
+   39,   31,   12, "N",    119,   48,   35, "W", "Reno", NV
+   50,   25,   11, "N",    104,   39,    0, "W", "Regina", SA
+   40,   10,   48, "N",    122,   14,   23, "W", "Red Bluff", CA
+   40,   19,   48, "N",     75,   55,   48, "W", "Reading", PA
+   41,    9,   35, "N",     81,   14,   23, "W", "Ravenna", OH 
+

diff --git a/media/testdata/resource.ics b/media/testdata/resource.ics
new file mode 100644 (file)
index 0000000..b9a263e
--- /dev/null
+++ b/media/testdata/resource.ics
@@ -0,0 +1,24 @@
+BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//ZContent.net//Zap Calendar 1.0//EN
+CALSCALE:GREGORIAN
+METHOD:PUBLISH
+BEGIN:VEVENT
+SUMMARY:Abraham Lincoln
+UID:c7614cff-3549-4a00-9152-d25cc1fe077d
+SEQUENCE:0
+STATUS:CONFIRMED
+TRANSP:TRANSPARENT
+RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=2;BYMONTHDAY=12
+DTSTART:20080212
+DTEND:20080213
+DTSTAMP:20150421T141403
+CATEGORIES:U.S. Presidents,Civil War People
+LOCATION:Hodgenville\, Kentucky
+GEO:37.5739497;-85.7399606
+DESCRIPTION:Born February 12\, 1809\nSixteenth President (1861-1865)\n\n\n
+ \nhttp://AmericanHistoryCalendar.com
+URL:http://americanhistorycalendar.com/peoplecalendar/1,328-abraham-lincol
+ n
+END:VEVENT
+END:VCALENDAR
\ No newline at end of file

diff --git a/media/testdata/resource.jpg b/media/testdata/resource.jpg
new file mode 100644 (file)
index 0000000..a9049e8
Binary files /dev/null and b/media/testdata/resource.jpg differ

diff --git a/media/testdata/resource.js b/media/testdata/resource.js
new file mode 100644 (file)
index 0000000..75ba3b7
--- /dev/null
+++ b/media/testdata/resource.js
@@ -0,0 +1,3 @@
+function foo() {
+    return "foo";
+}
\ No newline at end of file

diff --git a/media/testdata/resource.json b/media/testdata/resource.json
new file mode 100644 (file)
index 0000000..4468998
--- /dev/null
+++ b/media/testdata/resource.json
@@ -0,0 +1,14 @@
+{
+   "firstName": "Joe",
+   "lastName": "Jackson",
+   "gender": "male",
+   "age": 28,
+   "address": {
+       "streetAddress": "101",
+       "city": "San Diego",
+       "state": "CA"
+   },
+   "phoneNumbers": [
+       { "type": "home", "number": "7349282382" }
+   ]
+}
\ No newline at end of file

diff --git a/media/testdata/resource.png b/media/testdata/resource.png
new file mode 100644 (file)
index 0000000..08ae570
Binary files /dev/null and b/media/testdata/resource.png differ

diff --git a/media/testdata/resource.rss b/media/testdata/resource.rss
new file mode 100644 (file)
index 0000000..b20b0fc
--- /dev/null
+++ b/media/testdata/resource.rss
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<rss version="2.0">
+
+<channel>
+  <title>W3Schools Home Page</title>
+  <link>https://www.w3schools.com</link>
+  <description>Free web building tutorials</description>
+  <item>
+    <title>RSS Tutorial</title>
+    <link>https://www.w3schools.com/xml/xml_rss.asp</link>
+    <description>New RSS tutorial on W3Schools</description>
+  </item>
+  <item>
+    <title>XML Tutorial</title>
+    <link>https://www.w3schools.com/xml</link>
+    <description>New XML tutorial on W3Schools</description>
+  </item>
+</channel>
+
+</rss>
\ No newline at end of file

diff --git a/media/testdata/resource.sass b/media/testdata/resource.sass
new file mode 100644 (file)
index 0000000..ad857fa
--- /dev/null
+++ b/media/testdata/resource.sass
@@ -0,0 +1,6 @@
+$font-stack: Helvetica, sans-serif
+$primary-color: #333
+
+body
+  font: 100% $font-stack
+  color: $primary-color
\ No newline at end of file

diff --git a/media/testdata/resource.scss b/media/testdata/resource.scss
new file mode 100644 (file)
index 0000000..d63e420
--- /dev/null
+++ b/media/testdata/resource.scss
@@ -0,0 +1,7 @@
+$font-stack: Helvetica, sans-serif;
+$primary-color: #333;
+
+body {
+  font: 100% $font-stack;
+  color: $primary-color;
+}
\ No newline at end of file

diff --git a/media/testdata/resource.svg b/media/testdata/resource.svg
new file mode 100644 (file)
index 0000000..2759ae7
--- /dev/null
+++ b/media/testdata/resource.svg
@@ -0,0 +1,5 @@
+<svg height="100" width="100">
+  <circle cx="50" cy="50" r="40" stroke="black" stroke-width="3" fill="red" />
+  Sorry, your browser does not support inline SVG.  
+</svg> 
+ 
\ No newline at end of file

diff --git a/media/testdata/resource.ttf b/media/testdata/resource.ttf
new file mode 100644 (file)
index 0000000..8bc614d
Binary files /dev/null and b/media/testdata/resource.ttf differ

diff --git a/media/testdata/resource.webp b/media/testdata/resource.webp
new file mode 100644 (file)
index 0000000..4365e7b
Binary files /dev/null and b/media/testdata/resource.webp differ

diff --git a/media/testdata/resource.xml b/media/testdata/resource.xml
new file mode 100644 (file)
index 0000000..fa0c0a5
--- /dev/null
+++ b/media/testdata/resource.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<note>
+  <to>Tove</to>
+  <from>Jani</from>
+  <heading>Reminder</heading>
+  <body>Don't forget me this weekend!</body>
+</note>
\ No newline at end of file

diff --git a/resources/images/config.go b/resources/images/config.go
index c8990d5ca334f91317818be3d908cd811a942a71..a8b5412d65d7c159ebb1b187b5337a33439c8e09 100644 (file)
--- a/resources/images/config.go
+++ b/resources/images/config.go
@@ -20,6 +20,7 @@ import (
        "strings"
 
        "github.com/gohugoio/hugo/helpers"
+       "github.com/gohugoio/hugo/media"
 
        "github.com/pkg/errors"
 
@@ -45,6 +46,15 @@ var (
                ".webp": WEBP,
        }
 
+       imageFormatsBySubType = map[string]Format{
+               media.JPEGType.SubType: JPEG,
+               media.PNGType.SubType:  PNG,
+               media.TIFFType.SubType: TIFF,
+               media.BMPType.SubType:  BMP,
+               media.GIFType.SubType:  GIF,
+               media.WEBPType.SubType: WEBP,
+       }
+
        // Add or increment if changes to an image format's processing requires
        // re-generation.
        imageFormatsVersions = map[Format]int{
@@ -102,6 +112,11 @@ func ImageFormatFromExt(ext string) (Format, bool) {
        return f, found
 }
 
+func ImageFormatFromMediaSubType(sub string) (Format, bool) {
+       f, found := imageFormatsBySubType[sub]
+       return f, found
+}
+
 const (
        defaultJPEGQuality    = 75
        defaultResampleFilter = "box"

diff --git a/resources/images/filters.go b/resources/images/filters.go
index e166a0f9d983a37ba75f4a0c4e6646d9876a3e11..fd7e3145796f19ff1fa8c8ae43c1d620b4baeb99 100644 (file)
--- a/resources/images/filters.go
+++ b/resources/images/filters.go
@@ -66,6 +66,9 @@ func (*Filters) Text(text string, options ...interface{}) gift.Filter {
                        case "linespacing":
                                tf.linespacing = cast.ToInt(v)
                        case "font":
+                               if err, ok := v.(error); ok {
+                                       panic(fmt.Sprintf("invalid font source: %s", err))
+                               }
                                fontSource, ok1 := v.(hugio.ReadSeekCloserProvider)
                                identifier, ok2 := v.(resource.Identifier)
 

diff --git a/resources/postpub/fields_test.go b/resources/postpub/fields_test.go
index 19c3720f70053501c437f209076db47b97309030..c408e7791d3953790166e189a57c93bd3332b374 100644 (file)
--- a/resources/postpub/fields_test.go
+++ b/resources/postpub/fields_test.go
@@ -36,6 +36,7 @@ func TestCreatePlaceholders(t *testing.T) {
                "Suffixes":    "pre_foo.Suffixes_post",
                "Delimiter":   "pre_foo.Delimiter_post",
                "FirstSuffix": "pre_foo.FirstSuffix_post",
+               "IsText":      "pre_foo.IsText_post",
                "String":      "pre_foo.String_post",
                "Type":        "pre_foo.Type_post",
                "MainType":    "pre_foo.MainType_post",

diff --git a/resources/resource.go b/resources/resource.go
index 1f6246859b0646bd205324a73f0db5ba8fa243d0..4bf35f9aca798f21d1769dfbeafb6b04175680a2 100644 (file)
--- a/resources/resource.go
+++ b/resources/resource.go
@@ -69,6 +69,9 @@ type ResourceSourceDescriptor struct {
 
        Fs afero.Fs
 
+       // Set when its known up front, else it's resolved from the target filename.
+       MediaType media.Type
+
        // The relative target filename without any language code.
        RelTargetFilename string
 

diff --git a/resources/resource_factories/create/remote.go b/resources/resource_factories/create/remote.go
index 53e77bc5eeaa3324e40e62fe7ac81abee97135db..f6d3f13dd25663a230ea8474e6b71b73bfd019c2 100644 (file)
--- a/resources/resource_factories/create/remote.go
+++ b/resources/resource_factories/create/remote.go
@@ -29,6 +29,7 @@ import (
        "github.com/gohugoio/hugo/common/hugio"
        "github.com/gohugoio/hugo/common/types"
        "github.com/gohugoio/hugo/helpers"
+       "github.com/gohugoio/hugo/media"
        "github.com/gohugoio/hugo/resources"
        "github.com/gohugoio/hugo/resources/resource"
        "github.com/mitchellh/mapstructure"
@@ -99,7 +100,7 @@ func (c *Client) FromRemote(uri string, optionsm map[string]interface{}) (resour
 
        body, err := ioutil.ReadAll(res.Body)
        if err != nil {
-               return nil, errors.Wrapf(err, "failed to read remote resource %s", uri)
+               return nil, errors.Wrapf(err, "failed to read remote resource %q", uri)
        }
 
        filename := path.Base(rURL.Path)
@@ -109,33 +110,30 @@ func (c *Client) FromRemote(uri string, optionsm map[string]interface{}) (resour
                }
        }
 
-       var extension string
+       var extensionHint string
+
        if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 {
-               extension = arr[0]
+               extensionHint = arr[0]
        }
 
-       // If extension was not determined by header, look for a file extention
-       if extension == "" {
+       // Look for a file extention
+       if extensionHint == "" {
                if ext := path.Ext(filename); ext != "" {
-                       extension = ext
+                       extensionHint = ext
                }
        }
 
-       // If extension was not determined by header or file extention, try using content itself
-       if extension == "" {
-               if ct := http.DetectContentType(body); ct != "application/octet-stream" {
-                       if ct == "image/jpeg" {
-                               extension = ".jpg"
-                       } else if arr, _ := mime.ExtensionsByType(ct); arr != nil {
-                               extension = arr[0]
-                       }
-               }
+       // Now resolve the media type primarily using the content.
+       mediaType := media.FromContent(c.rs.MediaTypes, extensionHint, body)
+       if mediaType.IsZero() {
+               return nil, errors.Errorf("failed to resolve media type for remote resource %q", uri)
        }
 
-       resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + extension
+       resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + mediaType.FirstSuffix.FullSuffix
 
        return c.rs.New(
                resources.ResourceSourceDescriptor{
+                       MediaType:   mediaType,
                        LazyPublish: true,
                        OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
                                return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil

diff --git a/resources/resource_spec.go b/resources/resource_spec.go
index 897c1bbaacd5f7edb541b93d2284faf820bfda7f..cd1e5010d9b7a10c0ca368987804ac615be91310 100644 (file)
--- a/resources/resource_spec.go
+++ b/resources/resource_spec.go
@@ -272,21 +272,28 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (reso
                fd.RelTargetFilename = sourceFilename
        }
 
-       ext := strings.ToLower(filepath.Ext(fd.RelTargetFilename))
-       mimeType, suffixInfo, found := r.MediaTypes.GetFirstBySuffix(strings.TrimPrefix(ext, "."))
-       // TODO(bep) we need to handle these ambiguous types better, but in this context
-       // we most likely want the application/xml type.
-       if suffixInfo.Suffix == "xml" && mimeType.SubType == "rss" {
-               mimeType, found = r.MediaTypes.GetByType("application/xml")
-       }
+       mimeType := fd.MediaType
+       if mimeType.IsZero() {
+               ext := strings.ToLower(filepath.Ext(fd.RelTargetFilename))
+               var (
+                       found      bool
+                       suffixInfo media.SuffixInfo
+               )
+               mimeType, suffixInfo, found = r.MediaTypes.GetFirstBySuffix(strings.TrimPrefix(ext, "."))
+               // TODO(bep) we need to handle these ambiguous types better, but in this context
+               // we most likely want the application/xml type.
+               if suffixInfo.Suffix == "xml" && mimeType.SubType == "rss" {
+                       mimeType, found = r.MediaTypes.GetByType("application/xml")
+               }
 
-       if !found {
-               // A fallback. Note that mime.TypeByExtension is slow by Hugo standards,
-               // so we should configure media types to avoid this lookup for most
-               // situations.
-               mimeStr := mime.TypeByExtension(ext)
-               if mimeStr != "" {
-                       mimeType, _ = media.FromStringAndExt(mimeStr, ext)
+               if !found {
+                       // A fallback. Note that mime.TypeByExtension is slow by Hugo standards,
+                       // so we should configure media types to avoid this lookup for most
+                       // situations.
+                       mimeStr := mime.TypeByExtension(ext)
+                       if mimeStr != "" {
+                               mimeType, _ = media.FromStringAndExt(mimeStr, ext)
+                       }
                }
        }
 
@@ -301,7 +308,7 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (reso
                mimeType)
 
        if mimeType.MainType == "image" {
-               imgFormat, ok := images.ImageFormatFromExt(ext)
+               imgFormat, ok := images.ImageFormatFromMediaSubType(mimeType.SubType)
                if ok {
                        ir := &imageResource{
                                Image:        images.NewImage(imgFormat, r.imaging, nil, gr),

diff --git a/tpl/resources/resources.go b/tpl/resources/resources.go
index 4433e56e57d6b9508c7e91f8da10bba4279cefff..8cd670603bd3fee1baa1cb25d987714865c98884 100644 (file)
--- a/tpl/resources/resources.go
+++ b/tpl/resources/resources.go
@@ -110,30 +110,21 @@ func (ns *Namespace) getscssClientDartSass() (*dartsass.Client, error) {
 // Get locates the filename given in Hugo's assets filesystem and
 // creates a Resource object that can be used for
 // further transformations.
-func (ns *Namespace) Get(filename interface{}) resource.Resource {
-       get := func(args ...interface{}) (resource.Resource, error) {
-               filenamestr, err := cast.ToStringE(filename)
-               if err != nil {
-                       return nil, err
-               }
-               return ns.createClient.Get(filepath.Clean(filenamestr))
-       }
-
-       r, err := get(filename)
+func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) {
+       filenamestr, err := cast.ToStringE(filename)
        if err != nil {
-               // This allows the client to reason about the .Err in the template.
-               // This is not as relevant for local resources as remotes, but
-               // it makes this method work the same way as resources.GetRemote.
-               return resources.NewErrorResource(errors.Wrap(err, "error calling resources.Get"))
+               return nil, err
        }
-       return r
-
+       return ns.createClient.Get(filepath.Clean(filenamestr))
 }
 
 // GetRemote gets the URL (via HTTP(s)) in the first argument in args and creates Resource object that can be used for
 // further transformations.
 //
 // A second argument may be provided with an option map.
+//
+// Note: This method does not return any error as a second argument,
+// for any error situations the error can be checked in .Err.
 func (ns *Namespace) GetRemote(args ...interface{}) resource.Resource {
        get := func(args ...interface{}) (resource.Resource, error) {
                if len(args) < 1 {

diff --git a/tpl/tplimpl/template_funcs_test.go b/tpl/tplimpl/template_funcs_test.go
index 6ddf13b76e32b61a16da8fcb9afdef130bb741ba..711d1350d331d050fc00e17922f9319db60d7b06 100644 (file)
--- a/tpl/tplimpl/template_funcs_test.go
+++ b/tpl/tplimpl/template_funcs_test.go
@@ -37,7 +37,6 @@ import (
        "github.com/gohugoio/hugo/tpl/internal"
        "github.com/gohugoio/hugo/tpl/partials"
        "github.com/spf13/afero"
-       
 )
 
 var logger = loggers.NewErrorLogger()

diff --git a/tpl/transform/unmarshal.go b/tpl/transform/unmarshal.go
index aa84ca1f8bb3fe546525e8f92eb258757de25ade..c59269577cf629190aa8cbbfa326fc9ab75e5185 100644 (file)
--- a/tpl/transform/unmarshal.go
+++ b/tpl/transform/unmarshal.go
@@ -95,6 +95,10 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) {
                return nil, errors.Errorf("type %T not supported", data)
        }
 
+       if dataStr == "" {
+               return nil, errors.New("no data to transform")
+       }
+
        key := helpers.MD5String(dataStr)
 
        return ns.cache.GetOrCreate(key, func() (interface{}, error) {