resources: Cache Exif data to disk
authorBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Sun, 1 Sep 2019 15:27:21 +0000 (17:27 +0200)
committerBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Mon, 2 Sep 2019 14:37:28 +0000 (16:37 +0200)
```bash
name                        old time/op    new time/op    delta
ImageExif/Cold_cache-4         312µs ±28%     355µs ± 7%     ~     (p=0.343 n=4+4)
ImageExif/Cold_cache,_10-4     479µs ± 6%     546µs ± 0%  +13.91%  (p=0.029 n=4+4)
ImageExif/Warm_cache-4         272µs ± 1%      81µs ± 5%  -70.30%  (p=0.029 n=4+4)

name                        old alloc/op   new alloc/op   delta
ImageExif/Cold_cache-4         151kB ± 0%     161kB ± 0%   +6.46%  (p=0.029 n=4+4)
ImageExif/Cold_cache,_10-4     179kB ± 0%     189kB ± 0%   +5.49%  (p=0.029 n=4+4)
ImageExif/Warm_cache-4         151kB ± 0%      13kB ± 0%  -91.52%  (p=0.029 n=4+4)

name                        old allocs/op  new allocs/op  delta
ImageExif/Cold_cache-4         1.03k ± 0%     1.21k ± 0%  +17.78%  (p=0.029 n=4+4)
ImageExif/Cold_cache,_10-4     1.65k ± 0%     1.83k ± 0%  +11.09%  (p=0.029 n=4+4)
ImageExif/Warm_cache-4         1.03k ± 0%     0.28k ± 0%  -72.40%  (p=0.029 n=4+4)
```

Fixes #6291

go.mod
go.sum
hugolib/config.go
resources/image.go
resources/image_test.go
resources/images/exif/exif.go
resources/images/exif/exif_test.go

diff --git a/go.mod b/go.mod
index 2bb6d8f362c6c843580fdadce9bead768095a4a1..38c487fa104f0c07ee83ac38162134ecd20c085c 100644 (file)
--- a/go.mod
+++ b/go.mod
@@ -12,6 +12,7 @@ require (
        github.com/bep/debounce v1.2.0
        github.com/bep/gitmap v1.1.0
        github.com/bep/go-tocss v0.6.0
+       github.com/bep/tmc v0.5.0
        github.com/disintegration/gift v1.2.1
        github.com/dustin/go-humanize v1.0.0
        github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385
@@ -20,7 +21,7 @@ require (
        github.com/fsnotify/fsnotify v1.4.7
        github.com/gobwas/glob v0.2.3
        github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95
-       github.com/google/go-cmp v0.3.0
+       github.com/google/go-cmp v0.3.1
        github.com/gorilla/websocket v1.4.0
        github.com/jdkato/prose v1.1.0
        github.com/kyokomi/emoji v1.5.1
@@ -63,6 +64,7 @@ require (
        golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 // indirect
        google.golang.org/appengine v1.6.0 // indirect
        google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69 // indirect
+       gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
        gopkg.in/yaml.v2 v2.2.2
 )
 
diff --git a/go.sum b/go.sum
index 685d846a9d11e1ba2746e7fdee9520e011e4f8c9..e97e9333f471a04512bbe739d28d2322502e7fd0 100644 (file)
--- a/go.sum
+++ b/go.sum
@@ -65,6 +65,8 @@ github.com/bep/gitmap v1.1.0 h1:vOMIdVB+2gd1VbfJPNJdLqAmn0af6NK98t4fK/GoCdA=
 github.com/bep/gitmap v1.1.0/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kYaY=
 github.com/bep/go-tocss v0.6.0 h1:lJf+nIjsQDpifUr+NgHi9QMBnrr9cFvMvEBT+uV9Q9E=
 github.com/bep/go-tocss v0.6.0/go.mod h1:d9d3crzlTl+PUZLFzBUjfFCpp68K+ku10mzTlnqU/+A=
+github.com/bep/tmc v0.5.0 h1:AP43LlBcCeJuXqwuQkVbTUOG6gQCo04Et4dHqOOx4hA=
+github.com/bep/tmc v0.5.0/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0=
 github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764=
@@ -138,6 +140,8 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE=
@@ -449,6 +453,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
index 2c46a2f321bceb864f22d52e0f2c82f516589034..35724ec413aac55c9502ed5cd0ad51586f534ed2 100644 (file)
@@ -631,7 +631,7 @@ func loadDefaultSettingsFor(v *viper.Viper) error {
        v.SetDefault("disableAliases", false)
        v.SetDefault("debug", false)
        v.SetDefault("disableFastRender", false)
-       v.SetDefault("timeout", 10000) // 10 seconds
+       v.SetDefault("timeout", 15000) // 15 seconds
        v.SetDefault("enableInlineShortcodes", false)
 
        return nil
index f335e97da41ebf98ff7ae76e23db951f04b8ca36..8242f7119c3b9b53d68874dad84461ea2950e0c8 100644 (file)
 package resources
 
 import (
+       "encoding/json"
        "fmt"
        "image"
        "image/draw"
        _ "image/gif"
        _ "image/png"
+       "io"
+       "io/ioutil"
        "os"
+       "path"
        "strings"
        "sync"
 
+       "github.com/gohugoio/hugo/cache/filecache"
        "github.com/gohugoio/hugo/resources/images/exif"
 
        "github.com/gohugoio/hugo/resources/internal"
@@ -55,43 +60,81 @@ type imageResource struct {
        // original (first).
        root *imageResource
 
-       exifInit    sync.Once
-       exifInitErr error
-       exif        *exif.Exif
+       metaInit    sync.Once
+       metaInitErr error
+       meta        *imageMeta
 
        baseResource
 }
 
+type imageMeta struct {
+       Exif *exif.Exif
+}
+
 func (i *imageResource) Exif() (*exif.Exif, error) {
        return i.root.getExif()
 }
 
 func (i *imageResource) getExif() (*exif.Exif, error) {
 
-       i.exifInit.Do(func() {
+       i.metaInit.Do(func() {
+
                supportsExif := i.Format == images.JPEG || i.Format == images.TIFF
                if !supportsExif {
                        return
+
                }
 
-               f, err := i.root.ReadSeekCloser()
-               if err != nil {
-                       i.exifInitErr = err
-                       return
+               key := i.getImageMetaCacheTargetPath()
+
+               read := func(info filecache.ItemInfo, r io.Reader) error {
+                       meta := &imageMeta{}
+                       data, err := ioutil.ReadAll(r)
+                       if err != nil {
+                               return err
+                       }
+
+                       if err = json.Unmarshal(data, &meta); err != nil {
+                               return err
+                       }
+
+                       i.meta = meta
+
+                       return nil
                }
-               defer f.Close()
 
-               x, err := i.getSpec().imaging.DecodeExif(f)
-               if err != nil {
-                       i.exifInitErr = err
-                       return
+               create := func(info filecache.ItemInfo, w io.WriteCloser) (err error) {
+
+                       f, err := i.root.ReadSeekCloser()
+                       if err != nil {
+                               i.metaInitErr = err
+                               return
+                       }
+                       defer f.Close()
+
+                       x, err := i.getSpec().imaging.DecodeExif(f)
+                       if err != nil {
+                               i.metaInitErr = err
+                               return
+                       }
+
+                       i.meta = &imageMeta{Exif: x}
+
+                       // Also write it to cache
+                       enc := json.NewEncoder(w)
+                       return enc.Encode(i.meta)
+
                }
 
-               i.exif = x
+               _, i.metaInitErr = i.getSpec().imageCache.fileCache.ReadOrCreate(key, read, create)
 
        })
 
-       return i.exif, i.exifInitErr
+       if i.metaInitErr != nil {
+               return nil, i.metaInitErr
+       }
+
+       return i.meta.Exif, nil
 }
 
 func (i *imageResource) Clone() resource.Resource {
@@ -271,6 +314,17 @@ func (i *imageResource) setBasePath(conf images.ImageConfig) {
        i.getResourcePaths().relTargetDirFile = i.relTargetPathFromConfig(conf)
 }
 
+func (i *imageResource) getImageMetaCacheTargetPath() string {
+       const imageMetaVersionNumber = 1 // Increment to invalidate the meta cache
+
+       cfg := i.getSpec().imaging.Cfg
+       df := i.getResourcePaths().relTargetDirFile
+       p1, _ := helpers.FileAndExt(df.file)
+       h, _ := i.hash()
+       idStr := internal.HashString(h, i.size(), imageMetaVersionNumber, cfg)
+       return path.Join(df.dir, fmt.Sprintf("%s%s.json", p1, idStr))
+}
+
 func (i *imageResource) relTargetPathFromConfig(conf images.ImageConfig) dirFile {
        p1, p2 := helpers.FileAndExt(i.getResourcePaths().relTargetDirFile.file)
        if conf.Action == "trace" {
index 6d879d2717a18deae997c5b78f984c8cd4fa457a..5afc4e36a2210994c873bf92b3e5cc2757fdf14e 100644 (file)
@@ -15,6 +15,7 @@ package resources
 
 import (
        "fmt"
+       "math/big"
        "math/rand"
        "os"
        "path/filepath"
@@ -22,6 +23,7 @@ import (
        "strconv"
        "sync"
        "testing"
+       "time"
 
        "github.com/spf13/afero"
 
@@ -50,6 +52,14 @@ var eq = qt.CmpEquals(
        cmp.Comparer(func(m1, m2 media.Type) bool {
                return m1.Type() == m2.Type()
        }),
+       cmp.Comparer(
+               func(v1, v2 *big.Rat) bool {
+                       return v1.RatString() == v2.RatString()
+               },
+       ),
+       cmp.Comparer(func(v1, v2 time.Time) bool {
+               return v1.Unix() == v2.Unix()
+       }),
 )
 
 func TestImageTransformBasic(t *testing.T) {
@@ -336,27 +346,35 @@ func TestSVGImageContent(t *testing.T) {
 
 func TestImageExif(t *testing.T) {
        c := qt.New(t)
-       image := fetchImage(c, "sunset.jpg")
+       fs := afero.NewMemMapFs()
+       spec := newTestResourceSpec(specDescriptor{fs: fs, c: c})
+       image := fetchResourceForSpec(spec, c, "sunset.jpg").(resource.Image)
 
-       x, err := image.Exif()
-       c.Assert(err, qt.IsNil)
-       c.Assert(x, qt.Not(qt.IsNil))
+       getAndCheckExif := func(c *qt.C, image resource.Image) {
+               x, err := image.Exif()
+               c.Assert(err, qt.IsNil)
+               c.Assert(x, qt.Not(qt.IsNil))
 
-       c.Assert(x.Date.Format("2006-01-02"), qt.Equals, "2017-10-27")
+               c.Assert(x.Date.Format("2006-01-02"), qt.Equals, "2017-10-27")
 
-       // Malaga: https://goo.gl/taazZy
-       c.Assert(x.Lat, qt.Equals, float64(36.59744166666667))
-       c.Assert(x.Long, qt.Equals, float64(-4.50846))
+               // Malaga: https://goo.gl/taazZy
+               c.Assert(x.Lat, qt.Equals, float64(36.59744166666667))
+               c.Assert(x.Long, qt.Equals, float64(-4.50846))
 
-       v, found := x.Values["LensModel"]
-       c.Assert(found, qt.Equals, true)
-       lensModel, ok := v.(string)
-       c.Assert(ok, qt.Equals, true)
-       c.Assert(lensModel, qt.Equals, "smc PENTAX-DA* 16-50mm F2.8 ED AL [IF] SDM")
+               v, found := x.Tags["LensModel"]
+               c.Assert(found, qt.Equals, true)
+               lensModel, ok := v.(string)
+               c.Assert(ok, qt.Equals, true)
+               c.Assert(lensModel, qt.Equals, "smc PENTAX-DA* 16-50mm F2.8 ED AL [IF] SDM")
+               resized, _ := image.Resize("300x200")
+               x2, _ := resized.Exif()
+               c.Assert(x2, eq, x)
+       }
 
-       resized, _ := image.Resize("300x200")
-       x2, _ := resized.Exif()
-       c.Assert(x2, qt.Equals, x)
+       getAndCheckExif(c, image)
+       image = fetchResourceForSpec(spec, c, "sunset.jpg").(resource.Image)
+       // This will read from file cache.
+       getAndCheckExif(c, image)
 
 }
 
index 7a3c982c17344ef680ae655a12bb6494a5cbb9b1..b5161f770ce76405fe132001fb905aac9a50665b 100644 (file)
@@ -24,6 +24,8 @@ import (
        "unicode"
        "unicode/utf8"
 
+       "github.com/bep/tmc"
+
        _exif "github.com/rwcarlsen/goexif/exif"
        "github.com/rwcarlsen/goexif/tiff"
 )
@@ -31,10 +33,10 @@ import (
 const exifTimeLayout = "2006:01:02 15:04:05"
 
 type Exif struct {
-       Lat    float64
-       Long   float64
-       Date   time.Time
-       Values map[string]interface{}
+       Lat  float64
+       Long float64
+       Date time.Time
+       Tags Tags
 }
 
 type Decoder struct {
@@ -139,7 +141,7 @@ func (d *Decoder) Decode(r io.Reader) (ex *Exif, err error) {
                return
        }
 
-       ex = &Exif{Lat: lat, Long: long, Date: tm, Values: walker.vals}
+       ex = &Exif{Lat: lat, Long: long, Date: tm, Tags: walker.vals}
 
        return
 }
@@ -240,3 +242,30 @@ func nullString(in []byte) string {
 
        return ""
 }
+
+var tcodec *tmc.Codec
+
+func init() {
+       var err error
+       tcodec, err = tmc.New()
+       if err != nil {
+               panic(err)
+       }
+}
+
+type Tags map[string]interface{}
+
+func (v *Tags) UnmarshalJSON(b []byte) error {
+       vv := make(map[string]interface{})
+       if err := tcodec.Unmarshal(b, &vv); err != nil {
+               return err
+       }
+
+       *v = vv
+
+       return nil
+}
+
+func (v Tags) MarshalJSON() ([]byte, error) {
+       return tcodec.Marshal(v)
+}
index eee60c0896a5f25ba167aa95d83b597f9c9c01e2..c3cfad1cc3c3c1e4a59df8944e671caecd9e328b 100644 (file)
 package exif
 
 import (
+       "encoding/json"
+       "math/big"
        "os"
        "path/filepath"
        "testing"
        "time"
 
        "github.com/gohugoio/hugo/htesting/hqt"
+       "github.com/google/go-cmp/cmp"
 
        qt "github.com/frankban/quicktest"
 )
@@ -40,16 +43,24 @@ func TestExif(t *testing.T) {
        c.Assert(x.Lat, qt.Equals, float64(36.59744166666667))
        c.Assert(x.Long, qt.Equals, float64(-4.50846))
 
-       v, found := x.Values["LensModel"]
+       v, found := x.Tags["LensModel"]
        c.Assert(found, qt.Equals, true)
        lensModel, ok := v.(string)
        c.Assert(ok, qt.Equals, true)
        c.Assert(lensModel, qt.Equals, "smc PENTAX-DA* 16-50mm F2.8 ED AL [IF] SDM")
 
-       v, found = x.Values["DateTime"]
+       v, found = x.Tags["DateTime"]
        c.Assert(found, qt.Equals, true)
        c.Assert(v, hqt.IsSameType, time.Time{})
 
+       // Verify that it survives a round-trip to JSON and back.
+       data, err := json.Marshal(x)
+       c.Assert(err, qt.IsNil)
+       x2 := &Exif{}
+       err = json.Unmarshal(data, x2)
+
+       c.Assert(x2, eq, x)
+
 }
 
 func TestExifPNG(t *testing.T) {
@@ -81,3 +92,14 @@ func BenchmarkDecodeExif(b *testing.B) {
                f.Seek(0, 0)
        }
 }
+
+var eq = qt.CmpEquals(
+       cmp.Comparer(
+               func(v1, v2 *big.Rat) bool {
+                       return v1.RatString() == v2.RatString()
+               },
+       ),
+       cmp.Comparer(func(v1, v2 time.Time) bool {
+               return v1.Unix() == v2.Unix()
+       }),
+)