GetJSON/GetCSV: Add retry on invalid content

The retry gets triggered when the parsing of the content fails.

Fixes #1166

diff --git a/tpl/template_resources.go b/tpl/template_resources.go
index 9c485b04fafb9889eb5c1098ff94c7db9068e056..c413574d38f74d22bfe6a94e85659f778db45401 100644 (file)
--- a/tpl/template_resources.go
+++ b/tpl/template_resources.go
@@ -23,6 +23,7 @@ import (
        "net/url"
        "strings"
        "sync"
+       "time"
 
        "github.com/spf13/afero"
        "github.com/spf13/hugo/helpers"
@@ -31,7 +32,11 @@ import (
        "github.com/spf13/viper"
 )
 
-var remoteURLLock = &remoteLock{m: make(map[string]*sync.Mutex)}
+var (
+       remoteURLLock = &remoteLock{m: make(map[string]*sync.Mutex)}
+       resSleep      = time.Second * 2 // if JSON decoding failed sleep for n seconds before retrying
+       resRetries    = 1               // number of retries to load the JSON from URL or local file system
+)
 
 type remoteLock struct {
        sync.RWMutex
@@ -90,13 +95,21 @@ func resWriteCache(id string, c []byte, fs afero.Fs) error {
        fID := getCacheFileID(id)
        f, err := fs.Create(fID)
        if err != nil {
-               return err
+               return errors.New("Error: " + err.Error() + ". Failed to create file: " + fID)
        }
+       defer f.Close()
        n, err := f.Write(c)
        if n == 0 {
                return errors.New("No bytes written to file: " + fID)
        }
-       return err
+       if err != nil {
+               return errors.New("Error: " + err.Error() + ". Failed to write to file: " + fID)
+       }
+       return nil
+}
+
+func resDeleteCache(id string, fs afero.Fs) error {
+       return fs.Remove(getCacheFileID(id))
 }
 
 // resGetRemote loads the content of a remote file. This method is thread safe.
@@ -177,18 +190,25 @@ func resGetResource(url string) ([]byte, error) {
 // If you provide multiple parts they will be joined together to the final URL.
 // GetJSON returns nil or parsed JSON to use in a short code.
 func GetJSON(urlParts ...string) interface{} {
+       var v interface{}
        url := strings.Join(urlParts, "")
-       c, err := resGetResource(url)
-       if err != nil {
-               jww.ERROR.Printf("Failed to get json resource %s with error message %s", url, err)
-               return nil
-       }
 
-       var v interface{}
-       err = json.Unmarshal(c, &v)
-       if err != nil {
-               jww.ERROR.Printf("Cannot read json from resource %s with error message %s", url, err)
-               return nil
+       for i := 0; i <= resRetries; i++ {
+               c, err := resGetResource(url)
+               if err != nil {
+                       jww.ERROR.Printf("Failed to get json resource %s with error message %s", url, err)
+                       return nil
+               }
+
+               err = json.Unmarshal(c, &v)
+               if err != nil {
+                       jww.ERROR.Printf("Cannot read json from resource %s with error message %s", url, err)
+                       jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep)
+                       time.Sleep(resSleep)
+                       resDeleteCache(url, hugofs.SourceFs)
+                       continue
+               }
+               break
        }
        return v
 }
@@ -212,16 +232,34 @@ func parseCSV(c []byte, sep string) ([][]string, error) {
 // If you provide multiple parts for the URL they will be joined together to the final URL.
 // GetCSV returns nil or a slice slice to use in a short code.
 func GetCSV(sep string, urlParts ...string) [][]string {
+       var d [][]string
        url := strings.Join(urlParts, "")
-       c, err := resGetResource(url)
-       if err != nil {
-               jww.ERROR.Printf("Failed to get csv resource %s with error message %s", url, err)
-               return nil
+
+       var clearCacheSleep = func(i int, u string) {
+               jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep)
+               time.Sleep(resSleep)
+               resDeleteCache(url, hugofs.SourceFs)
        }
-       d, err := parseCSV(c, sep)
-       if err != nil {
-               jww.ERROR.Printf("Failed to read csv resource %s with error message %s", url, err)
-               return nil
+
+       for i := 0; i <= resRetries; i++ {
+               c, err := resGetResource(url)
+
+               if err == nil && false == bytes.Contains(c, []byte(sep)) {
+                       err = errors.New("Cannot find separator " + sep + " in CSV.")
+               }
+
+               if err != nil {
+                       jww.ERROR.Printf("Failed to read csv resource %s with error message %s", url, err)
+                       clearCacheSleep(i, url)
+                       continue
+               }
+
+               if d, err = parseCSV(c, sep); err != nil {
+                       jww.ERROR.Printf("Failed to parse csv file %s with error message %s", url, err)
+                       clearCacheSleep(i, url)
+                       continue
+               }
+               break
        }
        return d
 }

diff --git a/tpl/template_resources_test.go b/tpl/template_resources_test.go
index 1bdfc7d3ae71fc1fcd8b67c85785c0d5229fe083..4ae0ee0b40ebd1aca895c797323dfae396209c8a 100644 (file)
--- a/tpl/template_resources_test.go
+++ b/tpl/template_resources_test.go
@@ -15,14 +15,20 @@ package tpl
 
 import (
        "bytes"
+       "fmt"
        "net/http"
        "net/http/httptest"
        "net/url"
+       "os"
        "strings"
        "testing"
+       "time"
 
        "github.com/spf13/afero"
        "github.com/spf13/hugo/helpers"
+       "github.com/spf13/hugo/hugofs"
+       "github.com/spf13/viper"
+       "github.com/stretchr/testify/assert"
 )
 
 func TestScpCache(t *testing.T) {
@@ -195,3 +201,106 @@ func TestParseCSV(t *testing.T) {
 
        }
 }
+
+// https://twitter.com/francesc/status/603066617124126720
+// for the construct: defer testRetryWhenDone().Reset()
+type wd struct {
+       Reset func()
+}
+
+func testRetryWhenDone() wd {
+       cd := viper.GetString("CacheDir")
+       viper.Set("CacheDir", helpers.GetTempDir("", hugofs.SourceFs))
+       var tmpSleep time.Duration
+       tmpSleep, resSleep = resSleep, time.Millisecond
+       return wd{func() {
+               viper.Set("CacheDir", cd)
+               resSleep = tmpSleep
+       }}
+}
+
+func TestGetJSONFailParse(t *testing.T) {
+       defer testRetryWhenDone().Reset()
+
+       reqCount := 0
+       ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+               if reqCount > 0 {
+                       w.Header().Add("Content-type", "application/json")
+                       fmt.Fprintln(w, `{"gomeetup":["Sydney", "San Francisco", "Stockholm"]}`)
+               } else {
+                       w.WriteHeader(http.StatusInternalServerError)
+                       fmt.Fprintln(w, `ERROR 500`)
+               }
+               reqCount++
+       }))
+       defer ts.Close()
+       url := ts.URL + "/test.json"
+       defer os.Remove(getCacheFileID(url))
+
+       want := map[string]interface{}{"gomeetup": []interface{}{"Sydney", "San Francisco", "Stockholm"}}
+       have := GetJSON(url)
+       assert.NotNil(t, have)
+       if have != nil {
+               assert.EqualValues(t, want, have)
+       }
+}
+
+func TestGetCSVFailParseSep(t *testing.T) {
+       defer testRetryWhenDone().Reset()
+
+       reqCount := 0
+       ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+               if reqCount > 0 {
+                       w.Header().Add("Content-type", "application/json")
+                       fmt.Fprintln(w, `gomeetup,city`)
+                       fmt.Fprintln(w, `yes,Sydney`)
+                       fmt.Fprintln(w, `yes,San Francisco`)
+                       fmt.Fprintln(w, `yes,Stockholm`)
+               } else {
+                       w.WriteHeader(http.StatusInternalServerError)
+                       fmt.Fprintln(w, `ERROR 500`)
+               }
+               reqCount++
+       }))
+       defer ts.Close()
+       url := ts.URL + "/test.csv"
+       defer os.Remove(getCacheFileID(url))
+
+       want := [][]string{[]string{"gomeetup", "city"}, []string{"yes", "Sydney"}, []string{"yes", "San Francisco"}, []string{"yes", "Stockholm"}}
+       have := GetCSV(",", url)
+       assert.NotNil(t, have)
+       if have != nil {
+               assert.EqualValues(t, want, have)
+       }
+}
+
+func TestGetCSVFailParse(t *testing.T) {
+       defer testRetryWhenDone().Reset()
+
+       reqCount := 0
+       ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+               w.Header().Add("Content-type", "application/json")
+               if reqCount > 0 {
+                       fmt.Fprintln(w, `gomeetup,city`)
+                       fmt.Fprintln(w, `yes,Sydney`)
+                       fmt.Fprintln(w, `yes,San Francisco`)
+                       fmt.Fprintln(w, `yes,Stockholm`)
+               } else {
+                       fmt.Fprintln(w, `gomeetup,city`)
+                       fmt.Fprintln(w, `yes,Sydney,Bondi,`) // wrong number of fields in line
+                       fmt.Fprintln(w, `yes,San Francisco`)
+                       fmt.Fprintln(w, `yes,Stockholm`)
+               }
+               reqCount++
+       }))
+       defer ts.Close()
+       url := ts.URL + "/test.csv"
+       defer os.Remove(getCacheFileID(url))
+
+       want := [][]string{[]string{"gomeetup", "city"}, []string{"yes", "Sydney"}, []string{"yes", "San Francisco"}, []string{"yes", "Stockholm"}}
+       have := GetCSV(",", url)
+       assert.NotNil(t, have)
+       if have != nil {
+               assert.EqualValues(t, want, have)
+       }
+}