Add openapi3.Unmarshal
authorBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Tue, 30 Jun 2020 14:11:05 +0000 (16:11 +0200)
committerBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Mon, 6 Jul 2020 18:03:36 +0000 (20:03 +0200)
Fixes #7442
Fixes #7443

12 files changed:
common/types/convert.go
common/types/convert_test.go
go.mod
go.sum
hugolib/openapi_test.go [new file with mode: 0644]
parser/metadecoders/decoder.go
resources/resource/resourcetypes.go
tpl/openapi/openapi3/init.go [new file with mode: 0644]
tpl/openapi/openapi3/openapi3.go [new file with mode: 0644]
tpl/tplimpl/template_funcs.go
tpl/transform/unmarshal.go
tpl/transform/unmarshal_test.go

index 24e01c273f7a69d297532a57c889750a984077b9..137029a0e0cdb3f76312e09b1b169ee6e160b6a3 100644 (file)
@@ -14,6 +14,7 @@
 package types
 
 import (
+       "encoding/json"
        "html/template"
 
        "github.com/spf13/cast"
@@ -59,10 +60,20 @@ func TypeToString(v interface{}) (string, bool) {
 
 // ToString converts v to a string.
 func ToString(v interface{}) string {
+       s, _ := ToStringE(v)
+       return s
+}
+
+// ToStringE converts v to a string.
+func ToStringE(v interface{}) (string, error) {
        if s, ok := TypeToString(v); ok {
-               return s
+               return s, nil
        }
 
-       return cast.ToString(v)
-
+       switch s := v.(type) {
+       case json.RawMessage:
+               return string(s), nil
+       default:
+               return cast.ToStringE(v)
+       }
 }
index 7f86f4c8ac0b6c8a6f8d0e1bab146ab080b3fe78..8a4f04db21cf332f5470ca108e2f7aa9da17de47 100644 (file)
@@ -14,6 +14,7 @@
 package types
 
 import (
+       "encoding/json"
        "testing"
 
        qt "github.com/frankban/quicktest"
@@ -27,3 +28,11 @@ func TestToStringSlicePreserveString(t *testing.T) {
        c.Assert(ToStringSlicePreserveString(nil), qt.IsNil)
 
 }
+
+func TestToString(t *testing.T) {
+       c := qt.New(t)
+
+       c.Assert(ToString([]byte("Hugo")), qt.Equals, "Hugo")
+       c.Assert(ToString(json.RawMessage("Hugo")), qt.Equals, "Hugo")
+
+}
diff --git a/go.mod b/go.mod
index ad6ecda86a0767373b44594d8a54702779a15abc..3969f67fc1f331e90f6fcf628ba7379a4d3a6e96 100644 (file)
--- a/go.mod
+++ b/go.mod
@@ -18,6 +18,8 @@ require (
        github.com/fortytw2/leaktest v1.3.0
        github.com/frankban/quicktest v1.7.2
        github.com/fsnotify/fsnotify v1.4.7
+       github.com/getkin/kin-openapi v0.14.0
+       github.com/ghodss/yaml v1.0.0
        github.com/gobwas/glob v0.2.3
        github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95
        github.com/google/go-cmp v0.3.2-0.20191028172631-481baca67f93
@@ -65,7 +67,7 @@ require (
        google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69 // indirect
        gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
        gopkg.in/ini.v1 v1.51.1 // indirect
-       gopkg.in/yaml.v2 v2.2.7
+       gopkg.in/yaml.v2 v2.3.0
 )
 
 replace github.com/markbates/inflect => github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6
diff --git a/go.sum b/go.sum
index 98f010ece2b03fc6ee29d141f1e37ead0f60ebad..7549b18227ef97bbcef069fa1bf279f13db6144e 100644 (file)
--- a/go.sum
+++ b/go.sum
@@ -127,6 +127,9 @@ github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3B
 github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/getkin/kin-openapi v0.14.0 h1:hqwQL7kze/adt0wB+0UJR2nJm+gfUHqM0Gu4D8nByVc=
+github.com/getkin/kin-openapi v0.14.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw=
+github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -352,6 +355,7 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tdewolff/minify/v2 v2.6.1 h1:UJLhbs2Q/iDrqA79EEyKE48uYHeAMPVdiUzdtKsatJ8=
@@ -550,6 +554,8 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/hugolib/openapi_test.go b/hugolib/openapi_test.go
new file mode 100644 (file)
index 0000000..82f0803
--- /dev/null
@@ -0,0 +1,69 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+       "strings"
+       "testing"
+)
+
+func TestOpenAPI3(t *testing.T) {
+       const openapi3Yaml = `openapi: 3.0.0
+info:
+  title: Sample API
+  description: Optional multiline or single-line description in [CommonMark](http://commonmark.org/help/) or HTML.
+  version: 0.1.9
+servers:
+  - url: http://api.example.com/v1
+    description: Optional server description, e.g. Main (production) server
+  - url: http://staging-api.example.com
+    description: Optional server description, e.g. Internal staging server for testing
+paths:
+  /users:
+    get:
+      summary: Returns a list of users.
+      description: Optional extended description in CommonMark or HTML.
+      responses:
+        '200':    # status code
+          description: A JSON array of user names
+          content:
+            application/json:
+              schema: 
+                type: array
+                items: 
+                  type: string
+`
+
+       b := newTestSitesBuilder(t).Running()
+       b.WithSourceFile("assets/api/myapi.yaml", openapi3Yaml)
+
+       b.WithTemplatesAdded("index.html", `
+{{ $api := resources.Get "api/myapi.yaml" | openapi3.Unmarshal }}
+
+API: {{ $api.Info.Title | safeHTML }}
+
+
+`)
+
+       b.Build(BuildCfg{})
+
+       b.AssertFileContent("public/index.html", `API: Sample API`)
+
+       b.EditFiles("assets/api/myapi.yaml", strings.Replace(openapi3Yaml, "Sample API", "Hugo API", -1))
+
+       b.Build(BuildCfg{})
+
+       b.AssertFileContent("public/index.html", `API: Hugo API`)
+
+}
index 1a4a57076ba18e461014e175f7097a61d1d5aca1..2624ad16f6bdd027aadb39b7bf5f7ffe02f88bba 100644 (file)
@@ -63,7 +63,7 @@ func (d Decoder) UnmarshalToMap(data []byte, f Format) (map[string]interface{},
                return m, nil
        }
 
-       err := d.unmarshal(data, f, &m)
+       err := d.UnmarshalTo(data, f, &m)
 
        return m, err
 }
@@ -122,13 +122,13 @@ func (d Decoder) Unmarshal(data []byte, f Format) (interface{}, error) {
 
        }
        var v interface{}
-       err := d.unmarshal(data, f, &v)
+       err := d.UnmarshalTo(data, f, &v)
 
        return v, err
 }
 
-// unmarshal unmarshals data in format f into v.
-func (d Decoder) unmarshal(data []byte, f Format, v interface{}) error {
+// UnmarshalTo unmarshals data in format f into v.
+func (d Decoder) UnmarshalTo(data []byte, f Format, v interface{}) error {
 
        var err error
 
@@ -156,15 +156,17 @@ func (d Decoder) unmarshal(data []byte, f Format, v interface{}) error {
                case *interface{}:
                        ptr = *v.(*interface{})
                default:
-                       return errors.Errorf("unknown type %T in YAML unmarshal", v)
+                       // Not a map.
                }
 
-               if mm, changed := stringifyMapKeys(ptr); changed {
-                       switch v.(type) {
-                       case *map[string]interface{}:
-                               *v.(*map[string]interface{}) = mm.(map[string]interface{})
-                       case *interface{}:
-                               *v.(*interface{}) = mm
+               if ptr != nil {
+                       if mm, changed := stringifyMapKeys(ptr); changed {
+                               switch v.(type) {
+                               case *map[string]interface{}:
+                                       *v.(*map[string]interface{}) = mm.(map[string]interface{})
+                               case *interface{}:
+                                       *v.(*interface{}) = mm
+                               }
                        }
                }
        case CSV:
index 62431c06ceea62fbbbbafdc79ae4092e19cc03cf..13ffc5ae3eae51d16edfcb014148b84159b9ede1 100644 (file)
@@ -173,6 +173,12 @@ type TranslationKeyProvider interface {
        TranslationKey() string
 }
 
+// UnmarshableResource represents a Resource that can be unmarshaled to some other format.
+type UnmarshableResource interface {
+       ReadSeekCloserResource
+       Identifier
+}
+
 type resourceTypesHolder struct {
        mediaType    media.Type
        resourceType string
diff --git a/tpl/openapi/openapi3/init.go b/tpl/openapi/openapi3/init.go
new file mode 100644 (file)
index 0000000..1e1a4ae
--- /dev/null
@@ -0,0 +1,42 @@
+// Copyright 2020 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openapi3
+
+import (
+       "github.com/gohugoio/hugo/deps"
+       "github.com/gohugoio/hugo/tpl/internal"
+)
+
+const name = "openapi3"
+
+func init() {
+       f := func(d *deps.Deps) *internal.TemplateFuncsNamespace {
+               ctx := New(d)
+
+               ns := &internal.TemplateFuncsNamespace{
+                       Name:    name,
+                       Context: func(args ...interface{}) interface{} { return ctx },
+               }
+
+               ns.AddMethodMapping(ctx.Unmarshal,
+                       nil,
+                       [][2]string{},
+               )
+
+               return ns
+
+       }
+
+       internal.AddTemplateFuncsNamespace(f)
+}
diff --git a/tpl/openapi/openapi3/openapi3.go b/tpl/openapi/openapi3/openapi3.go
new file mode 100644 (file)
index 0000000..7dfd2f6
--- /dev/null
@@ -0,0 +1,97 @@
+// Copyright 2020 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openapi3
+
+import (
+       "io/ioutil"
+
+       gyaml "github.com/ghodss/yaml"
+
+       "github.com/pkg/errors"
+
+       kopenapi3 "github.com/getkin/kin-openapi/openapi3"
+       "github.com/gohugoio/hugo/cache/namedmemcache"
+       "github.com/gohugoio/hugo/deps"
+       "github.com/gohugoio/hugo/parser/metadecoders"
+       "github.com/gohugoio/hugo/resources/resource"
+)
+
+// New returns a new instance of the openapi3-namespaced template functions.
+func New(deps *deps.Deps) *Namespace {
+       // TODO1 consolidate when merging that "other branch" -- but be aware of the keys.
+       cache := namedmemcache.New()
+       deps.BuildStartListeners.Add(
+               func() {
+                       cache.Clear()
+               })
+
+       return &Namespace{
+               cache: cache,
+               deps:  deps,
+       }
+}
+
+// Namespace provides template functions for the "openapi3".
+type Namespace struct {
+       cache *namedmemcache.Cache
+       deps  *deps.Deps
+}
+
+func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*kopenapi3.Swagger, error) {
+
+       key := r.Key()
+       if key == "" {
+               return nil, errors.New("no Key set in Resource")
+       }
+
+       v, err := ns.cache.GetOrCreate(key, func() (interface{}, error) {
+               f := metadecoders.FormatFromMediaType(r.MediaType())
+               if f == "" {
+                       return nil, errors.Errorf("MIME %q not supported", r.MediaType())
+               }
+
+               reader, err := r.ReadSeekCloser()
+               if err != nil {
+                       return nil, err
+               }
+               defer reader.Close()
+
+               b, err := ioutil.ReadAll(reader)
+               if err != nil {
+                       return nil, err
+               }
+
+               s := &kopenapi3.Swagger{}
+               switch f {
+               case metadecoders.YAML:
+                       err = gyaml.Unmarshal(b, s)
+               default:
+                       err = metadecoders.Default.UnmarshalTo(b, f, s)
+               }
+               if err != nil {
+                       return nil, err
+               }
+
+               err = kopenapi3.NewSwaggerLoader().ResolveRefsIn(s, nil)
+
+               return s, err
+       })
+
+       if err != nil {
+               return nil, err
+       }
+
+       return v.(*kopenapi3.Swagger), nil
+
+}
index ccf33d8ba25bcd56994106bdee3c06011b51447a..9141de3f17faa03ffb662844364db6b2a416cfe8 100644 (file)
@@ -44,6 +44,7 @@ import (
        _ "github.com/gohugoio/hugo/tpl/inflect"
        _ "github.com/gohugoio/hugo/tpl/lang"
        _ "github.com/gohugoio/hugo/tpl/math"
+       _ "github.com/gohugoio/hugo/tpl/openapi/openapi3"
        _ "github.com/gohugoio/hugo/tpl/os"
        _ "github.com/gohugoio/hugo/tpl/partials"
        _ "github.com/gohugoio/hugo/tpl/path"
index da06b6aa12494e1187ea738f633bb56f5bf5c306..b606c870aa74da4cf6ad38f836f39e2cf6a24830 100644 (file)
@@ -17,17 +17,20 @@ import (
        "io/ioutil"
        "strings"
 
+       "github.com/gohugoio/hugo/resources/resource"
+
+       "github.com/gohugoio/hugo/common/types"
+
        "github.com/mitchellh/mapstructure"
 
        "github.com/gohugoio/hugo/helpers"
        "github.com/gohugoio/hugo/parser/metadecoders"
-       "github.com/gohugoio/hugo/resources/resource"
        "github.com/pkg/errors"
 
        "github.com/spf13/cast"
 )
 
-// Unmarshal unmarshals the data given, which can be either a string
+// Unmarshal unmarshals the data given, which can be either a string, json.RawMessage
 // or a Resource. Supported formats are JSON, TOML, YAML, and CSV.
 // You can optionally provide an options map as the first argument.
 func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) {
@@ -55,7 +58,7 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) {
                }
        }
 
-       if r, ok := data.(unmarshableResource); ok {
+       if r, ok := data.(resource.UnmarshableResource); ok {
                key := r.Key()
 
                if key == "" {
@@ -87,7 +90,7 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) {
                })
        }
 
-       dataStr, err := cast.ToStringE(data)
+       dataStr, err := types.ToStringE(data)
        if err != nil {
                return nil, errors.Errorf("type %T not supported", data)
        }
@@ -104,12 +107,6 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) {
        })
 }
 
-// All the relevant resources implements this interface.
-type unmarshableResource interface {
-       resource.ReadSeekCloserResource
-       resource.Identifier
-}
-
 func decodeDecoder(m map[string]interface{}) (metadecoders.Decoder, error) {
        opts := metadecoders.Default
 
index 7b0caa07f05dda81a8d68291dc7a95f173e5a5eb..183bdefd5bc5b96dc2e964e6c33d2c7ea45844ed 100644 (file)
@@ -20,11 +20,11 @@ import (
        "testing"
 
        "github.com/gohugoio/hugo/common/hugio"
+       "github.com/gohugoio/hugo/resources/resource"
 
        "github.com/gohugoio/hugo/media"
 
        qt "github.com/frankban/quicktest"
-       "github.com/gohugoio/hugo/resources/resource"
        "github.com/spf13/viper"
 )