From 84950ed2b215a42a3b80f684ffc7e914f5015d29 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Thu, 20 Nov 2025 11:01:23 +0100 Subject: [PATCH] tpl/openapi: Add support for OpenAPI external file references Fixes #8067 --- deps/deps.go | 3 + hugolib/integrationtest_builder.go | 21 +- hugolib/rebuild_test.go | 2 +- hugolib/site.go | 8 + resources/resource.go | 13 + tpl/openapi/openapi3/init.go | 12 + tpl/openapi/openapi3/openapi3.go | 114 +++++++- .../openapi3/openapi3_integration_test.go | 247 +++++++++++++++++- 8 files changed, 404 insertions(+), 16 deletions(-) diff --git a/deps/deps.go b/deps/deps.go index acafb850e..0d02bbda2 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -417,6 +417,9 @@ type DepsCfg struct { // i18n handling. TranslationProvider ResourceProvider + // Build triggered by the IntegrationTest framework. + IsIntegrationTest bool + // ChangesFromBuild for changes passed back to the server/watch process. ChangesFromBuild chan []identity.Identity } diff --git a/hugolib/integrationtest_builder.go b/hugolib/integrationtest_builder.go index c9d7beea9..163f03665 100644 --- a/hugolib/integrationtest_builder.go +++ b/hugolib/integrationtest_builder.go @@ -35,6 +35,7 @@ import ( "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugofs/hglob" "github.com/gohugoio/hugo/hugolib/sitesmatrix" + "github.com/gohugoio/hugo/identity" "github.com/spf13/afero" "github.com/spf13/cast" "golang.org/x/text/unicode/norm" @@ -838,7 +839,11 @@ func (s *IntegrationTestBuilder) initBuilder() error { s.Assert(err, qt.IsNil) - depsCfg := deps.DepsCfg{Configs: res, Fs: fs, LogLevel: logger.Level(), StdErr: logger.StdErr()} + // changes received from Hugo in watch mode. + // In the full setup, this channel is created in the commands package. + changesFromBuild := make(chan []identity.Identity, 10) + + depsCfg := deps.DepsCfg{Configs: res, Fs: fs, LogLevel: logger.Level(), StdErr: logger.StdErr(), ChangesFromBuild: changesFromBuild, IsIntegrationTest: true} sites, err := NewHugoSites(depsCfg) if err != nil { initErr = err @@ -849,6 +854,20 @@ func (s *IntegrationTestBuilder) initBuilder() error { return } + go func() { + for id := range changesFromBuild { + whatChanged := &WhatChanged{} + for _, v := range id { + whatChanged.Add(v) + } + bcfg := s.Cfg.BuildCfg + bcfg.WhatChanged = whatChanged + if err := s.build(bcfg); err != nil { + s.Fatalf("Build failed after change: %s", err) + } + } + }() + s.H = sites s.fs = fs diff --git a/hugolib/rebuild_test.go b/hugolib/rebuild_test.go index afb2d7909..c4d7f754f 100644 --- a/hugolib/rebuild_test.go +++ b/hugolib/rebuild_test.go @@ -845,7 +845,7 @@ func TestRebuildEditSectionRemoveDate(t *testing.T) { func TestRebuildVariations(t *testing.T) { // t.Parallel() not supported, see https://github.com/fortytw2/leaktest/issues/4 // This leaktest seems to be a little bit shaky on Travis. - if !htesting.IsCI() { + if !htesting.IsRealCI() { defer leaktest.CheckTimeout(t, 10*time.Second)() } diff --git a/hugolib/site.go b/hugolib/site.go index edcf2548a..a01afe6de 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -265,6 +265,14 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { return nil, err } + // Prevent leaking goroutines in tests. + if cfg.IsIntegrationTest && cfg.ChangesFromBuild != nil { + firstSiteDeps.BuildClosers.Add(types.CloserFunc(func() error { + close(cfg.ChangesFromBuild) + return nil + })) + } + batcherClient, err := esbuild.NewBatcherClient(firstSiteDeps) if err != nil { return nil, err diff --git a/resources/resource.go b/resources/resource.go index 0e7b36f85..9886f2db4 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -25,6 +25,7 @@ import ( "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/resources/internal" + "github.com/spf13/cast" "github.com/gohugoio/hugo/common/hashing" "github.com/gohugoio/hugo/common/herrors" @@ -703,6 +704,18 @@ func InternalResourceSourcePath(r resource.Resource) string { return "" } +// InternalResourceSourceContent is used internally to get the source content for a Resource. +func InternalResourceSourceContent(ctx context.Context, r resource.Resource) (string, error) { + if cp, ok := r.(resource.ContentProvider); ok { + c, err := cp.Content(ctx) + if err != nil { + return "", err + } + return cast.ToStringE(c) + } + return "", nil +} + // InternalResourceSourcePathBestEffort is used internally to get the source path for a Resource. // Used for error messages etc. // It will fall back to the target path if the source path is not available. diff --git a/tpl/openapi/openapi3/init.go b/tpl/openapi/openapi3/init.go index 4d88e148e..7afcf6670 100644 --- a/tpl/openapi/openapi3/init.go +++ b/tpl/openapi/openapi3/init.go @@ -18,6 +18,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/tpl/internal" + resourcestpl "github.com/gohugoio/hugo/tpl/resources" ) const name = "openapi3" @@ -29,6 +30,17 @@ func init() { ns := &internal.TemplateFuncsNamespace{ Name: name, Context: func(cctx context.Context, args ...any) (any, error) { return ctx, nil }, + OnCreated: func(m map[string]any) { + for _, v := range m { + switch v := v.(type) { + case *resourcestpl.Namespace: + ctx.resourcesNs = v + } + } + if ctx.resourcesNs == nil { + panic("resources namespace not found") + } + }, } ns.AddMethodMapping(ctx.Unmarshal, diff --git a/tpl/openapi/openapi3/openapi3.go b/tpl/openapi/openapi3/openapi3.go index 9bdc93eeb..4cadae523 100644 --- a/tpl/openapi/openapi3/openapi3.go +++ b/tpl/openapi/openapi3/openapi3.go @@ -15,16 +15,26 @@ package openapi3 import ( + "context" "errors" "fmt" "io" + "net/url" + "path" + "path/filepath" + "strings" kopenapi3 "github.com/getkin/kin-openapi/openapi3" "github.com/gohugoio/hugo/cache/dynacache" + "github.com/gohugoio/hugo/common/hashing" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/parser/metadecoders" + "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" + resourcestpl "github.com/gohugoio/hugo/tpl/resources" + "github.com/mitchellh/mapstructure" ) // New returns a new instance of the openapi3-namespaced template functions. @@ -37,8 +47,9 @@ func New(deps *deps.Deps) *Namespace { // Namespace provides template functions for the "openapi3". type Namespace struct { - cache *dynacache.Partition[string, *OpenAPIDocument] - deps *deps.Deps + cache *dynacache.Partition[string, *OpenAPIDocument] + deps *deps.Deps + resourcesNs *resourcestpl.Namespace } // OpenAPIDocument represents an OpenAPI 3 document. @@ -51,13 +62,35 @@ func (o *OpenAPIDocument) GetIdentityGroup() identity.Identity { return o.identityGroup } +type unmarshalOptions struct { + // Options passed to resources.GetRemote when resolving remote $ref. + GetRemote map[string]any +} + // Unmarshal unmarshals the given resource into an OpenAPI 3 document. -func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*OpenAPIDocument, error) { +func (ns *Namespace) Unmarshal(ctx context.Context, args ...any) (*OpenAPIDocument, error) { + if len(args) < 1 || len(args) > 2 { + return nil, errors.New("must provide a Resource and optionally an options map") + } + + r := args[0].(resource.UnmarshableResource) key := r.Key() if key == "" { return nil, errors.New("no Key set in Resource") } + var opts unmarshalOptions + if len(args) > 1 { + optsm, err := maps.ToStringMapE(args[1]) + if err != nil { + return nil, err + } + if err := mapstructure.WeakDecode(optsm, &opts); err != nil { + return nil, err + } + key += "_" + hashing.HashString(optsm) + } + v, err := ns.cache.GetOrCreate(key, func(string) (*OpenAPIDocument, error) { f := metadecoders.FormatFromStrings(r.MediaType().Suffixes()...) if f == "" { @@ -86,9 +119,37 @@ func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*OpenAPIDocument return nil, err } - err = kopenapi3.NewLoader().ResolveRefsIn(s, nil) + var resourcePath string + if res, ok := r.(resource.Resource); ok { + resourcePath = resources.InternalResourceSourcePath(res) + } + var relDir string + if resourcePath != "" { + if rel, ok := ns.deps.Assets.MakePathRelative(filepath.FromSlash(resourcePath), true); ok { + relDir = filepath.Dir(rel) + } + } + + var idm identity.Manager = identity.NopManager + if v := identity.GetDependencyManager(r); v != nil { + idm = v + } + idg := identity.FirstIdentity(r) + + resolver := &refResolver{ + ctx: ctx, + idm: idm, + opts: opts, + relBase: filepath.ToSlash(relDir), + ns: ns, + } - return &OpenAPIDocument{T: s, identityGroup: identity.FirstIdentity(r)}, err + loader := kopenapi3.NewLoader() + loader.IsExternalRefsAllowed = true + loader.ReadFromURIFunc = resolver.resolveExternalRef + err = loader.ResolveRefsIn(s, nil) + + return &OpenAPIDocument{T: s, identityGroup: idg}, err }) if err != nil { return nil, err @@ -96,3 +157,46 @@ func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*OpenAPIDocument return v, nil } + +type refResolver struct { + ctx context.Context + idm identity.Manager + opts unmarshalOptions + relBase string + ns *Namespace +} + +// resolveExternalRef resolves external references in OpenAPI documents by either fetching +// remote URLs or loading local files from the assets directory, depending on the reference location. +func (r *refResolver) resolveExternalRef(loader *kopenapi3.Loader, loc *url.URL) ([]byte, error) { + if loc.Scheme != "" && loc.Host != "" { + res, err := r.ns.resourcesNs.GetRemote(loc.String(), r.opts.GetRemote) + if err != nil { + return nil, fmt.Errorf("failed to get remote ref %q: %w", loc.String(), err) + } + content, err := resources.InternalResourceSourceContent(r.ctx, res) + if err != nil { + return nil, fmt.Errorf("failed to read remote ref %q: %w", loc.String(), err) + } + r.idm.AddIdentity(identity.FirstIdentity(res)) + return []byte(content), nil + } + + var filename string + if strings.HasPrefix(loc.Path, "/") { + filename = loc.Path + } else { + filename = path.Join(r.relBase, loc.Path) + } + + res := r.ns.resourcesNs.Get(filename) + if res == nil { + return nil, fmt.Errorf("local ref %q not found", loc.String()) + } + content, err := resources.InternalResourceSourceContent(r.ctx, res) + if err != nil { + return nil, fmt.Errorf("failed to read local ref %q: %w", loc.String(), err) + } + r.idm.AddIdentity(identity.FirstIdentity(res)) + return []byte(content), nil +} diff --git a/tpl/openapi/openapi3/openapi3_integration_test.go b/tpl/openapi/openapi3/openapi3_integration_test.go index 6914a60b3..9040f965b 100644 --- a/tpl/openapi/openapi3/openapi3_integration_test.go +++ b/tpl/openapi/openapi3/openapi3_integration_test.go @@ -14,8 +14,12 @@ package openapi3_test import ( + "fmt" + "net/http" + "net/http/httptest" "strings" "testing" + "time" "github.com/gohugoio/hugo/hugolib" ) @@ -49,20 +53,15 @@ paths: type: array items: type: string --- config.toml -- +-- hugo.toml -- baseURL = 'http://example.com/' --- layouts/index.html -- +disableLiveReload = true +-- layouts/home.html -- {{ $api := resources.Get "api/myapi.yaml" | openapi3.Unmarshal }} API: {{ $api.Info.Title | safeHTML }} ` - b := hugolib.NewIntegrationTestBuilder( - hugolib.IntegrationTestConfig{ - T: t, - Running: true, - TxtarString: files, - }, - ).Build() + b := hugolib.TestRunning(t, files) b.AssertFileContent("public/index.html", `API: Sample API`) @@ -72,3 +71,233 @@ API: {{ $api.Info.Title | safeHTML }} b.AssertFileContent("public/index.html", `API: Hugo API`) } + +// Test data borrowed/adapted from https://github.com/getkin/kin-openapi/tree/master/openapi3/testdata/refInLocalRef +// The templates below would be simpler if kin-openapi's T would serialize to JSON better, see https://github.com/getkin/kin-openapi/issues/561 +const refLocalTemplate = ` +-- hugo.toml -- +baseURL = 'http://example.com/' +disableKinds = ["page", "section", "taxonomy", "term", "sitemap", "robotsTXT", "404"] +disableLiveReload = true +[HTTPCache] +[HTTPCache.cache] +[HTTPCache.cache.for] +includes = ['**'] +[[HTTPCache.polls]] +high = '100ms' +low = '50ms' +[HTTPCache.polls.for] +includes = ['**'] +-- assets/api/myapi.json -- +{ + "openapi": "3.0.3", + "info": { + "title": "Reference in reference example", + "version": "1.0.0" + }, + "paths": { + "/api/test/ref/in/ref": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties" : { + "data": { + "$ref": "#/components/schemas/Request" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "$ref": "messages/response.json" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "Request": { + "$ref": "messages/request.json" + } + } + } +} +-- assets/api/messages/request.json -- +{ + "type": "object", + "required": [ + "definition_reference" + ], + "properties": { + "definition_reference": { + "$ref": "./data.json" + } + } +} +-- assets/api/messages/response.json -- +{ + "type": "object", + "properties": { + "id": { + "type": "integer", + "format": "uint64" + } + } +} +-- assets/api/messages/data.json -- +{ + "type": "object", + "properties": { + "id": { + "type": "integer", + "format": "int32" + }, + "ref_prop_part": { + "$ref": "DATAPART_REF" + } + } +} +-- assets/api/messages/dataPart.json -- +{ + "type": "object", + "properties": { + "idPart": { + "type": "integer", + "format": "int64" + } + } +} +-- layouts/home.html -- +{{ $getremote := dict + "method" "post" + "key" time.Now.UnixNano +}} +{{ $opts := dict + "getremote" $getremote +}} +{{ with resources.Get "api/myapi.json" }} + {{ with openapi3.Unmarshal . $opts }} + Title: {{ .Info.Title | safeHTML }} + {{ range $k, $v := .Paths.Map }} + {{ $post := index $v.Post }} + {{ with index $post.RequestBody.Value.Content }} + {{ $mt := (index . "application/json")}} + {{ $data := index $mt.Schema.Value.Properties "data" }} + RequestBody: {{ template "printschema" $mt.Schema.Value }}$ + {{ end }} + {{ $response := index (index $post.Responses.Map "200").Value.Content "application/json" }} + Response: {{ template "printschema" $response.Schema.Value }} + {{ end }} + {{ end }} +{{ end }} +{{ define "printschema" }}{{ with .Format}}Format: {{ . }}|{{ end }}{{ with .Properties }} Properties: {{ range $k, $v := . }}{{ $k}}: {{ template "printitems" . -}}{{ end }}{{ end -}}${{ end }} +{{ define "printitems" }}{{ if eq (printf "%T" .) "*openapi3.SchemaRef" }}{{ template "printschema" .Value }}{{ else }}{{ template "printitem" . -}}{{ end -}}{{ end }} +{{ define "printitem" }}{{ printf "%T: %s" . (.|debug.Dump) | safeHTML }}{{ end }} + +` + +func TestUnmarshalRefLocal(t *testing.T) { + files := strings.ReplaceAll(refLocalTemplate, "DATAPART_REF", "./dataPart.json") + + b := hugolib.Test(t, files) + + b.AssertFileContent("public/index.html", + `Reference in reference example`, + "RequestBody: Properties: data: Properties: definition_reference: Properties: id: Format: int32|$ref_prop_part: Properties: idPart: Format: int64|$", + "Response: Properties: id: Format: uint64|$", + ) +} + +func TestUnmarshalRefLocalEdit(t *testing.T) { + files := strings.ReplaceAll(refLocalTemplate, "DATAPART_REF", "./dataPart.json") + + b := hugolib.TestRunning(t, files) + + b.AssertFileContent("public/index.html", + "RequestBody: Properties: data: Properties: definition_reference: Properties: id: Format: int32|$ref_prop_part: Properties: idPart: Format: int64|$", + ) + + b.EditFileReplaceAll("assets/api/messages/dataPart.json", "int64", "int8").Build() + + b.AssertFileContent("public/index.html", + "RequestBody: Properties: data: Properties: definition_reference: Properties: id: Format: int32|$ref_prop_part: Properties: idPart: Format: int8|$", + ) +} + +func TestUnmarshalRefRemote(t *testing.T) { + createFiles := func(t *testing.T) string { + counter := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "POST" && r.URL.Path == "/api/messages/dataPart.json" { + n := 8 + + // This endpoint will also be called by the poller, so we distinguish + // between the first and subsequent calls. + if counter > 0 { + n = 16 + } + counter++ + + w.Header().Set("Content-Type", "application/json") + response := fmt.Sprintf(`{ + "type": "object", + "properties": { + "idPart": { + "type": "integer", + "format": "int%d" + } + } +}`, n) + w.Write([]byte(response)) + return + } + + http.Error(w, "Not found", http.StatusNotFound) + })) + + t.Cleanup(func() { + ts.Close() + }) + + dataPartRef := ts.URL + "/api/messages/dataPart.json" + return strings.ReplaceAll(refLocalTemplate, "DATAPART_REF", dataPartRef) + } + + t.Run("Build", func(t *testing.T) { + b := hugolib.Test(t, createFiles(t)) + + b.AssertFileContent("public/index.html", + `Reference in reference example`, + "RequestBody: Properties: data: Properties: definition_reference: Properties: id: Format: int32|$ref_prop_part: Properties: idPart: Format: int8|$", + "Response: Properties: id: Format: uint64|$", + ) + }) + + t.Run("Rebuild", func(t *testing.T) { + b := hugolib.TestRunning(t, createFiles(t)) + + b.AssertFileContent("public/index.html", + "idPart: Format: int8|$", + ) + + // Rebuild triggered by remote polling. + time.Sleep(800 * time.Millisecond) + + b.AssertFileContent("public/index.html", + "idPart: Format: int16|$", + ) + }) +} -- 2.39.5