-// Copyright 2018 The Hugo Authors. All rights reserved.
+// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
"sync"
"time"
+ "github.com/gohugoio/httpcache"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/hugofs"
return
}
+// NamedLock locks the given id. The lock is released when the returned function is called.
+func (c *Cache) NamedLock(id string) func() {
+ id = cleanID(id)
+ c.nlocker.Lock(id)
+ return func() {
+ c.nlocker.Unlock(id)
+ }
+}
+
// GetOrCreate tries to get the file with the given id from cache. If not found or expired, create will
// be invoked and the result cached.
// This method is protected by a named lock using the given id as identifier.
var buff bytes.Buffer
return info,
hugio.ToReadCloser(&buff),
- afero.WriteReader(c.Fs, id, io.TeeReader(r, &buff))
+ c.writeReader(id, io.TeeReader(r, &buff))
+}
+
+func (c *Cache) writeReader(id string, r io.Reader) error {
+ dir := filepath.Dir(id)
+ if dir != "" {
+ _ = c.Fs.MkdirAll(dir, 0o777)
+ }
+ f, err := c.Fs.Create(id)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ _, _ = io.Copy(f, r)
+
+ return nil
}
// GetOrCreateBytes is the same as GetOrCreate, but produces a byte slice.
return info, b, nil
}
- if err := afero.WriteReader(c.Fs, id, bytes.NewReader(b)); err != nil {
+ if err := c.writeReader(id, bytes.NewReader(b)); err != nil {
return info, nil, err
}
+
return info, b, nil
}
return nil
}
- if c.maxAge > 0 {
- fi, err := c.Fs.Stat(id)
- if err != nil {
- return nil
- }
-
- if c.isExpired(fi.ModTime()) {
- c.Fs.Remove(id)
- return nil
- }
+ if removed, err := c.removeIfExpired(id); err != nil || removed {
+ return nil
}
f, err := c.Fs.Open(id)
return f
}
+func (c *Cache) getBytesAndRemoveIfExpired(id string) ([]byte, bool) {
+ if c.maxAge == 0 {
+ // No caching.
+ return nil, false
+ }
+
+ f, err := c.Fs.Open(id)
+ if err != nil {
+ return nil, false
+ }
+ defer f.Close()
+
+ b, err := io.ReadAll(f)
+ if err != nil {
+ return nil, false
+ }
+
+ removed, err := c.removeIfExpired(id)
+ if err != nil {
+ return nil, false
+ }
+
+ return b, removed
+}
+
+func (c *Cache) removeIfExpired(id string) (bool, error) {
+ if c.maxAge <= 0 {
+ return false, nil
+ }
+
+ fi, err := c.Fs.Stat(id)
+ if err != nil {
+ return false, err
+ }
+
+ if c.isExpired(fi.ModTime()) {
+ c.Fs.Remove(id)
+ return true, nil
+ }
+
+ return false, nil
+}
+
func (c *Cache) isExpired(modTime time.Time) bool {
if c.maxAge < 0 {
return false
func cleanID(name string) string {
return strings.TrimPrefix(filepath.Clean(name), helpers.FilePathSeparator)
}
+
+// AsHTTPCache returns an httpcache.Cache implementation for this file cache.
+// Note that none of the methods are protected by named locks, so you need to make sure
+// to do that in your own code.
+func (c *Cache) AsHTTPCache() httpcache.Cache {
+ return &httpCache{c: c}
+}
+
+type httpCache struct {
+ c *Cache
+}
+
+func (h *httpCache) Get(id string) (resp []byte, ok bool) {
+ id = cleanID(id)
+ b, removed := h.c.getBytesAndRemoveIfExpired(id)
+
+ return b, !removed
+}
+
+func (h *httpCache) Set(id string, resp []byte) {
+ if h.c.maxAge == 0 {
+ return
+ }
+
+ id = cleanID(id)
+
+ if err := h.c.writeReader(id, bytes.NewReader(resp)); err != nil {
+ panic(err)
+ }
+}
+
+func (h *httpCache) Delete(key string) {
+ h.c.Fs.Remove(key)
+}
--- /dev/null
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpcache
+
+import (
+ "encoding/json"
+ "time"
+
+ "github.com/gobwas/glob"
+ "github.com/gohugoio/hugo/common/predicate"
+ "github.com/gohugoio/hugo/config"
+ "github.com/mitchellh/mapstructure"
+)
+
+// DefaultConfig holds the default configuration for the HTTP cache.
+var DefaultConfig = Config{
+ Cache: Cache{
+ For: GlobMatcher{
+ Excludes: []string{"**"},
+ },
+ },
+ Polls: []PollConfig{
+ {
+ For: GlobMatcher{
+ Includes: []string{"**"},
+ },
+ Disable: true,
+ },
+ },
+}
+
+// Config holds the configuration for the HTTP cache.
+type Config struct {
+ // Configures the HTTP cache behaviour (RFC 9111).
+ // When this is not enabled for a resource, Hugo will go straight to the file cache.
+ Cache Cache
+
+ // Polls holds a list of configurations for polling remote resources to detect changes in watch mode.
+ // This can be disabled for some resources, typically if they are known to not change.
+ Polls []PollConfig
+}
+
+type Cache struct {
+ // Enable HTTP cache behaviour (RFC 9111) for these rsources.
+ For GlobMatcher
+}
+
+func (c *Config) Compile() (ConfigCompiled, error) {
+ var cc ConfigCompiled
+
+ p, err := c.Cache.For.CompilePredicate()
+ if err != nil {
+ return cc, err
+ }
+
+ cc.For = p
+
+ for _, pc := range c.Polls {
+
+ p, err := pc.For.CompilePredicate()
+ if err != nil {
+ return cc, err
+ }
+
+ cc.PollConfigs = append(cc.PollConfigs, PollConfigCompiled{
+ For: p,
+ Config: pc,
+ })
+ }
+
+ return cc, nil
+}
+
+// PollConfig holds the configuration for polling remote resources to detect changes in watch mode.
+// TODO1 make sure this enabled only in watch mode.
+type PollConfig struct {
+ // What remote resources to apply this configuration to.
+ For GlobMatcher
+
+ // Disable polling for this configuration.
+ Disable bool
+
+ // Low is the lower bound for the polling interval.
+ // This is the starting point when the resource has recently changed,
+ // if that resource stops changing, the polling interval will gradually increase towards High.
+ Low time.Duration
+
+ // High is the upper bound for the polling interval.
+ // This is the interval used when the resource is stable.
+ High time.Duration
+}
+
+func (c PollConfig) MarshalJSON() (b []byte, err error) {
+ // Marshal the durations as strings.
+ type Alias PollConfig
+ return json.Marshal(&struct {
+ Low string
+ High string
+ Alias
+ }{
+ Low: c.Low.String(),
+ High: c.High.String(),
+ Alias: (Alias)(c),
+ })
+}
+
+type GlobMatcher struct {
+ // Excludes holds a list of glob patterns that will be excluded.
+ Excludes []string
+
+ // Includes holds a list of glob patterns that will be included.
+ Includes []string
+}
+
+type ConfigCompiled struct {
+ For predicate.P[string]
+ PollConfigs []PollConfigCompiled
+}
+
+func (c *ConfigCompiled) PollConfigFor(s string) PollConfigCompiled {
+ for _, pc := range c.PollConfigs {
+ if pc.For(s) {
+ return pc
+ }
+ }
+ return PollConfigCompiled{}
+}
+
+func (c *ConfigCompiled) IsPollingDisabled() bool {
+ for _, pc := range c.PollConfigs {
+ if !pc.Config.Disable {
+ return false
+ }
+ }
+ return true
+}
+
+type PollConfigCompiled struct {
+ For predicate.P[string]
+ Config PollConfig
+}
+
+func (p PollConfigCompiled) IsZero() bool {
+ return p.For == nil
+}
+
+func (gm *GlobMatcher) CompilePredicate() (func(string) bool, error) {
+ var p predicate.P[string]
+ for _, include := range gm.Includes {
+ g, err := glob.Compile(include, '/')
+ if err != nil {
+ return nil, err
+ }
+ fn := func(s string) bool {
+ return g.Match(s)
+ }
+ p = p.Or(fn)
+ }
+
+ for _, exclude := range gm.Excludes {
+ g, err := glob.Compile(exclude, '/')
+ if err != nil {
+ return nil, err
+ }
+ fn := func(s string) bool {
+ return !g.Match(s)
+ }
+ p = p.And(fn)
+ }
+
+ return p, nil
+}
+
+func DecodeConfig(bcfg config.BaseConfig, m map[string]any) (Config, error) {
+ if len(m) == 0 {
+ return DefaultConfig, nil
+ }
+
+ var c Config
+
+ dc := &mapstructure.DecoderConfig{
+ Result: &c,
+ DecodeHook: mapstructure.StringToTimeDurationHookFunc(),
+ WeaklyTypedInput: true,
+ }
+
+ decoder, err := mapstructure.NewDecoder(dc)
+ if err != nil {
+ return c, err
+ }
+
+ if err := decoder.Decode(m); err != nil {
+ return c, err
+ }
+
+ return c, nil
+}
--- /dev/null
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpcache_test
+
+import (
+ "testing"
+ "time"
+
+ qt "github.com/frankban/quicktest"
+ "github.com/gohugoio/hugo/hugolib"
+)
+
+func TestConfigCustom(t *testing.T) {
+ files := `
+-- hugo.toml --
+[httpcache]
+[httpcache.cache.for]
+includes = ["**gohugo.io**"]
+[[httpcache.polls]]
+low = "5s"
+high = "32s"
+[httpcache.polls.for]
+includes = ["**gohugo.io**"]
+
+
+`
+
+ b := hugolib.Test(t, files)
+
+ httpcacheConf := b.H.Configs.Base.HTTPCache
+ compiled := b.H.Configs.Base.C.HTTPCache
+
+ b.Assert(httpcacheConf.Cache.For.Includes, qt.DeepEquals, []string{"**gohugo.io**"})
+ b.Assert(httpcacheConf.Cache.For.Excludes, qt.IsNil)
+
+ pc := compiled.PollConfigFor("https://gohugo.io/foo.jpg")
+ b.Assert(pc.Config.Low, qt.Equals, 5*time.Second)
+ b.Assert(pc.Config.High, qt.Equals, 32*time.Second)
+ b.Assert(compiled.PollConfigFor("https://example.com/foo.jpg").IsZero(), qt.IsTrue)
+}
+
+func TestConfigDefault(t *testing.T) {
+ files := `
+-- hugo.toml --
+`
+ b := hugolib.Test(t, files)
+
+ compiled := b.H.Configs.Base.C.HTTPCache
+
+ b.Assert(compiled.For("https://gohugo.io/posts.json"), qt.IsFalse)
+ b.Assert(compiled.For("https://gohugo.io/foo.jpg"), qt.IsFalse)
+ b.Assert(compiled.PollConfigFor("https://gohugo.io/foo.jpg").Config.Disable, qt.IsTrue)
+}
--- /dev/null
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpcache
+
+import (
+ "testing"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func TestGlobMatcher(t *testing.T) {
+ c := qt.New(t)
+
+ g := GlobMatcher{
+ Includes: []string{"**/*.jpg", "**.png", "**/bar/**"},
+ Excludes: []string{"**/foo.jpg", "**.css"},
+ }
+
+ p, err := g.CompilePredicate()
+ c.Assert(err, qt.IsNil)
+
+ c.Assert(p("foo.jpg"), qt.IsFalse)
+ c.Assert(p("foo.png"), qt.IsTrue)
+ c.Assert(p("foo/bar.jpg"), qt.IsTrue)
+ c.Assert(p("foo/bar.png"), qt.IsTrue)
+ c.Assert(p("foo/bar/foo.jpg"), qt.IsFalse)
+ c.Assert(p("foo/bar/foo.css"), qt.IsFalse)
+ c.Assert(p("foo.css"), qt.IsFalse)
+ c.Assert(p("foo/bar/foo.css"), qt.IsFalse)
+ c.Assert(p("foo/bar/foo.xml"), qt.IsTrue)
+}
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/hugolib"
+ "github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/resources/kinds"
"github.com/spf13/afero"
"github.com/spf13/cobra"
commonConfigs *lazycache.Cache[int32, *commonConfig]
hugoSites *lazycache.Cache[int32, *hugolib.HugoSites]
+ // changesFromBuild received from Hugo in watch mode.
+ changesFromBuild chan []identity.Identity
+
commands []simplecobra.Commander
// Flags
func (r *rootCommand) HugFromConfig(conf *commonConfig) (*hugolib.HugoSites, error) {
h, _, err := r.hugoSites.GetOrCreate(r.configVersionID.Load(), func(key int32) (*hugolib.HugoSites, error) {
- depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level()}
+ depsCfg := r.newDepsConfig(conf)
return hugolib.NewHugoSites(depsCfg)
})
return h, err
if err != nil {
return nil, err
}
- depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level()}
+ depsCfg := r.newDepsConfig(conf)
return hugolib.NewHugoSites(depsCfg)
})
return h, err
}
+func (r *rootCommand) newDepsConfig(conf *commonConfig) deps.DepsCfg {
+ return deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level(), ChangesFromBuild: r.changesFromBuild}
+}
+
func (r *rootCommand) Name() string {
return "hugo"
}
return err
}
+ r.changesFromBuild = make(chan []identity.Identity, 10)
+
r.commonConfigs = lazycache.New(lazycache.Options[int32, *commonConfig]{MaxEntries: 5})
// We don't want to keep stale HugoSites in memory longer than needed.
r.hugoSites = lazycache.New(lazycache.Options[int32, *hugolib.HugoSites]{
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/hugolib"
"github.com/gohugoio/hugo/hugolib/filesystems"
+ "github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/livereload"
"github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/watcher"
go func() {
for {
select {
+ case changes := <-c.r.changesFromBuild:
+ unlock, err := h.LockBuild()
+ if err != nil {
+ c.r.logger.Errorln("Failed to acquire a build lock: %s", err)
+ return
+ }
+ c.changeDetector.PrepareNew()
+ err = c.rebuildSitesForChanges(changes)
+ if err != nil {
+ c.r.logger.Errorln("Error while watching:", err)
+ }
+ if c.s != nil && c.s.doLiveReload {
+ if c.changeDetector == nil || len(c.changeDetector.changed()) > 0 {
+ livereload.ForceRefresh()
+ }
+ }
+ unlock()
+
case evs := <-watcher.Events:
unlock, err := h.LockBuild()
if err != nil {
return h.Build(hugolib.BuildCfg{NoBuildLock: true, RecentlyVisited: c.visitedURLs, ErrRecovery: c.errState.wasErr()}, events...)
}
+func (c *hugoBuilder) rebuildSitesForChanges(ids []identity.Identity) error {
+ c.errState.setBuildErr(nil)
+ h, err := c.hugo()
+ if err != nil {
+ return err
+ }
+ whatChanged := &hugolib.WhatChanged{}
+ whatChanged.Add(ids...)
+ err = h.Build(hugolib.BuildCfg{NoBuildLock: true, WhatChanged: whatChanged, RecentlyVisited: c.visitedURLs, ErrRecovery: c.errState.wasErr()})
+ c.errState.setBuildErr(err)
+ return err
+}
+
func (c *hugoBuilder) reloadConfig() error {
c.r.Reset()
c.r.configVersionID.Add(1)
}
// LookupEqualFold finds key in m with case insensitive equality checks.
-func LookupEqualFold[T any | string](m map[string]T, key string) (T, bool) {
+func LookupEqualFold[T any | string](m map[string]T, key string) (T, string, bool) {
if v, found := m[key]; found {
- return v, true
+ return v, key, true
}
for k, v := range m {
if strings.EqualFold(k, key) {
- return v, true
+ return v, k, true
}
}
var s T
- return s, false
+ return s, "", false
}
// MergeShallow merges src into dst, but only if the key does not already exist in dst.
"B": "bv",
}
- v, found := LookupEqualFold(m1, "b")
+ v, k, found := LookupEqualFold(m1, "b")
c.Assert(found, qt.IsTrue)
c.Assert(v, qt.Equals, "bv")
+ c.Assert(k, qt.Equals, "B")
m2 := map[string]string{
"a": "av",
"B": "bv",
}
- v, found = LookupEqualFold(m2, "b")
+ v, k, found = LookupEqualFold(m2, "b")
c.Assert(found, qt.IsTrue)
+ c.Assert(k, qt.Equals, "B")
c.Assert(v, qt.Equals, "bv")
}
return false
}
}
+ if p == nil {
+ return true
+ }
return p(v)
}
}
return true
}
}
+ if p == nil {
+ return false
+ }
return p(v)
}
}
--- /dev/null
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tasks
+
+import (
+ "sync"
+ "time"
+)
+
+// RunEvery runs a function at intervals defined by the function itself.
+// Functions can be added and removed while running.
+type RunEvery struct {
+ // Any error returned from the function will be passed to this function.
+ HandleError func(string, error)
+
+ // If set, the function will be run immediately.
+ RunImmediately bool
+
+ // The named functions to run.
+ funcs map[string]*Func
+
+ mu sync.Mutex
+ started bool
+ closed bool
+ quit chan struct{}
+}
+
+type Func struct {
+ // The shortest interval between each run.
+ IntervalLow time.Duration
+
+ // The longest interval between each run.
+ IntervalHigh time.Duration
+
+ // The function to run.
+ F func(interval time.Duration) (time.Duration, error)
+
+ interval time.Duration
+ last time.Time
+}
+
+func (r *RunEvery) Start() error {
+ if r.started {
+ return nil
+ }
+
+ r.started = true
+ r.quit = make(chan struct{})
+
+ go func() {
+ if r.RunImmediately {
+ r.run()
+ }
+ ticker := time.NewTicker(500 * time.Millisecond)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-r.quit:
+ return
+ case <-ticker.C:
+ r.run()
+ }
+ }
+ }()
+
+ return nil
+}
+
+// Close stops the RunEvery from running.
+func (r *RunEvery) Close() error {
+ if r.closed {
+ return nil
+ }
+ r.closed = true
+ if r.quit != nil {
+ close(r.quit)
+ }
+ return nil
+}
+
+// Add adds a function to the RunEvery.
+func (r *RunEvery) Add(name string, f Func) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ if r.funcs == nil {
+ r.funcs = make(map[string]*Func)
+ }
+ if f.IntervalLow == 0 {
+ f.IntervalLow = 500 * time.Millisecond
+ }
+ if f.IntervalHigh <= f.IntervalLow {
+ f.IntervalHigh = 20 * time.Second
+ }
+
+ start := f.IntervalHigh / 3
+ if start < f.IntervalLow {
+ start = f.IntervalLow
+ }
+ f.interval = start
+ f.last = time.Now()
+
+ r.funcs[name] = &f
+}
+
+// Remove removes a function from the RunEvery.
+func (r *RunEvery) Remove(name string) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ delete(r.funcs, name)
+}
+
+// Has returns whether the RunEvery has a function with the given name.
+func (r *RunEvery) Has(name string) bool {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ _, found := r.funcs[name]
+ return found
+}
+
+func (r *RunEvery) run() {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ for name, f := range r.funcs {
+ if time.Now().Before(f.last.Add(f.interval)) {
+ continue
+ }
+ f.last = time.Now()
+ interval, err := f.F(f.interval)
+ if err != nil && r.HandleError != nil {
+ r.HandleError(name, err)
+ }
+
+ if interval < f.IntervalLow {
+ interval = f.IntervalLow
+ }
+
+ if interval > f.IntervalHigh {
+ interval = f.IntervalHigh
+ }
+ f.interval = interval
+ }
+}
--- /dev/null
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+import "sync"
+
+type Closer interface {
+ Close() error
+}
+
+type CloseAdder interface {
+ Add(Closer)
+}
+
+type Closers struct {
+ mu sync.Mutex
+ cs []Closer
+}
+
+func (cs *Closers) Add(c Closer) {
+ cs.mu.Lock()
+ defer cs.mu.Unlock()
+ cs.cs = append(cs.cs, c)
+}
+
+func (cs *Closers) Close() error {
+ cs.mu.Lock()
+ defer cs.mu.Unlock()
+ for _, c := range cs.cs {
+ c.Close()
+ }
+
+ cs.cs = cs.cs[:0]
+
+ return nil
+}
"time"
"github.com/gohugoio/hugo/cache/filecache"
+ "github.com/gohugoio/hugo/cache/httpcache"
"github.com/gohugoio/hugo/common/hugo"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/maps"
// <docsmeta>{"identifiers": ["caches"] }</docsmeta>
Caches filecache.Configs `mapstructure:"-"`
+ // The httpcache configuration section contains HTTP-cache-related configuration options.
+ // <docsmeta>{"identifiers": ["httpcache"] }</docsmeta>
+ HTTPCache httpcache.Config `mapstructure:"-"`
+
// The markup configuration section contains markup-related configuration options.
// <docsmeta>{"identifiers": ["markup"] }</docsmeta>
Markup markup_config.Config `mapstructure:"-"`
}
}
+ httpCache, err := c.HTTPCache.Compile()
+ if err != nil {
+ return err
+ }
+
c.C = &ConfigCompiled{
Timeout: timeout,
BaseURL: baseURL,
SegmentFilter: c.Segments.Config.Get(func(s string) { logger.Warnf("Render segment %q not found in configuration", s) }, c.RootConfig.RenderSegments...),
MainSections: c.MainSections,
Clock: clock,
+ HTTPCache: httpCache,
transientErr: transientErr,
}
SegmentFilter segments.SegmentFilter
MainSections []string
Clock time.Time
+ HTTPCache httpcache.ConfigCompiled
// This is set to the last transient error found during config compilation.
// With themes/modules we compute the configuration in multiple passes, and
"strings"
"github.com/gohugoio/hugo/cache/filecache"
+
+ "github.com/gohugoio/hugo/cache/httpcache"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/config"
return err
},
},
+ "httpcache": {
+ key: "httpcache",
+ decode: func(d decodeWeight, p decodeConfig) error {
+ var err error
+ p.c.HTTPCache, err = httpcache.DecodeConfig(p.bcfg, p.p.GetStringMap(d.key))
+ if p.c.IgnoreCache {
+ p.c.HTTPCache.Cache.For.Excludes = []string{"**"}
+ p.c.HTTPCache.Cache.For.Includes = []string{}
+ }
+ return err
+ },
+ },
"build": {
key: "build",
decode: func(d decodeWeight, p decodeConfig) error {
return c.m.Modules
case "deployment":
return c.config.Deployment
+ case "httpCacheCompiled":
+ return c.config.C.HTTPCache
default:
panic("not implemented: " + s)
}
"github.com/gohugoio/hugo/cache/filecache"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/loggers"
+ "github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/config/allconfig"
"github.com/gohugoio/hugo/config/security"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
+ "github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/resources/postpub"
BuildEndListeners *Listeners
// Resources that gets closed when the build is done or the server shuts down.
- BuildClosers *Closers
+ BuildClosers *types.Closers
// This is common/global for all sites.
BuildState *BuildState
}
if d.BuildClosers == nil {
- d.BuildClosers = &Closers{}
+ d.BuildClosers = &types.Closers{}
}
if d.Metrics == nil && d.Conf.TemplateMetrics() {
return fmt.Errorf("failed to create file caches from configuration: %w", err)
}
- resourceSpec, err := resources.NewSpec(d.PathSpec, common, fileCaches, d.MemCache, d.BuildState, d.Log, d, d.ExecHelper)
+ resourceSpec, err := resources.NewSpec(d.PathSpec, common, fileCaches, d.MemCache, d.BuildState, d.Log, d, d.ExecHelper, d.BuildClosers, d.BuildState)
if err != nil {
return fmt.Errorf("failed to create resource spec: %w", err)
}
// i18n handling.
TranslationProvider ResourceProvider
+
+ // ChangesFromBuild for changes passed back to the server/watch process.
+ ChangesFromBuild chan []identity.Identity
}
// BuildState are state used during a build.
mu sync.Mutex // protects state below.
+ OnSignalRebuild func(ids ...identity.Identity)
+
// A set of filenames in /public that
// contains a post-processing prefix.
filenamesWithPostPrefix map[string]bool
}
+var _ identity.SignalRebuilder = (*BuildState)(nil)
+
+func (b *BuildState) SignalRebuild(ids ...identity.Identity) {
+ b.OnSignalRebuild(ids...)
+}
+
func (b *BuildState) AddFilenameWithPostPrefix(filename string) {
b.mu.Lock()
defer b.mu.Unlock()
func (b *BuildState) Incr() int {
return int(atomic.AddUint64(&b.counter, uint64(1)))
}
-
-type Closer interface {
- Close() error
-}
-
-type Closers struct {
- mu sync.Mutex
- cs []Closer
-}
-
-func (cs *Closers) Add(c Closer) {
- cs.mu.Lock()
- defer cs.mu.Unlock()
- cs.cs = append(cs.cs, c)
-}
-
-func (cs *Closers) Close() error {
- cs.mu.Lock()
- defer cs.mu.Unlock()
- for _, c := range cs.cs {
- c.Close()
- }
-
- cs.cs = cs.cs[:0]
-
- return nil
-}
github.com/gobuffalo/flect v1.0.2
github.com/gobwas/glob v0.2.3
github.com/gohugoio/go-i18n/v2 v2.1.3-0.20230805085216-e63c13218d0e
+ github.com/gohugoio/httpcache v0.6.0
github.com/gohugoio/hugo-goldmark-extensions/extras v0.1.0
github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.2.0
github.com/gohugoio/locales v0.14.0
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 h1:d81/ng9rET2YqdVkVwkb6EXeRrLJIwyGnJcAlAWKwhs=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4=
github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU=
+github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
github.com/alecthomas/chroma/v2 v2.13.0 h1:VP72+99Fb2zEcYM0MeaWJmV+xQvz5v5cxRHd+ooU1lI=
github.com/alecthomas/chroma/v2 v2.13.0/go.mod h1:BUGjjsD+ndS6eX37YgTchSEG+Jg9Jv1GiZs9sqPqztk=
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
+github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c h1:651/eoCRnQ7YtSjAnSzRucrJz+3iGEFt+ysraELS81M=
github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aws/aws-sdk-go v1.50.7 h1:odKb+uneeGgF2jgAerKjFzpljiyZxleV4SHB7oBK+YA=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
+github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/getkin/kin-openapi v0.123.0 h1:zIik0mRwFNLyvtXK274Q6ut+dPh6nlxBp0x7mNrPhs8=
github.com/getkin/kin-openapi v0.123.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM=
-github.com/getkin/kin-openapi v0.124.0 h1:VSFNMB9C9rTKBnQ/fpyDU8ytMTr4dWI9QovSKj9kz/M=
-github.com/getkin/kin-openapi v0.124.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-openapi/swag v0.22.8 h1:/9RjDSQ0vbFR+NyjGMkFTsA1IA0fmhKSThmfGZjicbw=
github.com/go-openapi/swag v0.22.8/go.mod h1:6QT22icPLEqAM/z/TChgb4WAveCHF92+2gF0CNjHpPI=
github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
+github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA=
github.com/gobuffalo/flect v1.0.2/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gohugoio/go-i18n/v2 v2.1.3-0.20230805085216-e63c13218d0e h1:QArsSubW7eDh8APMXkByjQWvuljwPGAGQpJEFn0F0wY=
github.com/gohugoio/go-i18n/v2 v2.1.3-0.20230805085216-e63c13218d0e/go.mod h1:3Ltoo9Banwq0gOtcOwxuHG6omk+AwsQPADyw2vQYOJQ=
+github.com/gohugoio/httpcache v0.5.0 h1:9xi4VuXd+KT3h0jOs8DlZxTMu5CtjDr0BvQMAuL/O5I=
+github.com/gohugoio/httpcache v0.5.0/go.mod h1:fMlPrdY/vVJhAriLZnrF5QpN3BNAcoBClgAyQd+lGFI=
+github.com/gohugoio/httpcache v0.6.0 h1:5pYJM43Yoc4uvIJ+/e770PS48srTumvuQZpuBfGFZV0=
+github.com/gohugoio/httpcache v0.6.0/go.mod h1:fMlPrdY/vVJhAriLZnrF5QpN3BNAcoBClgAyQd+lGFI=
+github.com/gohugoio/httpcache v0.7.0 h1:ukPnn04Rgvx48JIinZvZetBfHaWE7I01JR2Q2RrQ3Vs=
+github.com/gohugoio/httpcache v0.7.0/go.mod h1:fMlPrdY/vVJhAriLZnrF5QpN3BNAcoBClgAyQd+lGFI=
github.com/gohugoio/hugo-goldmark-extensions/extras v0.1.0 h1:YhxZNU8y2vxV6Ibr7QJzzUlpr8oHHWX/l+Q1R/a5Zao=
github.com/gohugoio/hugo-goldmark-extensions/extras v0.1.0/go.mod h1:0cuvOnGKW7WeXA3i7qK6IS07FH1bgJ2XzOjQ7BMJYH4=
github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.2.0 h1:PCtO5l++psZf48yen2LxQ3JiOXxaRC6v0594NeHvGZg=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-replayers/grpcreplay v1.1.0 h1:S5+I3zYyZ+GQz68OfbURDdt/+cSMqCK1wrvNx7WBzTE=
+github.com/google/go-replayers/grpcreplay v1.1.0/go.mod h1:qzAvJ8/wi57zq7gWqaE6AwLM6miiXUQwP1S+I9icmhk=
github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk=
+github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
+github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
+github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/tdewolff/parse/v2 v2.7.13/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W1aghka0soA=
github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03u/dMQK9g+Iw9ewps4mCl1nB8Sscbo=
+github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8=
github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
+github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/neurosnap/sentences.v1 v1.0.6/go.mod h1:YlK+SN+fLQZj+kY3r8DkGDhDr91+S3JmTb5LSxFRQo0=
gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
type sitePagesAssembler struct {
*Site
- assembleChanges *whatChanged
+ assembleChanges *WhatChanged
ctx context.Context
}
type BuildCfg struct {
// Skip rendering. Useful for testing.
SkipRender bool
+
// Use this to indicate what changed (for rebuilds).
- whatChanged *whatChanged
+ WhatChanged *WhatChanged
// This is a partial re-render of some selected pages.
PartialReRender bool
// Need a pointer as this may be modified.
conf := &config
- if conf.whatChanged == nil {
+ if conf.WhatChanged == nil {
// Assume everything has changed
- conf.whatChanged = &whatChanged{needsPagesAssembly: true}
+ conf.WhatChanged = &WhatChanged{needsPagesAssembly: true}
}
var prepareErr error
s.Deps.BuildStartListeners.Notify()
}
- if len(events) > 0 {
+ if len(events) > 0 || len(conf.WhatChanged.Changes()) > 0 {
// Rebuild
if err := h.initRebuild(conf); err != nil {
return fmt.Errorf("initRebuild: %w", err)
})
for _, s := range h.Sites {
- s.resetBuildState(config.whatChanged.needsPagesAssembly)
+ s.resetBuildState(config.WhatChanged.needsPagesAssembly)
}
h.reset(config)
if len(events) > 0 {
// This is a rebuild
- return h.processPartial(ctx, l, config, init, events)
+ return h.processPartialFileEvents(ctx, l, config, init, events)
+ } else if len(config.WhatChanged.Changes()) > 0 {
+ return h.processPartialRebuildChanges(ctx, l, config)
}
return h.processFull(ctx, l, config)
}
l = l.WithField("step", "assemble")
defer loggers.TimeTrackf(l, time.Now(), nil, "")
- if !bcfg.whatChanged.needsPagesAssembly {
- changes := bcfg.whatChanged.Drain()
+ if !bcfg.WhatChanged.needsPagesAssembly {
+ changes := bcfg.WhatChanged.Drain()
if len(changes) > 0 {
if err := h.resolveAndClearStateForIdentities(ctx, l, nil, changes); err != nil {
return err
for i, s := range h.Sites {
assemblers[i] = &sitePagesAssembler{
Site: s,
- assembleChanges: bcfg.whatChanged,
+ assembleChanges: bcfg.WhatChanged,
ctx: ctx,
}
}
return err
}
- changes := bcfg.whatChanged.Drain()
+ changes := bcfg.WhatChanged.Drain()
// Changes from the assemble step (e.g. lastMod, cascade) needs a re-calculation
// of what needs to be re-built.
return p.delete || p.isDir
}
-// processPartial prepares the Sites' sources for a partial rebuild.
-func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, config *BuildCfg, init func(config *BuildCfg) error, events []fsnotify.Event) error {
+func (h *HugoSites) processPartialRebuildChanges(ctx context.Context, l logg.LevelLogger, config *BuildCfg) error {
+ if err := h.resolveAndClearStateForIdentities(ctx, l, nil, config.WhatChanged.Drain()); err != nil {
+ return err
+ }
+
+ if err := h.processContentAdaptersOnRebuild(ctx, config); err != nil {
+ return err
+ }
+ return nil
+}
+
+// processPartialFileEvents prepares the Sites' sources for a partial rebuild.
+func (h *HugoSites) processPartialFileEvents(ctx context.Context, l logg.LevelLogger, config *BuildCfg, init func(config *BuildCfg) error, events []fsnotify.Event) error {
h.Log.Trace(logg.StringFunc(func() string {
var sb strings.Builder
sb.WriteString("File events:\n")
resourceFiles := h.fileEventsContentPaths(addedOrChangedContent)
- changed := &whatChanged{
+ changed := &WhatChanged{
needsPagesAssembly: needsPagesAssemble,
identitySet: make(identity.Identities),
}
changed.Add(changes...)
- config.whatChanged = changed
+ config.WhatChanged = changed
if err := init(config); err != nil {
return err
}
if len(bi.ChangedIdentities) > 0 {
- buildConfig.whatChanged.Add(bi.ChangedIdentities...)
- buildConfig.whatChanged.needsPagesAssembly = true
+ buildConfig.WhatChanged.Add(bi.ChangedIdentities...)
+ buildConfig.WhatChanged.needsPagesAssembly = true
}
for _, p := range bi.DeletedPaths {
pp := path.Join(bi.Path.Base(), p)
if v, ok := s.pageMap.treePages.Delete(pp); ok {
- buildConfig.whatChanged.Add(v.GetIdentity())
+ buildConfig.WhatChanged.Add(v.GetIdentity())
}
}
}
return s.h != nil && s.h.Configs.Base.Internal.Watch
}
-type whatChanged struct {
+type WhatChanged struct {
mu sync.Mutex
needsPagesAssembly bool
identitySet identity.Identities
}
-func (w *whatChanged) Add(ids ...identity.Identity) {
+func (w *WhatChanged) Add(ids ...identity.Identity) {
w.mu.Lock()
defer w.mu.Unlock()
}
}
-func (w *whatChanged) Clear() {
+func (w *WhatChanged) Clear() {
w.mu.Lock()
defer w.mu.Unlock()
w.clear()
}
-func (w *whatChanged) clear() {
+func (w *WhatChanged) clear() {
w.identitySet = identity.Identities{}
}
-func (w *whatChanged) Changes() []identity.Identity {
+func (w *WhatChanged) Changes() []identity.Identity {
if w == nil || w.identitySet == nil {
return nil
}
return w.identitySet.AsSlice()
}
-func (w *whatChanged) Drain() []identity.Identity {
+func (w *WhatChanged) Drain() []identity.Identity {
w.mu.Lock()
defer w.mu.Unlock()
ids := w.identitySet.AsSlice()
memCache := dynacache.New(dynacache.Options{Watching: conf.Watching(), Log: logger})
+ var h *HugoSites
+ onSignalRebuild := func(ids ...identity.Identity) {
+ // This channel is buffered, but make sure we do this in a non-blocking way.
+ if cfg.ChangesFromBuild != nil {
+ go func() {
+ cfg.ChangesFromBuild <- ids
+ }()
+ }
+ }
+
firstSiteDeps := &deps.Deps{
- Fs: cfg.Fs,
- Log: logger,
- Conf: conf,
+ Fs: cfg.Fs,
+ Log: logger,
+ Conf: conf,
+ BuildState: &deps.BuildState{
+ OnSignalRebuild: onSignalRebuild,
+ },
MemCache: memCache,
TemplateProvider: tplimpl.DefaultTemplateProvider,
TranslationProvider: i18n.NewTranslationProvider(),
return li.Lang < lj.Lang
})
- h, err := newHugoSites(cfg, firstSiteDeps, pageTrees, sites)
+ var err error
+ h, err = newHugoSites(cfg, firstSiteDeps, pageTrees, sites)
if err == nil && h == nil {
panic("hugo: newHugoSitesNew returned nil error and nil HugoSites")
}
GetIdentity() Identity
}
+// SignalRebuilder is an optional interface for types that can signal a rebuild.
+type SignalRebuilder interface {
+ SignalRebuild(ids ...Identity)
+}
+
// IncrementByOne implements Incrementer adding 1 every time Incr is called.
type IncrementByOne struct {
counter uint64
return nil, nil, err
}
mm := maps.ToStringMap(v)
- suffixes, found := maps.LookupEqualFold(mm, "suffixes")
+ suffixes, _, found := maps.LookupEqualFold(mm, "suffixes")
if found {
mediaType.SuffixesCSV = strings.TrimSpace(strings.ToLower(strings.Join(cast.ToStringSlice(suffixes), ",")))
}
Value any
}
+var preserveUpperCaseKeyRe = regexp.MustCompile(`^"HTTP`)
+
+func preserveUpperCaseKey(match []byte) bool {
+ return preserveUpperCaseKeyRe.Match(match)
+}
+
func (c LowerCaseCamelJSONMarshaller) MarshalJSON() ([]byte, error) {
marshalled, err := json.Marshal(c.Value)
// Empty keys are valid JSON, only lowercase if we do not have an
// empty key.
- if len(match) > 2 {
+ if len(match) > 2 && !preserveUpperCaseKey(match) {
// Decode first rune after the double quotes
r, width := utf8.DecodeRune(match[1:])
r = unicode.ToLower(r)
"/res1",
dynacache.OptionsPartition{ClearWhen: dynacache.ClearOnChange, Weight: 40},
),
+ CacheResourceRemote: dynacache.GetOrCreatePartition[string, resource.Resource](
+ memCache,
+ "/resr",
+ dynacache.OptionsPartition{ClearWhen: dynacache.ClearOnChange, Weight: 40},
+ ),
cacheResources: dynacache.GetOrCreatePartition[string, resource.Resources](
memCache,
"/ress",
sync.RWMutex
cacheResource *dynacache.Partition[string, resource.Resource]
+ CacheResourceRemote *dynacache.Partition[string, resource.Resource]
cacheResources *dynacache.Partition[string, resource.Resources]
cacheResourceTransformation *dynacache.Partition[string, *resourceAdapterInner]
"strings"
"time"
+ "github.com/bep/logg"
+ "github.com/gohugoio/httpcache"
+ hhttpcache "github.com/gohugoio/hugo/cache/httpcache"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs/glob"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/cache/dynacache"
"github.com/gohugoio/hugo/cache/filecache"
+ "github.com/gohugoio/hugo/common/hcontext"
"github.com/gohugoio/hugo/common/hugio"
+ "github.com/gohugoio/hugo/common/tasks"
"github.com/gohugoio/hugo/resources"
"github.com/gohugoio/hugo/resources/resource"
)
// Client contains methods to create Resource objects.
// tasks to Resource objects.
type Client struct {
- rs *resources.Spec
- httpClient *http.Client
- cacheGetResource *filecache.Cache
+ rs *resources.Spec
+ httpClient *http.Client
+ httpCacheConfig hhttpcache.ConfigCompiled
+ cacheGetResource *filecache.Cache
+ resourceIDDispatcher hcontext.ContextDispatcher[string]
+
+ // Set when watching.
+ remoteResourceChecker *tasks.RunEvery
+ remoteResourceLogger logg.LevelLogger
}
+type contextKey string
+
// New creates a new Client with the given specification.
func New(rs *resources.Spec) *Client {
+ fileCache := rs.FileCaches.GetResourceCache()
+ resourceIDDispatcher := hcontext.NewContextDispatcher[string](contextKey("resourceID"))
+ httpCacheConfig := rs.Cfg.GetConfigSection("httpCacheCompiled").(hhttpcache.ConfigCompiled)
+ var remoteResourceChecker *tasks.RunEvery
+ if rs.Cfg.Watching() && !httpCacheConfig.IsPollingDisabled() {
+ remoteResourceChecker = &tasks.RunEvery{
+ HandleError: func(name string, err error) {
+ rs.Logger.Warnf("Failed to check remote resource: %s", err)
+ },
+ RunImmediately: false,
+ }
+
+ if err := remoteResourceChecker.Start(); err != nil {
+ panic(err)
+ }
+
+ rs.BuildClosers.Add(remoteResourceChecker)
+ }
+
+ httpTimeout := 2 * time.Minute // Need to cover retries.
+ if httpTimeout < (rs.Cfg.Timeout() + 30*time.Second) {
+ httpTimeout = rs.Cfg.Timeout() + 30*time.Second
+ }
+
return &Client{
- rs: rs,
+ rs: rs,
+ httpCacheConfig: httpCacheConfig,
+ resourceIDDispatcher: resourceIDDispatcher,
+ remoteResourceChecker: remoteResourceChecker,
+ remoteResourceLogger: rs.Logger.InfoCommand("remote"),
httpClient: &http.Client{
- Timeout: time.Minute,
+ Timeout: httpTimeout,
+ Transport: &httpcache.Transport{
+ Cache: fileCache.AsHTTPCache(),
+ CacheKey: func(req *http.Request) string {
+ return resourceIDDispatcher.Get(req.Context())
+ },
+ Around: func(req *http.Request, key string) func() {
+ return fileCache.NamedLock(key)
+ },
+ AlwaysUseCachedResponse: func(req *http.Request, key string) bool {
+ return !httpCacheConfig.For(req.URL.String())
+ },
+ ShouldCache: func(req *http.Request, resp *http.Response, key string) bool {
+ return shouldCache(resp.StatusCode)
+ },
+ MarkCachedResponses: true,
+ EnableETagPair: true,
+ Transport: &transport{
+ Cfg: rs.Cfg,
+ Logger: rs.Logger,
+ },
+ },
},
- cacheGetResource: rs.FileCaches.GetResourceCache(),
+ cacheGetResource: fileCache,
}
}
// This is hard to get stable on GitHub Actions, it sometimes succeeds due to timing issues.
if err != nil {
b.AssertLogContains("Got Err")
- b.AssertLogContains("Retry timeout")
- b.AssertLogContains("ContentLength:0")
+ b.AssertLogContains("retry timeout")
}
})
}
package create
import (
- "bufio"
"bytes"
+ "context"
"fmt"
"io"
"math/rand"
"mime"
"net/http"
- "net/http/httputil"
"net/url"
"path"
"strings"
"time"
+ gmaps "maps"
+
+ "github.com/gohugoio/httpcache"
"github.com/gohugoio/hugo/common/hugio"
+ "github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/maps"
+ "github.com/gohugoio/hugo/common/tasks"
"github.com/gohugoio/hugo/common/types"
+ "github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/resources"
504: true,
}
+func (c *Client) configurePollingIfEnabled(uri, optionsKey string, getRes func() (*http.Response, error)) {
+ if c.remoteResourceChecker == nil {
+ return
+ }
+
+ // Set up polling for changes to this resource.
+ pollingConfig := c.httpCacheConfig.PollConfigFor(uri)
+ if pollingConfig.IsZero() || pollingConfig.Config.Disable {
+ return
+ }
+
+ if c.remoteResourceChecker.Has(optionsKey) {
+ return
+ }
+
+ var lastChange time.Time
+ c.remoteResourceChecker.Add(optionsKey,
+ tasks.Func{
+ IntervalLow: pollingConfig.Config.Low,
+ IntervalHigh: pollingConfig.Config.High,
+ F: func(interval time.Duration) (time.Duration, error) {
+ start := time.Now()
+ defer func() {
+ duration := time.Since(start)
+ c.rs.Logger.Debugf("Polled remote resource for changes in %13s. Interval: %4s (low: %4s high: %4s) resource: %q ", duration, interval, pollingConfig.Config.Low, pollingConfig.Config.High, uri)
+ }()
+ // TODO(bep) figure out a ways to remove unused tasks.
+ res, err := getRes()
+ if err != nil {
+ return pollingConfig.Config.High, err
+ }
+ // The caching is delayed until the body is read.
+ io.Copy(io.Discard, res.Body)
+ res.Body.Close()
+ x1, x2 := res.Header.Get(httpcache.XETag1), res.Header.Get(httpcache.XETag2)
+ if x1 != x2 {
+ lastChange = time.Now()
+ c.remoteResourceLogger.Logf("detected change in remote resource %q", uri)
+ c.rs.Rebuilder.SignalRebuild(identity.StringIdentity(optionsKey))
+ }
+
+ if time.Since(lastChange) < 10*time.Second {
+ // The user is typing, check more often.
+ return 0, nil
+ }
+
+ // Increase the interval to avoid hammering the server.
+ interval += 1 * time.Second
+
+ return interval, nil
+ },
+ })
+}
+
// FromRemote expects one or n-parts of a URL to a resource
// If you provide multiple parts they will be joined together to the final URL.
func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resource, error) {
}
method := "GET"
- if s, ok := maps.LookupEqualFold(optionsm, "method"); ok {
+ if s, _, ok := maps.LookupEqualFold(optionsm, "method"); ok {
method = strings.ToUpper(s.(string))
}
isHeadMethod := method == "HEAD"
- resourceID := calculateResourceID(uri, optionsm)
+ optionsm = gmaps.Clone(optionsm)
+ userKey, optionsKey := remoteResourceKeys(uri, optionsm)
+
+ // A common pattern is to use the key in the options map as
+ // a way to control cache eviction,
+ // so make sure we use any user provided kehy as the file cache key,
+ // but the auto generated and more stable key for everything else.
+ filecacheKey := userKey
- _, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) {
+ return c.rs.ResourceCache.CacheResourceRemote.GetOrCreate(optionsKey, func(key string) (resource.Resource, error) {
options, err := decodeRemoteOptions(optionsm)
if err != nil {
return nil, fmt.Errorf("failed to decode options for resource %s: %w", uri, err)
}
+
if err := c.validateFromRemoteArgs(uri, options); err != nil {
return nil, err
}
- var (
- start time.Time
- nextSleep = time.Duration((rand.Intn(1000) + 100)) * time.Millisecond
- nextSleepLimit = time.Duration(5) * time.Second
- )
+ getRes := func() (*http.Response, error) {
+ ctx := context.Background()
+ ctx = c.resourceIDDispatcher.Set(ctx, filecacheKey)
- for {
- b, retry, err := func() ([]byte, bool, error) {
- req, err := options.NewRequest(uri)
- if err != nil {
- return nil, false, fmt.Errorf("failed to create request for resource %s: %w", uri, err)
- }
-
- res, err := c.httpClient.Do(req)
- if err != nil {
- return nil, false, err
- }
- defer res.Body.Close()
-
- if res.StatusCode != http.StatusNotFound {
- if res.StatusCode < 200 || res.StatusCode > 299 {
- return nil, temporaryHTTPStatusCodes[res.StatusCode], toHTTPError(fmt.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode)), res, !isHeadMethod)
- }
- }
-
- b, err := httputil.DumpResponse(res, true)
- if err != nil {
- return nil, false, toHTTPError(err, res, !isHeadMethod)
- }
-
- return b, false, nil
- }()
+ req, err := options.NewRequest(uri)
if err != nil {
- if retry {
- if start.IsZero() {
- start = time.Now()
- } else if d := time.Since(start) + nextSleep; d >= c.rs.Cfg.Timeout() {
- c.rs.Logger.Errorf("Retry timeout (configured to %s) fetching remote resource.", c.rs.Cfg.Timeout())
- return nil, err
- }
- time.Sleep(nextSleep)
- if nextSleep < nextSleepLimit {
- nextSleep *= 2
- }
- continue
- }
- return nil, err
+ return nil, fmt.Errorf("failed to create request for resource %s: %w", uri, err)
}
- return hugio.ToReadCloser(bytes.NewReader(b)), nil
+ req = req.WithContext(ctx)
+ return c.httpClient.Do(req)
}
- })
- if err != nil {
- return nil, err
- }
- defer httpResponse.Close()
-
- res, err := http.ReadResponse(bufio.NewReader(httpResponse), nil)
- if err != nil {
- return nil, err
- }
- defer res.Body.Close()
- if res.StatusCode == http.StatusNotFound {
- // Not found. This matches how looksup for local resources work.
- return nil, nil
- }
-
- var (
- body []byte
- mediaType media.Type
- )
- // A response to a HEAD method should not have a body. If it has one anyway, that body must be ignored.
- // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD
- if !isHeadMethod && res.Body != nil {
- body, err = io.ReadAll(res.Body)
+ res, err := getRes()
if err != nil {
- return nil, fmt.Errorf("failed to read remote resource %q: %w", uri, err)
+ return nil, err
}
- }
+ defer res.Body.Close()
- filename := path.Base(rURL.Path)
- if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil {
- if _, ok := params["filename"]; ok {
- filename = params["filename"]
+ c.configurePollingIfEnabled(uri, optionsKey, getRes)
+
+ if res.StatusCode == http.StatusNotFound {
+ // Not found. This matches how lookups for local resources work.
+ return nil, nil
}
- }
- contentType := res.Header.Get("Content-Type")
+ if res.StatusCode < 200 || res.StatusCode > 299 {
+ return nil, toHTTPError(fmt.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode)), res, !isHeadMethod)
+ }
- // For HEAD requests we have no body to work with, so we need to use the Content-Type header.
- if isHeadMethod || c.rs.ExecHelper.Sec().HTTP.MediaTypes.Accept(contentType) {
- var found bool
- mediaType, found = c.rs.MediaTypes().GetByType(contentType)
- if !found {
- // A media type not configured in Hugo, just create one from the content type string.
- mediaType, _ = media.FromString(contentType)
+ var (
+ body []byte
+ mediaType media.Type
+ )
+ // A response to a HEAD method should not have a body. If it has one anyway, that body must be ignored.
+ // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD
+ if !isHeadMethod && res.Body != nil {
+ body, err = io.ReadAll(res.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read remote resource %q: %w", uri, err)
+ }
}
- }
- if mediaType.IsZero() {
+ filename := path.Base(rURL.Path)
+ if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil {
+ if _, ok := params["filename"]; ok {
+ filename = params["filename"]
+ }
+ }
- var extensionHints []string
+ contentType := res.Header.Get("Content-Type")
- // mime.ExtensionsByType gives a long list of extensions for text/plain,
- // just use ".txt".
- if strings.HasPrefix(contentType, "text/plain") {
- extensionHints = []string{".txt"}
- } else {
- exts, _ := mime.ExtensionsByType(contentType)
- if exts != nil {
- extensionHints = exts
+ // For HEAD requests we have no body to work with, so we need to use the Content-Type header.
+ if isHeadMethod || c.rs.ExecHelper.Sec().HTTP.MediaTypes.Accept(contentType) {
+ var found bool
+ mediaType, found = c.rs.MediaTypes().GetByType(contentType)
+ if !found {
+ // A media type not configured in Hugo, just create one from the content type string.
+ mediaType, _ = media.FromString(contentType)
}
}
- // Look for a file extension. If it's .txt, look for a more specific.
- if extensionHints == nil || extensionHints[0] == ".txt" {
- if ext := path.Ext(filename); ext != "" {
- extensionHints = []string{ext}
+ if mediaType.IsZero() {
+
+ var extensionHints []string
+
+ // mime.ExtensionsByType gives a long list of extensions for text/plain,
+ // just use ".txt".
+ if strings.HasPrefix(contentType, "text/plain") {
+ extensionHints = []string{".txt"}
+ } else {
+ exts, _ := mime.ExtensionsByType(contentType)
+ if exts != nil {
+ extensionHints = exts
+ }
}
- }
- // Now resolve the media type primarily using the content.
- mediaType = media.FromContent(c.rs.MediaTypes(), extensionHints, body)
+ // Look for a file extension. If it's .txt, look for a more specific.
+ if extensionHints == nil || extensionHints[0] == ".txt" {
+ if ext := path.Ext(filename); ext != "" {
+ extensionHints = []string{ext}
+ }
+ }
- }
+ // Now resolve the media type primarily using the content.
+ mediaType = media.FromContent(c.rs.MediaTypes(), extensionHints, body)
- if mediaType.IsZero() {
- return nil, fmt.Errorf("failed to resolve media type for remote resource %q", uri)
- }
+ }
- resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + mediaType.FirstSuffix.FullSuffix
- data := responseToData(res, false)
-
- return c.rs.NewResource(
- resources.ResourceSourceDescriptor{
- MediaType: mediaType,
- Data: data,
- GroupIdentity: identity.StringIdentity(resourceID),
- LazyPublish: true,
- OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
- return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
- },
- TargetPath: resourceID,
- })
+ if mediaType.IsZero() {
+ return nil, fmt.Errorf("failed to resolve media type for remote resource %q", uri)
+ }
+
+ userKey = filename[:len(filename)-len(path.Ext(filename))] + "_" + userKey + mediaType.FirstSuffix.FullSuffix
+ data := responseToData(res, false)
+
+ return c.rs.NewResource(
+ resources.ResourceSourceDescriptor{
+ MediaType: mediaType,
+ Data: data,
+ GroupIdentity: identity.StringIdentity(optionsKey),
+ LazyPublish: true,
+ OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
+ return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
+ },
+ TargetPath: userKey,
+ })
+ })
}
func (c *Client) validateFromRemoteArgs(uri string, options fromRemoteOptions) error {
return nil
}
-func calculateResourceID(uri string, optionsm map[string]any) string {
- if key, found := maps.LookupEqualFold(optionsm, "key"); found {
- return identity.HashString(key)
+func remoteResourceKeys(uri string, optionsm map[string]any) (string, string) {
+ var userKey string
+ if key, k, found := maps.LookupEqualFold(optionsm, "key"); found {
+ userKey = identity.HashString(key)
+ delete(optionsm, k)
+ }
+ optionsKey := identity.HashString(uri, optionsm)
+ if userKey == "" {
+ userKey = optionsKey
}
- return identity.HashString(uri, optionsm)
+ return userKey, optionsKey
}
func addDefaultHeaders(req *http.Request) {
return options, nil
}
+
+var _ http.RoundTripper = (*transport)(nil)
+
+type transport struct {
+ Cfg config.AllProvider
+ Logger loggers.Logger
+}
+
+func (t *transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
+ defer func() {
+ if resp != nil && resp.StatusCode != http.StatusNotFound && resp.StatusCode != http.StatusNotModified {
+ t.Logger.Debugf("Fetched remote resource: %s", req.URL.String())
+ }
+ }()
+
+ var (
+ start time.Time
+ nextSleep = time.Duration((rand.Intn(1000) + 100)) * time.Millisecond
+ nextSleepLimit = time.Duration(5) * time.Second
+ retry bool
+ )
+
+ for {
+ resp, retry, err = func() (*http.Response, bool, error) {
+ resp2, err := http.DefaultTransport.RoundTrip(req)
+ if err != nil {
+ return resp2, false, err
+ }
+
+ if resp2.StatusCode != http.StatusNotFound && resp2.StatusCode != http.StatusNotModified {
+ if resp2.StatusCode < 200 || resp2.StatusCode > 299 {
+ return resp2, temporaryHTTPStatusCodes[resp2.StatusCode], nil
+ }
+ }
+ return resp2, false, nil
+ }()
+
+ if retry {
+ if start.IsZero() {
+ start = time.Now()
+ } else if d := time.Since(start) + nextSleep; d >= t.Cfg.Timeout() {
+ msg := "<nil>"
+ if resp != nil {
+ msg = resp.Status
+ }
+ err := toHTTPError(fmt.Errorf("retry timeout (configured to %s) fetching remote resource: %s", t.Cfg.Timeout(), msg), resp, req.Method != "HEAD")
+ return resp, err
+ }
+ time.Sleep(nextSleep)
+ if nextSleep < nextSleepLimit {
+ nextSleep *= 2
+ }
+ continue
+ }
+
+ return
+ }
+}
+
+// We need to send the redirect responses back to the HTTP client from RoundTrip,
+// but we don't want to cache them.
+func shouldCache(statusCode int) bool {
+ switch statusCode {
+ case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
+ return false
+ }
+ return true
+}
c.Assert(req.Header["User-Agent"], qt.DeepEquals, []string{"foo"})
}
-func TestCalculateResourceID(t *testing.T) {
+func TestRemoteResourceKeys(t *testing.T) {
t.Parallel()
c := qt.New(t)
- c.Assert(calculateResourceID("foo", nil), qt.Equals, "5917621528921068675")
- c.Assert(calculateResourceID("foo", map[string]any{"bar": "baz"}), qt.Equals, "7294498335241413323")
+ check := func(uri string, optionsm map[string]any, expect1, expect2 string) {
+ got1, got2 := remoteResourceKeys(uri, optionsm)
+ c.Assert(got1, qt.Equals, expect1)
+ c.Assert(got2, qt.Equals, expect2)
+ }
- c.Assert(calculateResourceID("foo", map[string]any{"key": "1234", "bar": "baz"}), qt.Equals, "14904296279238663669")
- c.Assert(calculateResourceID("asdf", map[string]any{"key": "1234", "bar": "asdf"}), qt.Equals, "14904296279238663669")
- c.Assert(calculateResourceID("asdf", map[string]any{"key": "12345", "bar": "asdf"}), qt.Equals, "12191037851845371770")
+ check("foo", nil, "5917621528921068675", "5917621528921068675")
+ check("foo", map[string]any{"bar": "baz"}, "7294498335241413323", "7294498335241413323")
+ check("foo", map[string]any{"key": "1234", "bar": "baz"}, "14904296279238663669", "7294498335241413323")
+ check("foo", map[string]any{"key": "12345", "bar": "baz"}, "12191037851845371770", "7294498335241413323")
+ check("asdf", map[string]any{"key": "1234", "bar": "asdf"}, "14904296279238663669", "3787889110563790121")
+ check("asdf", map[string]any{"key": "12345", "bar": "asdf"}, "12191037851845371770", "3787889110563790121")
}
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/paths"
+ "github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/identity"
logger loggers.Logger,
errorHandler herrors.ErrorSender,
execHelper *hexec.Exec,
+ buildClosers types.CloseAdder,
+ rebuilder identity.SignalRebuilder,
) (*Spec, error) {
conf := s.Cfg.GetConfig().(*allconfig.Config)
imgConfig := conf.Imaging
}
rs := &Spec{
- PathSpec: s,
- Logger: logger,
- ErrorSender: errorHandler,
- imaging: imaging,
+ PathSpec: s,
+ Logger: logger,
+ ErrorSender: errorHandler,
+ BuildClosers: buildClosers,
+ Rebuilder: rebuilder,
+ imaging: imaging,
ImageCache: newImageCache(
fileCaches.ImageCache(),
memCache,
type Spec struct {
*helpers.PathSpec
- Logger loggers.Logger
- ErrorSender herrors.ErrorSender
+ Logger loggers.Logger
+ ErrorSender herrors.ErrorSender
+ BuildClosers types.CloseAdder
+ Rebuilder identity.SignalRebuilder
TextTemplates tpl.TemplateParseFinder
}
if m != nil {
- if t, found := maps.LookupEqualFold(m, "transpiler"); found {
+ if t, _, found := maps.LookupEqualFold(m, "transpiler"); found {
switch t {
case transpilerDart, transpilerLibSass:
transpiler = cast.ToString(t)
var options babel.Options
if m != nil {
options, err = babel.DecodeOptions(m)
-
if err != nil {
return nil, err
}
projects / brevno-suite / hugo / commitdiff