Also fix a logical flaw in the cache resizer that made it too aggressive. After this I haven't been able to reproduce #11988, but I need to look closer.
Closes #11973
Updates #11988
"github.com/bep/lazycache"
"github.com/bep/logg"
+ "github.com/gohugoio/hugo/common/collections"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/paths"
infol := opts.Log.InfoCommand("dynacache")
+ evictedIdentities := collections.NewStack[identity.Identity]()
+
+ onEvict := func(k, v any) {
+ if !opts.Running {
+ return
+ }
+ identity.WalkIdentitiesShallow(v, func(level int, id identity.Identity) bool {
+ evictedIdentities.Push(id)
+ return false
+ })
+ resource.MarkStale(v)
+ }
+
c := &Cache{
- partitions: make(map[string]PartitionManager),
- opts: opts,
- stats: stats,
- infol: infol,
+ partitions: make(map[string]PartitionManager),
+ onEvict: onEvict,
+ evictedIdentities: evictedIdentities,
+ opts: opts,
+ stats: stats,
+ infol: infol,
}
c.stop = c.start()
mu sync.RWMutex
partitions map[string]PartitionManager
- opts Options
- infol logg.LevelLogger
+
+ onEvict func(k, v any)
+ evictedIdentities *collections.Stack[identity.Identity]
+
+ opts Options
+ infol logg.LevelLogger
stats *stats
stopOnce sync.Once
stop func()
}
+// DrainEvictedIdentities drains the evicted identities from the cache.
+func (c *Cache) DrainEvictedIdentities() []identity.Identity {
+ return c.evictedIdentities.Drain()
+}
+
// ClearMatching clears all partition for which the predicate returns true.
func (c *Cache) ClearMatching(predicate func(k, v any) bool) {
g := rungroup.Run[PartitionManager](context.Background(), rungroup.Config[PartitionManager]{
const numberOfPartitionsEstimate = 10
maxSize := opts.CalculateMaxSize(c.opts.MaxSize / numberOfPartitionsEstimate)
+ onEvict := func(k K, v V) {
+ c.onEvict(k, v)
+ }
+
// Create a new partition and cache it.
partition := &Partition[K, V]{
- c: lazycache.New(lazycache.Options[K, V]{MaxEntries: maxSize}),
+ c: lazycache.New(lazycache.Options[K, V]{MaxEntries: maxSize, OnEvict: onEvict}),
maxSize: maxSize,
trace: c.opts.Log.Logger().WithLevel(logg.LevelTrace).WithField("partition", name),
opts: opts,
},
),
)
- resource.MarkStale(v)
return true
}
return false
if newMaxSize < minMaxSize {
newMaxSize = minMaxSize
}
+ oldMaxSize := p.maxSize
+ if newMaxSize == oldMaxSize {
+ return 0
+ }
p.maxSize = newMaxSize
// fmt.Println("Adjusting max size of partition from", oldMaxSize, "to", newMaxSize)
return p.c.Resize(newMaxSize)
func (s *stats) adjustCurrentMaxSize() bool {
newCurrentMaxSize := int(math.Floor(float64(s.opts.MaxSize) * s.adjustmentFactor))
- if newCurrentMaxSize < s.opts.MaxSize {
+ if newCurrentMaxSize < s.opts.MinMaxSize {
newCurrentMaxSize = int(s.opts.MinMaxSize)
}
changed := newCurrentMaxSize != s.currentMaxSize
cfg.Set("environment", c.r.environment)
cfg.Set("internal", maps.Params{
- "running": running,
- "watch": watch,
- "verbose": c.r.isVerbose(),
+ "running": running,
+ "watch": watch,
+ "verbose": c.r.isVerbose(),
+ "fastRenderMode": c.fastRenderMode,
})
conf, err := c.r.ConfigFromProvider(c.r.configVersionID.Load(), flagsToCfg(cd, cfg))
--- /dev/null
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collections
+
+import "sync"
+
+// Stack is a simple LIFO stack that is safe for concurrent use.
+type Stack[T any] struct {
+ items []T
+ zero T
+ mu sync.RWMutex
+}
+
+func NewStack[T any]() *Stack[T] {
+ return &Stack[T]{}
+}
+
+func (s *Stack[T]) Push(item T) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.items = append(s.items, item)
+}
+
+func (s *Stack[T]) Pop() (T, bool) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if len(s.items) == 0 {
+ return s.zero, false
+ }
+ item := s.items[len(s.items)-1]
+ s.items = s.items[:len(s.items)-1]
+ return item, true
+}
+
+func (s *Stack[T]) Peek() (T, bool) {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ if len(s.items) == 0 {
+ return s.zero, false
+ }
+ return s.items[len(s.items)-1], true
+}
+
+func (s *Stack[T]) Len() int {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ return len(s.items)
+}
+
+func (s *Stack[T]) Drain() []T {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ items := s.items
+ s.items = nil
+ return items
+}
Verbose bool
Clock string
Watch bool
+ FastRenderMode bool
LiveReloadPort int
}
return c.m.IsMultihost
}
+func (c ConfigLanguage) FastRenderMode() bool {
+ return c.config.Internal.FastRenderMode
+}
+
func (c ConfigLanguage) IsMultiLingual() bool {
return len(c.m.Languages) > 1
}
BuildDrafts() bool
Running() bool
Watching() bool
+ FastRenderMode() bool
PrintUnusedTemplates() bool
EnableMissingTranslationPlaceholders() bool
TemplateMetrics() bool
b = cachebuster(s)
}
- if b {
- identity.WalkIdentitiesShallow(v, func(level int, id identity.Identity) bool {
- // Add them to the change set so we can reset any page that depends on them.
- changes = append(changes, id)
- return false
- })
- }
-
return b
}
}
}
+ // Drain the the cache eviction stack.
+ evicted := h.Deps.MemCache.DrainEvictedIdentities()
+ if len(evicted) < 200 {
+ changes = append(changes, evicted...)
+ } else {
+ // Mass eviction, we might as well invalidate everything.
+ changes = []identity.Identity{identity.GenghisKhan}
+ }
+
// Remove duplicates
seen := make(map[identity.Identity]bool)
var n int
*fatalErrorHandler
*buildCounters
+ // Tracks invocations of the Build method.
+ buildCounter atomic.Uint64
}
// ShouldSkipFileChangeEvent allows skipping filesystem event early before
return false
}
- fastRenderMode := cfg.RecentlyVisited.Len() > 0
+ fastRenderMode := p.s.Conf.FastRenderMode()
- if !fastRenderMode {
- // Not in fast render mode or first time render.
+ if !fastRenderMode || p.s.h.buildCounter.Load() == 0 {
return shouldRender
}
func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
infol := h.Log.InfoCommand("build")
defer loggers.TimeTrackf(infol, time.Now(), nil, "")
+ defer func() {
+ h.buildCounter.Add(1)
+ }()
if h.Deps == nil {
panic("must have deps")
}
case files.ComponentFolderAssets:
logger.Println("Asset changed", pathInfo.Path())
- r, _ := h.ResourceSpec.ResourceCache.Get(context.Background(), dynacache.CleanKey(pathInfo.Base()))
+
var hasID bool
+ r, _ := h.ResourceSpec.ResourceCache.Get(context.Background(), dynacache.CleanKey(pathInfo.Base()))
identity.WalkIdentitiesShallow(r, func(level int, rid identity.Identity) bool {
hasID = true
changes = append(changes, rid)
projects / brevno-suite / hugo / commitdiff