Misc resource fixes/improvements

* Add --pprof flag to server to enable profile debugging.
* Don't cache the resource content, it seem to eat memory on bigger sites.
* Keep --printMemoryUsag running in server

Fixes #11974

diff --git a/commands/commandeer.go b/commands/commandeer.go
index 1aac08c42f3817434f4ffb873f13d56a8067e92f..0052d91b4c62115d17f0171ec0790cef0da4913d 100644 (file)
--- a/commands/commandeer.go
+++ b/commands/commandeer.go
@@ -22,6 +22,7 @@ import (
        "os"
        "os/signal"
        "path/filepath"
+       "runtime"
        "strings"
        "sync"
        "sync/atomic"
@@ -341,8 +342,12 @@ func (r *rootCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args
                if r.buildWatch {
                        defer r.timeTrack(time.Now(), "Built")
                }
-               err := b.build()
-               return err
+               close, err := b.build()
+               if err != nil {
+                       return err
+               }
+               close()
+               return nil
        }()
        if err != nil {
                return err
@@ -411,6 +416,7 @@ func (r *rootCommand) PreRun(cd, runner *simplecobra.Commandeer) error {
                MaxEntries: 1,
                OnEvict: func(key int32, value *hugolib.HugoSites) {
                        value.Close()
+                       runtime.GC()
                },
        })
 

diff --git a/commands/hugobuilder.go b/commands/hugobuilder.go
index 41f42ae6d42ecaf7ca08c7b8a82372d8fdf0f937..ddc92129c26b076045a029f3e7b114bbf97e02ab 100644 (file)
--- a/commands/hugobuilder.go
+++ b/commands/hugobuilder.go
@@ -361,34 +361,32 @@ func (c *hugoBuilder) newWatcher(pollIntervalStr string, dirList ...string) (*wa
        return watcher, nil
 }
 
-func (c *hugoBuilder) build() error {
+func (c *hugoBuilder) build() (func(), error) {
        stopProfiling, err := c.initProfiling()
        if err != nil {
-               return err
+               return nil, err
        }
 
-       defer func() {
-               if stopProfiling != nil {
-                       stopProfiling()
-               }
-       }()
-
        if err := c.fullBuild(false); err != nil {
-               return err
+               return nil, err
        }
 
        if !c.r.quiet {
                c.r.Println()
                h, err := c.hugo()
                if err != nil {
-                       return err
+                       return nil, err
                }
 
                h.PrintProcessingStats(os.Stdout)
                c.r.Println()
        }
 
-       return nil
+       return func() {
+               if stopProfiling != nil {
+                       stopProfiling()
+               }
+       }, nil
 }
 
 func (c *hugoBuilder) buildSites(noBuildLock bool) (err error) {

diff --git a/commands/server.go b/commands/server.go
index 97cf405b7ef23aed733a843fad7d5d932679423b..574f5c3404c092df84c7c49e8ebb4fdeba7165a5 100644 (file)
--- a/commands/server.go
+++ b/commands/server.go
@@ -25,6 +25,7 @@ import (
        "io"
        "net"
        "net/http"
+       _ "net/http/pprof"
        "net/url"
        "os"
        "os/signal"
@@ -451,6 +452,7 @@ type serverCommand struct {
        tlsCertFile         string
        tlsKeyFile          string
        tlsAuto             bool
+       pprof               bool
        serverPort          int
        liveReloadPort      int
        serverWatch         bool
@@ -465,6 +467,11 @@ func (c *serverCommand) Name() string {
 }
 
 func (c *serverCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error {
+       if c.pprof {
+               go func() {
+                       http.ListenAndServe("localhost:8080", nil)
+               }()
+       }
        // Watch runs its own server as part of the routine
        if c.serverWatch {
 
@@ -487,15 +494,19 @@ func (c *serverCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, arg
 
        }
 
+       var close func()
        err := func() error {
                defer c.r.timeTrack(time.Now(), "Built")
-               err := c.build()
+               var err error
+               close, err = c.build()
                return err
        }()
        if err != nil {
                return err
        }
 
+       defer close()
+
        return c.serve()
 }
 
@@ -520,6 +531,7 @@ of a second, you will be able to save and see your changes nearly instantly.`
        cmd.Flags().StringVarP(&c.tlsCertFile, "tlsCertFile", "", "", "path to TLS certificate file")
        cmd.Flags().StringVarP(&c.tlsKeyFile, "tlsKeyFile", "", "", "path to TLS key file")
        cmd.Flags().BoolVar(&c.tlsAuto, "tlsAuto", false, "generate and use locally-trusted certificates.")
+       cmd.Flags().BoolVar(&c.pprof, "pprof", false, "enable the pprof server (port 8080)")
        cmd.Flags().BoolVarP(&c.serverWatch, "watch", "w", true, "watch filesystem for changes and recreate as needed")
        cmd.Flags().BoolVar(&c.noHTTPCache, "noHTTPCache", false, "prevent HTTP caching")
        cmd.Flags().BoolVarP(&c.serverAppend, "appendPort", "", true, "append port to baseURL")

diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go
index 257949334c9a4ad7153001386f4b8b1a31d0bfa2..a15e155049118d90e0e71971b28baac3d780cfda 100644 (file)
--- a/hugolib/hugo_sites_build.go
+++ b/hugolib/hugo_sites_build.go
@@ -720,9 +720,11 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf
                        h.pageTrees.treeTaxonomyEntries.DeletePrefix("")
 
                        if delete {
+
                                _, ok := h.pageTrees.treePages.LongestPrefixAll(pathInfo.Base())
                                if ok {
                                        h.pageTrees.treePages.DeleteAll(pathInfo.Base())
+                                       h.pageTrees.resourceTrees.DeleteAll(pathInfo.Base())
                                        if pathInfo.IsBundle() {
                                                // Assume directory removed.
                                                h.pageTrees.treePages.DeletePrefixAll(pathInfo.Base() + "/")

diff --git a/hugolib/rebuild_test.go b/hugolib/rebuild_test.go
index d3ac5665d1f3a69e7400f31baaa1b68d5a6d23b5..74b04fe0116577015cc911df99b63edd9767eaf8 100644 (file)
--- a/hugolib/rebuild_test.go
+++ b/hugolib/rebuild_test.go
@@ -101,10 +101,10 @@ func TestRebuildEditTextFileInBranchBundle(t *testing.T) {
 
 func TestRebuildRenameTextFileInLeafBundle(t *testing.T) {
        b := TestRunning(t, rebuildFilesSimple)
-       b.AssertFileContent("public/mysection/mysectionbundle/index.html", "My Section Bundle Text 2 Content.")
+       b.AssertFileContent("public/mysection/mysectionbundle/index.html", "My Section Bundle Text 2 Content.", "Len Resources: 2|")
 
        b.RenameFile("content/mysection/mysectionbundle/mysectionbundletext.txt", "content/mysection/mysectionbundle/mysectionbundletext2.txt").Build()
-       b.AssertFileContent("public/mysection/mysectionbundle/index.html", "mysectionbundletext2", "My Section Bundle Text 2 Content.")
+       b.AssertFileContent("public/mysection/mysectionbundle/index.html", "mysectionbundletext2", "My Section Bundle Text 2 Content.", "Len Resources: 2|")
        b.AssertRenderCountPage(3)
        b.AssertRenderCountContent(3)
 }

diff --git a/resources/resource.go b/resources/resource.go
index e78dd12cb913c0e0ef5e29c191584301ecb6d4c8..6bef1b2756e96f942bf300d4d95c93619be5c763 100644 (file)
--- a/resources/resource.go
+++ b/resources/resource.go
@@ -343,7 +343,7 @@ func GetTestInfoForResource(r resource.Resource) GenericResourceTestInfo {
 
 // genericResource represents a generic linkable resource.
 type genericResource struct {
-       *resourceContent
+       publishInit *sync.Once
 
        sd    ResourceSourceDescriptor
        paths internal.ResourcePaths
@@ -412,11 +412,18 @@ func (l *genericResource) cloneTo(targetPath string) resource.Resource {
 }
 
 func (l *genericResource) Content(context.Context) (any, error) {
-       if err := l.initContent(); err != nil {
-               return nil, err
+       r, err := l.ReadSeekCloser()
+       if err != nil {
+               return "", err
        }
+       defer r.Close()
 
-       return l.content, nil
+       var b []byte
+       b, err = io.ReadAll(r)
+       if err != nil {
+               return "", err
+       }
+       return string(b), nil
 }
 
 func (r *genericResource) Err() resource.ResourceError {
@@ -527,28 +534,6 @@ func (l *genericResource) Title() string {
        return l.title
 }
 
-func (l *genericResource) initContent() error {
-       var err error
-       l.contentInit.Do(func() {
-               var r hugio.ReadSeekCloser
-               r, err = l.ReadSeekCloser()
-               if err != nil {
-                       return
-               }
-               defer r.Close()
-
-               var b []byte
-               b, err = io.ReadAll(r)
-               if err != nil {
-                       return
-               }
-
-               l.content = string(b)
-       })
-
-       return err
-}
-
 func (l *genericResource) getSpec() *Spec {
        return l.spec
 }
@@ -588,12 +573,9 @@ func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResour
        r := rc.clone()
 
        if u.content != nil {
-               r.contentInit.Do(func() {
-                       r.content = *u.content
-                       r.sd.OpenReadSeekCloser = func() (hugio.ReadSeekCloser, error) {
-                               return hugio.NewReadSeekerNoOpCloserFromString(r.content), nil
-                       }
-               })
+               r.sd.OpenReadSeekCloser = func() (hugio.ReadSeekCloser, error) {
+                       return hugio.NewReadSeekerNoOpCloserFromString(*u.content), nil
+               }
        }
 
        r.sd.MediaType = u.mediaType
@@ -620,7 +602,7 @@ func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResour
 }
 
 func (l genericResource) clone() *genericResource {
-       l.resourceContent = &resourceContent{}
+       l.publishInit = &sync.Once{}
        return &l
 }
 
@@ -633,13 +615,6 @@ type targetPather interface {
        TargetPath() string
 }
 
-type resourceContent struct {
-       content     string
-       contentInit sync.Once
-
-       publishInit sync.Once
-}
-
 type resourceHash struct {
        value    string
        size     int64

diff --git a/resources/resource_spec.go b/resources/resource_spec.go
index 66f56d147b35826ab9cedbee8ea779aad390d617..5fdcdd831f970baba5fd19e90d530bdcc79ec2dc 100644 (file)
--- a/resources/resource_spec.go
+++ b/resources/resource_spec.go
@@ -163,15 +163,15 @@ func (r *Spec) NewResource(rd ResourceSourceDescriptor) (resource.Resource, erro
        }
 
        gr := &genericResource{
-               Staler:          &AtomicStaler{},
-               h:               &resourceHash{},
-               paths:           rp,
-               spec:            r,
-               sd:              rd,
-               params:          make(map[string]any),
-               name:            rd.Name,
-               title:           rd.Name,
-               resourceContent: &resourceContent{},
+               Staler:      &AtomicStaler{},
+               h:           &resourceHash{},
+               publishInit: &sync.Once{},
+               paths:       rp,
+               spec:        r,
+               sd:          rd,
+               params:      make(map[string]any),
+               name:        rd.Name,
+               title:       rd.Name,
        }
 
        if rd.MediaType.MainType == "image" {