Fix /static performance regression from Hugo 0.103.0

In `v0.103.0` we added support for `resources.PostProcess` for all file types, not just HTML. We had benchmarks that said we were fine in that department, but those did not consider the static file syncing.

This fixes that by:

* Making sure that the /static syncer always gets its own file system without any checks for the post process token.
* For dynamic files (e.g. rendered HTML files) we add an additional check to make sure that we skip binary files (e.g. images)

Fixes #10328

diff --git a/commands/hugo.go b/commands/hugo.go
index d127d37212f259531dbe39e5a55bf2dfe1b0429e..e26f052d451d8c72300a5d1aeb756d3bf04814c7 100644 (file)
--- a/commands/hugo.go
+++ b/commands/hugo.go
@@ -653,10 +653,7 @@ func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint6
        syncer.NoChmod = c.Cfg.GetBool("noChmod")
        syncer.ChmodFilter = chmodFilter
        syncer.SrcFs = fs
-       syncer.DestFs = c.Fs.PublishDir
-       if c.renderStaticToDisk {
-               syncer.DestFs = c.Fs.PublishDirStatic
-       }
+       syncer.DestFs = c.Fs.PublishDirStatic
        // Now that we are using a unionFs for the static directories
        // We can effectively clean the publishDir on initial sync
        syncer.Delete = c.Cfg.GetBool("cleanDestinationDir")

diff --git a/deps/deps.go b/deps/deps.go
index e1cbfce069e35cc47a5d9c0df09928bc3406ae10..02730e825a03cd7c28f25dbf8d84632861e08333 100644 (file)
--- a/deps/deps.go
+++ b/deps/deps.go
@@ -2,6 +2,8 @@ package deps
 
 import (
        "fmt"
+       "path/filepath"
+       "strings"
        "sync"
        "sync/atomic"
        "time"
@@ -246,16 +248,30 @@ func New(cfg DepsCfg) (*Deps, error) {
        execHelper := hexec.New(securityConfig)
 
        var filenameHasPostProcessPrefixMu sync.Mutex
-       cb := func(name string, match bool) {
+       hashBytesReceiverFunc := func(name string, match bool) {
                if !match {
                        return
                }
                filenameHasPostProcessPrefixMu.Lock()
                d.FilenameHasPostProcessPrefix = append(d.FilenameHasPostProcessPrefix, name)
                filenameHasPostProcessPrefixMu.Unlock()
+       }
 
+       // Skip binary files.
+       hashBytesSHouldCheck := func(name string) bool {
+               ext := strings.TrimPrefix(filepath.Ext(name), ".")
+               mime, _, found := cfg.MediaTypes.GetBySuffix(ext)
+               if !found {
+                       return false
+               }
+               switch mime.MainType {
+               case "text", "application":
+                       return true
+               default:
+                       return false
+               }
        }
-       fs.PublishDir = hugofs.NewHasBytesReceiver(fs.PublishDir, cb, []byte(postpub.PostProcessPrefix))
+       fs.PublishDir = hugofs.NewHasBytesReceiver(fs.PublishDir, hashBytesSHouldCheck, hashBytesReceiverFunc, []byte(postpub.PostProcessPrefix))
 
        ps, err := helpers.NewPathSpec(fs, cfg.Language, logger)
        if err != nil {

diff --git a/hugofs/fs.go b/hugofs/fs.go
index 63c25a4c0d77decd82eaf9d9b1b7de1cc0eeb4fc..51bbe061925b3103efff2f3c21345075a0e765c6 100644 (file)
--- a/hugofs/fs.go
+++ b/hugofs/fs.go
@@ -40,8 +40,7 @@ type Fs struct {
        // It's mounted inside publishDir (default /public).
        PublishDir afero.Fs
 
-       // PublishDirStatic is the file system used for static files  when --renderStaticToDisk is set.
-       // When this is set, PublishDir is set to write to memory.
+       // PublishDirStatic is the file system used for static files.
        PublishDirStatic afero.Fs
 
        // PublishDirServer is the file system used for serving the public directory with Hugo's development server.
@@ -142,7 +141,6 @@ func isWrite(flag int) bool {
 // MakeReadableAndRemoveAllModulePkgDir makes any subdir in dir readable and then
 // removes the root.
 // TODO(bep) move this to a more suitable place.
-//
 func MakeReadableAndRemoveAllModulePkgDir(fs afero.Fs, dir string) (int, error) {
        // Safe guard
        if !strings.Contains(dir, "pkg") {

diff --git a/hugofs/hasbytes_fs.go b/hugofs/hasbytes_fs.go
index b5f82877e8cc7240ce98b0275629b7e0790366b4..3d32a828f1cb83c920b7ddd7270b1d20cf9aa449 100644 (file)
--- a/hugofs/hasbytes_fs.go
+++ b/hugofs/hasbytes_fs.go
@@ -27,12 +27,13 @@ var (
 
 type hasBytesFs struct {
        afero.Fs
+       shouldCheck      func(name string) bool
        hasBytesCallback func(name string, match bool)
        pattern          []byte
 }
 
-func NewHasBytesReceiver(delegate afero.Fs, hasBytesCallback func(name string, match bool), pattern []byte) afero.Fs {
-       return &hasBytesFs{Fs: delegate, hasBytesCallback: hasBytesCallback, pattern: pattern}
+func NewHasBytesReceiver(delegate afero.Fs, shouldCheck func(name string) bool, hasBytesCallback func(name string, match bool), pattern []byte) afero.Fs {
+       return &hasBytesFs{Fs: delegate, shouldCheck: shouldCheck, hasBytesCallback: hasBytesCallback, pattern: pattern}
 }
 
 func (fs *hasBytesFs) UnwrapFilesystem() afero.Fs {
@@ -56,6 +57,9 @@ func (fs *hasBytesFs) OpenFile(name string, flag int, perm os.FileMode) (afero.F
 }
 
 func (fs *hasBytesFs) wrapFile(f afero.File) afero.File {
+       if !fs.shouldCheck(f.Name()) {
+               return f
+       }
        return &hasBytesFile{
                File: f,
                hbw: &hugio.HasBytesWriter{

diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go
index a380857cddd7bf843a493b71bbe7f25a8abacc24..e0fed6f3e6b918040900730fa4f410c568105799 100644 (file)
--- a/hugolib/filesystems/basefs.go
+++ b/hugolib/filesystems/basefs.go
@@ -67,7 +67,7 @@ type BaseFs struct {
        // This usually maps to /my-project/public.
        PublishFs afero.Fs
 
-       // The filesystem used for renderStaticToDisk.
+       // The filesystem used for static files.
        PublishFsStatic afero.Fs
 
        // A read-only filesystem starting from the project workDir.

diff --git a/hugolib/pages_process.go b/hugolib/pages_process.go
index 04ac0218ad4c01659ee4dc0a41e157860eefbaf7..196a566f0958ed1e14554dc056e42c63d21e2020 100644 (file)
--- a/hugolib/pages_process.go
+++ b/hugolib/pages_process.go
@@ -32,10 +32,9 @@ func newPagesProcessor(h *HugoSites, sp *source.SourceSpec) *pagesProcessor {
        procs := make(map[string]pagesCollectorProcessorProvider)
        for _, s := range h.Sites {
                procs[s.Lang()] = &sitePagesProcessor{
-                       m:                  s.pageMap,
-                       errorSender:        s.h,
-                       itemChan:           make(chan interface{}, config.GetNumWorkerMultiplier()*2),
-                       renderStaticToDisk: h.Cfg.GetBool("renderStaticToDisk"),
+                       m:           s.pageMap,
+                       errorSender: s.h,
+                       itemChan:    make(chan interface{}, config.GetNumWorkerMultiplier()*2),
                }
        }
        return &pagesProcessor{
@@ -118,8 +117,6 @@ type sitePagesProcessor struct {
        ctx       context.Context
        itemChan  chan any
        itemGroup *errgroup.Group
-
-       renderStaticToDisk bool
 }
 
 func (p *sitePagesProcessor) Process(item any) error {
@@ -164,10 +161,7 @@ func (p *sitePagesProcessor) copyFile(fim hugofs.FileMetaInfo) error {
 
        defer f.Close()
 
-       fs := s.PublishFs
-       if p.renderStaticToDisk {
-               fs = s.PublishFsStatic
-       }
+       fs := s.PublishFsStatic
 
        return s.publish(&s.PathSpec.ProcessingStats.Files, target, f, fs)
 }