Allow rendering static files to disk and dynamic to memory in server mode

Updates #9625

diff --git a/commands/commandeer.go b/commands/commandeer.go
index bf42501e0ce1016f276e8ec760bd15e12255b8ca..07181f2447a15877ddb18ed0a1d2118e3d8f83f6 100644 (file)
--- a/commands/commandeer.go
+++ b/commands/commandeer.go
@@ -92,6 +92,7 @@ type commandeer struct {
        languagesConfigured bool
        languages           langs.Languages
        doLiveReload        bool
+       renderStaticToDisk  bool
        fastRenderMode      bool
        showErrorInBrowser  bool
        wasError            bool
@@ -368,8 +369,9 @@ func (c *commandeer) loadConfig() error {
        }
 
        createMemFs := config.GetBool("renderToMemory")
+       c.renderStaticToDisk = config.GetBool("renderStaticToDisk")
 
-       if createMemFs {
+       if createMemFs && !c.renderStaticToDisk {
                // Rendering to memoryFS, publish to Root regardless of publishDir.
                config.Set("publishDir", "/")
        }
@@ -380,6 +382,14 @@ func (c *commandeer) loadConfig() error {
                if c.destinationFs != nil {
                        // Need to reuse the destination on server rebuilds.
                        fs.Destination = c.destinationFs
+               } else if createMemFs && c.renderStaticToDisk {
+                       // Writes the dynamic output on memory,
+                       // while serve others directly from publishDir
+                       publishDir := config.GetString("publishDir")
+                       writableFs := afero.NewBasePathFs(afero.NewMemMapFs(), publishDir)
+                       publicFs := afero.NewOsFs()
+                       fs.Destination = afero.NewCopyOnWriteFs(afero.NewReadOnlyFs(publicFs), writableFs)
+                       fs.DestinationStatic = publicFs
                } else if createMemFs {
                        // Hugo writes the output to memory instead of the disk.
                        fs.Destination = new(afero.MemMapFs)
@@ -397,11 +407,13 @@ func (c *commandeer) loadConfig() error {
 
                        changeDetector.PrepareNew()
                        fs.Destination = hugofs.NewHashingFs(fs.Destination, changeDetector)
+                       fs.DestinationStatic = hugofs.NewHashingFs(fs.DestinationStatic, changeDetector)
                        c.changeDetector = changeDetector
                }
 
                if c.Cfg.GetBool("logPathWarnings") {
                        fs.Destination = hugofs.NewCreateCountingFs(fs.Destination)
+                       fs.DestinationStatic = hugofs.NewCreateCountingFs(fs.DestinationStatic)
                }
 
                // To debug hard-to-find path issues.

diff --git a/commands/hugo.go b/commands/hugo.go
index 8c5294f006daffce37887977f87afaec5f716a4d..21140fa4307b89f9a95ad68f55918220c9cf5b55 100644 (file)
--- a/commands/hugo.go
+++ b/commands/hugo.go
@@ -652,6 +652,9 @@ func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint6
        syncer.ChmodFilter = chmodFilter
        syncer.SrcFs = fs
        syncer.DestFs = c.Fs.Destination
+       if c.renderStaticToDisk {
+               syncer.DestFs = c.Fs.DestinationStatic
+       }
        // Now that we are using a unionFs for the static directories
        // We can effectively clean the publishDir on initial sync
        syncer.Delete = c.Cfg.GetBool("cleanDestinationDir")

diff --git a/commands/server.go b/commands/server.go
index 7d9462b36554f8891c74a1d708f01815f0140d99..3f52908167293ce036971db27774c45dd2cf699f 100644 (file)
--- a/commands/server.go
+++ b/commands/server.go
@@ -48,15 +48,16 @@ type serverCmd struct {
        // Can be used to stop the server. Useful in tests
        stop <-chan bool
 
-       disableLiveReload bool
-       navigateToChanged bool
-       renderToDisk      bool
-       serverAppend      bool
-       serverInterface   string
-       serverPort        int
-       liveReloadPort    int
-       serverWatch       bool
-       noHTTPCache       bool
+       disableLiveReload  bool
+       navigateToChanged  bool
+       renderToDisk       bool
+       renderStaticToDisk bool
+       serverAppend       bool
+       serverInterface    string
+       serverPort         int
+       liveReloadPort     int
+       serverWatch        bool
+       noHTTPCache        bool
 
        disableFastRender   bool
        disableBrowserError bool
@@ -101,6 +102,7 @@ of a second, you will be able to save and see your changes nearly instantly.`,
        cc.cmd.Flags().BoolVar(&cc.renderToDisk, "renderToDisk", false, "render to Destination path (default is render to memory & serve from there)")
        cc.cmd.Flags().BoolVar(&cc.disableFastRender, "disableFastRender", false, "enables full re-renders on changes")
        cc.cmd.Flags().BoolVar(&cc.disableBrowserError, "disableBrowserError", false, "do not show build errors in the browser")
+       cc.cmd.Flags().BoolVar(&cc.renderStaticToDisk, "renderStaticToDisk", false, "render static files to disk but dynamic files render to memory.")
 
        cc.cmd.Flags().String("memstats", "", "log memory usage to this file")
        cc.cmd.Flags().String("meminterval", "100ms", "interval to poll memory usage (requires --memstats), valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\".")
@@ -141,6 +143,7 @@ func (sc *serverCmd) server(cmd *cobra.Command, args []string) error {
 
        cfgInit := func(c *commandeer) error {
                c.Set("renderToMemory", !sc.renderToDisk)
+               c.Set("renderStaticToDisk", sc.renderStaticToDisk)
                if cmd.Flags().Changed("navigateToChanged") {
                        c.Set("navigateToChanged", sc.navigateToChanged)
                }
@@ -332,6 +335,8 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, erro
        if i == 0 {
                if f.s.renderToDisk {
                        jww.FEEDBACK.Println("Serving pages from " + absPublishDir)
+               } else if f.s.renderStaticToDisk {
+                       jww.FEEDBACK.Println("Serving pages from memory and static files from " + absPublishDir)
                } else {
                        jww.FEEDBACK.Println("Serving pages from memory")
                }

diff --git a/commands/static_syncer.go b/commands/static_syncer.go
index 5569d4de6882c435b1c22a78b811db60094746c6..2eb2b666233883d883a2c33639c436fa6d2f60bd 100644 (file)
--- a/commands/static_syncer.go
+++ b/commands/static_syncer.go
@@ -56,6 +56,9 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
                syncer.ChmodFilter = chmodFilter
                syncer.SrcFs = sourceFs.Fs
                syncer.DestFs = c.Fs.Destination
+               if c.renderStaticToDisk {
+                       syncer.DestFs = c.Fs.DestinationStatic
+               }
 
                // prevent spamming the log on changes
                logger := helpers.NewDistinctErrorLogger()
@@ -101,7 +104,11 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
                                        toRemove := filepath.Join(publishDir, relPath)
 
                                        logger.Println("File no longer exists in static dir, removing", toRemove)
-                                       _ = c.Fs.Destination.RemoveAll(toRemove)
+                                       if c.renderStaticToDisk {
+                                               _ = c.Fs.DestinationStatic.RemoveAll(toRemove)
+                                       } else {
+                                               _ = c.Fs.Destination.RemoveAll(toRemove)
+                                       }
                                } else if err == nil {
                                        // If file still exists, sync it
                                        logger.Println("Syncing", relPath, "to", publishDir)

diff --git a/hugofs/fs.go b/hugofs/fs.go
index 54d962553e70ae6ba2f8a5e7babc40fb72fca01c..95645204e9f6d0e434ec4ad214ffd4a77fc2315c 100644 (file)
--- a/hugofs/fs.go
+++ b/hugofs/fs.go
@@ -35,6 +35,9 @@ type Fs struct {
        // Destination is Hugo's destination file system.
        Destination afero.Fs
 
+       // Destination used for `renderStaticToDisk`
+       DestinationStatic afero.Fs
+
        // Os is an OS file system.
        // NOTE: Field is currently unused.
        Os afero.Fs
@@ -69,10 +72,11 @@ func NewFrom(fs afero.Fs, cfg config.Provider) *Fs {
 
 func newFs(base afero.Fs, cfg config.Provider) *Fs {
        return &Fs{
-               Source:      base,
-               Destination: base,
-               Os:          &afero.OsFs{},
-               WorkingDir:  getWorkingDirFs(base, cfg),
+               Source:            base,
+               Destination:       base,
+               DestinationStatic: base,
+               Os:                &afero.OsFs{},
+               WorkingDir:        getWorkingDirFs(base, cfg),
        }
 }
 

diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go
index 2e32932c66b0502ef25523839f88f0363e30b9b0..1614bd0b31a1400f80280019975386dd930b0296 100644 (file)
--- a/hugolib/filesystems/basefs.go
+++ b/hugolib/filesystems/basefs.go
@@ -71,6 +71,9 @@ type BaseFs struct {
        // A read-only filesystem starting from the project workDir.
        WorkDir afero.Fs
 
+       // The filesystem used for renderStaticToDisk.
+       PublishFsStatic afero.Fs
+
        theBigFs *filesystemsCollector
 
        // Locks.
@@ -438,15 +441,17 @@ func NewBase(p *paths.Paths, logger loggers.Logger, options ...func(*BaseFs) err
 
        publishFs := hugofs.NewBaseFileDecorator(afero.NewBasePathFs(fs.Destination, p.AbsPublishDir))
        sourceFs := hugofs.NewBaseFileDecorator(afero.NewBasePathFs(fs.Source, p.WorkingDir))
+       publishFsStatic := afero.NewBasePathFs(fs.Source, p.AbsPublishDir)
 
        // Same as sourceFs, but no decoration. This is what's used by os.ReadDir etc.
        workDir := afero.NewBasePathFs(afero.NewReadOnlyFs(fs.Source), p.WorkingDir)
 
        b := &BaseFs{
-               SourceFs:  sourceFs,
-               WorkDir:   workDir,
-               PublishFs: publishFs,
-               buildMu:   lockedfile.MutexAt(filepath.Join(p.WorkingDir, lockFileBuild)),
+               SourceFs:        sourceFs,
+               WorkDir:         workDir,
+               PublishFs:       publishFs,
+               PublishFsStatic: publishFsStatic,
+               buildMu:         lockedfile.MutexAt(filepath.Join(p.WorkingDir, lockFileBuild)),
        }
 
        for _, opt := range options {

diff --git a/hugolib/pages_process.go b/hugolib/pages_process.go
index 541c0ae3e4537ec05dba31e391e0b9c216967643..59b20dabc70508c13de627675166ad0127017219 100644 (file)
--- a/hugolib/pages_process.go
+++ b/hugolib/pages_process.go
@@ -33,9 +33,10 @@ func newPagesProcessor(h *HugoSites, sp *source.SourceSpec) *pagesProcessor {
        procs := make(map[string]pagesCollectorProcessorProvider)
        for _, s := range h.Sites {
                procs[s.Lang()] = &sitePagesProcessor{
-                       m:           s.pageMap,
-                       errorSender: s.h,
-                       itemChan:    make(chan interface{}, config.GetNumWorkerMultiplier()*2),
+                       m:                  s.pageMap,
+                       errorSender:        s.h,
+                       itemChan:           make(chan interface{}, config.GetNumWorkerMultiplier()*2),
+                       renderStaticToDisk: h.Cfg.GetBool("renderStaticToDisk"),
                }
        }
        return &pagesProcessor{
@@ -118,6 +119,8 @@ type sitePagesProcessor struct {
        ctx       context.Context
        itemChan  chan interface{}
        itemGroup *errgroup.Group
+
+       renderStaticToDisk bool
 }
 
 func (p *sitePagesProcessor) Process(item interface{}) error {
@@ -162,7 +165,12 @@ func (p *sitePagesProcessor) copyFile(fim hugofs.FileMetaInfo) error {
 
        defer f.Close()
 
-       return s.publish(&s.PathSpec.ProcessingStats.Files, target, f)
+       fs := s.PublishFs
+       if p.renderStaticToDisk {
+               fs = s.PublishFsStatic
+       }
+
+       return s.publish(&s.PathSpec.ProcessingStats.Files, target, f, fs)
 }
 
 func (p *sitePagesProcessor) doProcess(item interface{}) error {

diff --git a/hugolib/site.go b/hugolib/site.go
index c76bdc141bc26739a31cfde874ab58951e230a47..0b8e807dd4033c0ad2ae74a4666ba8583b5c78d4 100644 (file)
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -1824,10 +1824,10 @@ func (s *Site) lookupTemplate(layouts ...string) (tpl.Template, bool) {
        return nil, false
 }
 
-func (s *Site) publish(statCounter *uint64, path string, r io.Reader) (err error) {
+func (s *Site) publish(statCounter *uint64, path string, r io.Reader, fs afero.Fs) (err error) {
        s.PathSpec.ProcessingStats.Incr(statCounter)
 
-       return helpers.WriteToDisk(filepath.Clean(path), r, s.BaseFs.PublishFs)
+       return helpers.WriteToDisk(filepath.Clean(path), r, fs)
 }
 
 func (s *Site) kindFromFileInfoOrSections(fi *fileInfo, sections []string) string {