deploy: walkLocal worker pool for performance

diff --git a/deploy/deploy.go b/deploy/deploy.go
index 4b90881a60aabb6874bc7577d0f7fd3ed5005c0b..57e1f41a265a132d37d2eeb1b81fa9dd99d47170 100644 (file)
--- a/deploy/deploy.go
+++ b/deploy/deploy.go
@@ -36,6 +36,7 @@ import (
        "github.com/dustin/go-humanize"
        "github.com/gobwas/glob"
        "github.com/gohugoio/hugo/common/loggers"
+       "github.com/gohugoio/hugo/common/para"
        "github.com/gohugoio/hugo/config"
        "github.com/gohugoio/hugo/deploy/deployconfig"
        "github.com/gohugoio/hugo/media"
@@ -487,7 +488,12 @@ func knownHiddenDirectory(name string) bool {
 // walkLocal walks the source directory and returns a flat list of files,
 // using localFile.SlashPath as the map keys.
 func (d *Deployer) walkLocal(fs afero.Fs, matchers []*deployconfig.Matcher, include, exclude glob.Glob, mediaTypes media.Types, mappath func(string) string) (map[string]*localFile, error) {
-       retval := map[string]*localFile{}
+       retval := make(map[string]*localFile)
+       var mu sync.Mutex
+
+       workers := para.New(d.cfg.Workers)
+       g, _ := workers.Start(context.Background())
+
        err := afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error {
                if err != nil {
                        return err
@@ -508,45 +514,54 @@ func (d *Deployer) walkLocal(fs afero.Fs, matchers []*deployconfig.Matcher, incl
                        return nil
                }
 
-               // When a file system is HFS+, its filepath is in NFD form.
-               if runtime.GOOS == "darwin" {
-                       path = norm.NFC.String(path)
-               }
+               // Process each file in a worker
+               g.Run(func() error {
+                       // When a file system is HFS+, its filepath is in NFD form.
+                       if runtime.GOOS == "darwin" {
+                               path = norm.NFC.String(path)
+                       }
 
-               // Check include/exclude matchers.
-               slashpath := filepath.ToSlash(path)
-               if include != nil && !include.Match(slashpath) {
-                       d.logger.Infof("  dropping %q due to include\n", slashpath)
-                       return nil
-               }
-               if exclude != nil && exclude.Match(slashpath) {
-                       d.logger.Infof("  dropping %q due to exclude\n", slashpath)
-                       return nil
-               }
+                       // Check include/exclude matchers.
+                       slashpath := filepath.ToSlash(path)
+                       if include != nil && !include.Match(slashpath) {
+                               d.logger.Infof("  dropping %q due to include\n", slashpath)
+                               return nil
+                       }
+                       if exclude != nil && exclude.Match(slashpath) {
+                               d.logger.Infof("  dropping %q due to exclude\n", slashpath)
+                               return nil
+                       }
 
-               // Find the first matching matcher (if any).
-               var m *deployconfig.Matcher
-               for _, cur := range matchers {
-                       if cur.Matches(slashpath) {
-                               m = cur
-                               break
+                       // Find the first matching matcher (if any).
+                       var m *deployconfig.Matcher
+                       for _, cur := range matchers {
+                               if cur.Matches(slashpath) {
+                                       m = cur
+                                       break
+                               }
                        }
-               }
-               // Apply any additional modifications to the local path, to map it to
-               // the remote path.
-               if mappath != nil {
-                       slashpath = mappath(slashpath)
-               }
-               lf, err := newLocalFile(fs, path, slashpath, m, mediaTypes)
-               if err != nil {
-                       return err
-               }
-               retval[lf.SlashPath] = lf
+                       // Apply any additional modifications to the local path, to map it to
+                       // the remote path.
+                       if mappath != nil {
+                               slashpath = mappath(slashpath)
+                       }
+                       lf, err := newLocalFile(fs, path, slashpath, m, mediaTypes)
+                       if err != nil {
+                               return err
+                       }
+                       mu.Lock()
+                       retval[lf.SlashPath] = lf
+                       mu.Unlock()
+                       return nil
+               })
                return nil
        })
        if err != nil {
                return nil, err
        }
+       if err := g.Wait(); err != nil {
+               return nil, err
+       }
        return retval, nil
 }
 

diff --git a/deploy/deploy_test.go b/deploy/deploy_test.go
index 3bbc9ff709e01353ba405e109f25e31050bd25ac..bdc8299a08b8a2d81da751d0999be0dc04f4c935 100644 (file)
--- a/deploy/deploy_test.go
+++ b/deploy/deploy_test.go
@@ -623,7 +623,7 @@ func TestEndToEndSync(t *testing.T) {
                                localFs:    test.fs,
                                bucket:     test.bucket,
                                mediaTypes: media.DefaultTypes,
-                               cfg:        deployconfig.DeployConfig{MaxDeletes: -1},
+                               cfg:        deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1},
                        }
 
                        // Initial deployment should sync remote with local.
@@ -706,7 +706,7 @@ func TestMaxDeletes(t *testing.T) {
                                localFs:    test.fs,
                                bucket:     test.bucket,
                                mediaTypes: media.DefaultTypes,
-                               cfg:        deployconfig.DeployConfig{MaxDeletes: -1},
+                               cfg:        deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1},
                        }
 
                        // Sync remote with local.
@@ -836,7 +836,7 @@ func TestIncludeExclude(t *testing.T) {
                        }
                        deployer := &Deployer{
                                localFs: fsTest.fs,
-                               cfg:     deployconfig.DeployConfig{MaxDeletes: -1}, bucket: fsTest.bucket,
+                               cfg:     deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1}, bucket: fsTest.bucket,
                                target:     tgt,
                                mediaTypes: media.DefaultTypes,
                        }
@@ -893,7 +893,7 @@ func TestIncludeExcludeRemoteDelete(t *testing.T) {
                        }
                        deployer := &Deployer{
                                localFs: fsTest.fs,
-                               cfg:     deployconfig.DeployConfig{MaxDeletes: -1}, bucket: fsTest.bucket,
+                               cfg:     deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1}, bucket: fsTest.bucket,
                                mediaTypes: media.DefaultTypes,
                        }
 
@@ -945,7 +945,7 @@ func TestCompression(t *testing.T) {
                        deployer := &Deployer{
                                localFs:    test.fs,
                                bucket:     test.bucket,
-                               cfg:        deployconfig.DeployConfig{MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: ".*", Gzip: true, Re: regexp.MustCompile(".*")}}},
+                               cfg:        deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: ".*", Gzip: true, Re: regexp.MustCompile(".*")}}},
                                mediaTypes: media.DefaultTypes,
                        }
 
@@ -1000,7 +1000,7 @@ func TestMatching(t *testing.T) {
                        deployer := &Deployer{
                                localFs:    test.fs,
                                bucket:     test.bucket,
-                               cfg:        deployconfig.DeployConfig{MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: "^subdir/aaa$", Force: true, Re: regexp.MustCompile("^subdir/aaa$")}}},
+                               cfg:        deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: "^subdir/aaa$", Force: true, Re: regexp.MustCompile("^subdir/aaa$")}}},
                                mediaTypes: media.DefaultTypes,
                        }
 
@@ -1097,5 +1097,6 @@ func verifyRemote(ctx context.Context, bucket *blob.Bucket, local []*fileData) (
 func newDeployer() *Deployer {
        return &Deployer{
                logger: loggers.NewDefault(),
+               cfg:    deployconfig.DeployConfig{Workers: 2},
        }
 }