Add HUGO_NUMWORKERMULTIPLIER

And use that to calculate number of workers, if set, else fall back to number of logical CPUs.

Also tweak the relevant related settings to match the new setup.

Also remove the setting of `runtime.GOMAXPROCS` as this has been the default behaviour since Go 1.5.

Fixes #5814

diff --git a/config/env.go b/config/env.go
new file mode 100644 (file)
index 0000000..adf6f9b
--- /dev/null
+++ b/config/env.go
@@ -0,0 +1,33 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+       "os"
+       "runtime"
+       "strconv"
+)
+
+// GetNumWorkerMultiplier returns the base value used to calculate the number
+// of workers to use for Hugo's parallel execution.
+// It returns the value in HUGO_NUMWORKERMULTIPLIER OS env variable if set to a
+// positive integer, else the number of logical CPUs.
+func GetNumWorkerMultiplier() int {
+       if gmp := os.Getenv("HUGO_NUMWORKERMULTIPLIER"); gmp != "" {
+               if p, err := strconv.Atoi(gmp); err == nil && p > 0 {
+                       return p
+               }
+       }
+       return runtime.NumCPU()
+}

diff --git a/docs/content/en/getting-started/configuration.md b/docs/content/en/getting-started/configuration.md
index 5b310db15ccd72ce9c7327590d36c974b2a60e61..4ef42465f6bb3ca3495516c7c8f0ce5f76a1e22b 100644 (file)
--- a/docs/content/en/getting-started/configuration.md
+++ b/docs/content/en/getting-started/configuration.md
@@ -297,6 +297,11 @@ enableemoji: true
 ```
 {{% /note %}}
 
+## Configuration Environment Variables
+
+HUGO_NUMWORKERMULTIPLIER
+: Can be set to increase or reduce the number of workers used in parallel processing in Hugo. If not set, the number of logical CPUs will be used.
+
 ## Configuration Lookup Order
 
 Similar to the template [lookup order][], Hugo has a default set of rules for searching for a configuration file in the root of your website's source directory as a default behavior:

diff --git a/hugolib/pagebundler.go b/hugolib/pagebundler.go
index 682221d8c6b9a1d47d785e41431752e72fd92438..5149968bcfb32fcd76297d133544d7f696c0996f 100644 (file)
--- a/hugolib/pagebundler.go
+++ b/hugolib/pagebundler.go
@@ -18,7 +18,8 @@ import (
        "fmt"
        "math"
        "path/filepath"
-       "runtime"
+
+       "github.com/gohugoio/hugo/config"
 
        _errors "github.com/pkg/errors"
 
@@ -73,10 +74,7 @@ func (s *siteContentProcessor) processAsset(asset pathLangFile) {
 }
 
 func newSiteContentProcessor(ctx context.Context, partialBuild bool, s *Site) *siteContentProcessor {
-       numWorkers := 12
-       if n := runtime.NumCPU() * 3; n > numWorkers {
-               numWorkers = n
-       }
+       numWorkers := config.GetNumWorkerMultiplier() * 3
 
        numWorkers = int(math.Ceil(float64(numWorkers) / float64(len(s.h.Sites))))
 

diff --git a/hugolib/pagebundler_capture.go b/hugolib/pagebundler_capture.go
index 17a4b865a4da1a68553782c9364d589c1c9a9dd4..7c01a751dbfe6051ae72044553a704cd00b40364 100644 (file)
--- a/hugolib/pagebundler_capture.go
+++ b/hugolib/pagebundler_capture.go
@@ -19,7 +19,8 @@ import (
        "os"
        "path"
        "path/filepath"
-       "runtime"
+
+       "github.com/gohugoio/hugo/config"
 
        "github.com/gohugoio/hugo/common/loggers"
        _errors "github.com/pkg/errors"
@@ -70,10 +71,7 @@ func newCapturer(
        contentChanges *contentChangeMap,
        filenames ...string) *capturer {
 
-       numWorkers := 4
-       if n := runtime.NumCPU(); n > numWorkers {
-               numWorkers = n
-       }
+       numWorkers := config.GetNumWorkerMultiplier()
 
        // TODO(bep) the "index" vs "_index" check/strings should be moved in one place.
        isBundleHeader := func(filename string) bool {

diff --git a/hugolib/site.go b/hugolib/site.go
index 84ee7823cab4de93b4d05c62426e007dbf2d1cae..616b5b37d075173a9095911fb1a2e750d71130f6 100644 (file)
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -1881,15 +1881,6 @@ func (s *Site) newPage(kind string, sections ...string) *pageState {
        return p
 }
 
-func getGoMaxProcs() int {
-       if gmp := os.Getenv("GOMAXPROCS"); gmp != "" {
-               if p, err := strconv.Atoi(gmp); err == nil {
-                       return p
-               }
-       }
-       return 1
-}
-
 func (s *Site) shouldBuild(p page.Page) bool {
        return shouldBuild(s.BuildFuture, s.BuildExpired,
                s.BuildDrafts, p.Draft(), p.PublishDate(), p.ExpiryDate())

diff --git a/hugolib/site_render.go b/hugolib/site_render.go
index f3df09f094286869d1a724dfaed818f4d4488cb5..1d8b14b0acf1e1eab742f98414aec10bec5d99df 100644 (file)
--- a/hugolib/site_render.go
+++ b/hugolib/site_render.go
@@ -19,6 +19,8 @@ import (
        "strings"
        "sync"
 
+       "github.com/gohugoio/hugo/config"
+
        "github.com/gohugoio/hugo/output"
        "github.com/pkg/errors"
 
@@ -55,7 +57,7 @@ func (s siteRenderContext) renderSingletonPages() bool {
 // TODO(bep np doc
 func (s *Site) renderPages(ctx *siteRenderContext) error {
 
-       numWorkers := getGoMaxProcs() * 4
+       numWorkers := config.GetNumWorkerMultiplier()
 
        results := make(chan error)
        pages := make(chan *pageState, numWorkers) // buffered for performance

diff --git a/main.go b/main.go
index d040121111e637338dbfbb72f57b7cced03ae440..ecb423e60a29f4a5b8625cfd49daf3c22e708c2d 100644 (file)
--- a/main.go
+++ b/main.go
@@ -14,16 +14,12 @@
 package main
 
 import (
-       "runtime"
-
        "os"
 
        "github.com/gohugoio/hugo/commands"
 )
 
 func main() {
-
-       runtime.GOMAXPROCS(runtime.NumCPU())
        resp := commands.Execute(os.Args[1:])
 
        if resp.Err != nil {