Fix CreatePages

This fixes #450.  There are two problems:

1.) We're creating a new goroutine for every page.
2.) We're calling s.Pages = append(s.Pages, page) inside each goroutine.

1 is a problem if in that if you have a ton of pages, that's a ton of goroutines.  It's not really useful to have more than a few goroutines at a time, and lots can actually make your code much slower, and, evidently, crash.

2 is a problem in that append is not thread safe. Sometimes it returns a new slice with a larger capacity, when the original slice isn't large enough.  This can cause problems if two goroutines do this at the same time.

The solution for 1 is to use a limited number of workers (I chose 2*GOMAXPROCS as a nice guess).
The solution for 2 is to serialize access to s.Pages, which I did by doing it in a single goroutine.

diff --git a/hugolib/site.go b/hugolib/site.go
index f849bf6fda23a8fe202da9934092b0554f0c9e28..c0e0bb0758a799495f46fea7701ec70e1a0ba517 100644 (file)
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -19,6 +19,7 @@ import (
        "html/template"
        "io"
        "os"
+       "strconv"
        "strings"
        "sync"
        "time"
@@ -311,60 +312,107 @@ func (s *Site) checkDirectories() (err error) {
        return
 }
 
-func (s *Site) CreatePages() (err error) {
+type pageRenderResult struct {
+       page *Page
+       err  error
+}
+
+func (s *Site) CreatePages() error {
        if s.Source == nil {
                panic(fmt.Sprintf("s.Source not set %s", s.absContentDir()))
        }
        if len(s.Source.Files()) < 1 {
-               return
+               return nil
        }
 
-       var wg sync.WaitGroup
-       for _, fi := range s.Source.Files() {
-               wg.Add(1)
-               go func(file *source.File) (err error) {
-                       defer wg.Done()
+       files := s.Source.Files()
 
-                       page, err := NewPage(file.LogicalName)
-                       if err != nil {
-                               return err
-                       }
-                       err = page.ReadFrom(file.Contents)
-                       if err != nil {
-                               return err
-                       }
-                       page.Site = &s.Info
-                       page.Tmpl = s.Tmpl
-                       page.Section = file.Section
-                       page.Dir = file.Dir
+       results := make(chan pageRenderResult)
+       input := make(chan *source.File)
 
-                       //Handling short codes prior to Conversion to HTML
-                       page.ProcessShortcodes(s.Tmpl)
+       procs := getGoMaxProcs()
 
-                       err = page.Convert()
-                       if err != nil {
-                               return err
-                       }
+       wg := &sync.WaitGroup{}
 
-                       if page.ShouldBuild() {
-                               s.Pages = append(s.Pages, page)
-                       }
+       for i := 0; i < procs*2; i++ {
+               wg.Add(1)
+               go pageRenderer(s, input, results, wg)
+       }
 
-                       if page.IsDraft() {
-                               s.draftCount += 1
-                       }
+       errs := make(chan error)
 
-                       if page.IsFuture() {
-                               s.futureCount += 1
-                       }
+       // we can only have exactly one result collator, since it makes changes that
+       // must be synchronized.
+       go resultCollator(s, results, errs)
 
-                       return
-               }(fi)
+       for _, fi := range files {
+               input <- fi
        }
 
+       close(input)
+
        wg.Wait()
+
+       close(results)
+
+       return <-errs
+}
+
+func pageRenderer(s *Site, input <-chan *source.File, results chan<- pageRenderResult, wg *sync.WaitGroup) {
+       for file := range input {
+               page, err := NewPage(file.LogicalName)
+               if err != nil {
+                       results <- pageRenderResult{nil, err}
+                       continue
+               }
+               err = page.ReadFrom(file.Contents)
+               if err != nil {
+                       results <- pageRenderResult{nil, err}
+                       continue
+               }
+               page.Site = &s.Info
+               page.Tmpl = s.Tmpl
+               page.Section = file.Section
+               page.Dir = file.Dir
+
+               //Handling short codes prior to Conversion to HTML
+               page.ProcessShortcodes(s.Tmpl)
+
+               err = page.Convert()
+               if err != nil {
+                       results <- pageRenderResult{nil, err}
+                       continue
+               }
+               results <- pageRenderResult{page, nil}
+       }
+       wg.Done()
+}
+
+func resultCollator(s *Site, results <-chan pageRenderResult, errs chan<- error) {
+       errMsgs := []string{}
+       for r := range results {
+               if r.err != nil {
+                       errMsgs = append(errMsgs, r.err.Error())
+                       continue
+               }
+
+               if r.page.ShouldBuild() {
+                       s.Pages = append(s.Pages, r.page)
+               }
+
+               if r.page.IsDraft() {
+                       s.draftCount += 1
+               }
+
+               if r.page.IsFuture() {
+                       s.futureCount += 1
+               }
+       }
        s.Pages.Sort()
-       return
+       if len(errMsgs) == 0 {
+               errs <- nil
+       }
+       errs <- fmt.Errorf("Errors rendering pages: %s", strings.Join(errMsgs, "\n"))
 }
 
 func (s *Site) BuildSiteMeta() (err error) {
@@ -1010,3 +1058,12 @@ func (s *Site) futureStats() string {
 
        return "0 of " + msg
 }
+
+func getGoMaxProcs() int {
+       if gmp := os.Getenv("GOMAXPROCS"); gmp != "" {
+               if p, err := strconv.Atoi(gmp); err != nil {
+                       return p
+               }
+       }
+       return 1
+}