From: Bjørn Erik Pedersen Date: Tue, 23 Oct 2018 20:21:21 +0000 (+0200) Subject: hugolib: Rename some page_* files X-Git-Tag: v0.50~31 X-Git-Url: http://git.maquefel.me/?a=commitdiff_plain;h=e3ed4a83b8e92ce9bf070f7b41780798b006e848;p=brevno-suite%2Fhugo hugolib: Rename some page_* files To make it easier to see/work with the source files that is about the `Page` struct. --- diff --git a/hugolib/page_bundler.go b/hugolib/page_bundler.go deleted file mode 100644 index 62ef2b52..00000000 --- a/hugolib/page_bundler.go +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "context" - "fmt" - "math" - "runtime" - - _errors "github.com/pkg/errors" - - "golang.org/x/sync/errgroup" -) - -type siteContentProcessor struct { - site *Site - - handleContent contentHandler - - ctx context.Context - - // The input file bundles. - fileBundlesChan chan *bundleDir - - // The input file singles. - fileSinglesChan chan *fileInfo - - // These assets should be just copied to destination. - fileAssetsChan chan []pathLangFile - - numWorkers int - - // The output Pages - pagesChan chan *Page - - // Used for partial rebuilds (aka. live reload) - // Will signal replacement of pages in the site collection. - partialBuild bool -} - -func (s *siteContentProcessor) processBundle(b *bundleDir) { - select { - case s.fileBundlesChan <- b: - case <-s.ctx.Done(): - } -} - -func (s *siteContentProcessor) processSingle(fi *fileInfo) { - select { - case s.fileSinglesChan <- fi: - case <-s.ctx.Done(): - } -} - -func (s *siteContentProcessor) processAssets(assets []pathLangFile) { - select { - case s.fileAssetsChan <- assets: - case <-s.ctx.Done(): - } -} - -func newSiteContentProcessor(ctx context.Context, partialBuild bool, s *Site) *siteContentProcessor { - numWorkers := 12 - if n := runtime.NumCPU() * 3; n > numWorkers { - numWorkers = n - } - - numWorkers = int(math.Ceil(float64(numWorkers) / float64(len(s.owner.Sites)))) - - return &siteContentProcessor{ - ctx: ctx, - partialBuild: partialBuild, - site: s, - handleContent: newHandlerChain(s), - fileBundlesChan: make(chan *bundleDir, numWorkers), - fileSinglesChan: make(chan *fileInfo, numWorkers), - fileAssetsChan: make(chan []pathLangFile, numWorkers), - numWorkers: numWorkers, - pagesChan: make(chan *Page, numWorkers), - } -} - -func (s *siteContentProcessor) closeInput() { - close(s.fileSinglesChan) - close(s.fileBundlesChan) - close(s.fileAssetsChan) -} - -func (s *siteContentProcessor) process(ctx context.Context) error { - g1, ctx := errgroup.WithContext(ctx) - g2, ctx := errgroup.WithContext(ctx) - - // There can be only one of these per site. - g1.Go(func() error { - for p := range s.pagesChan { - if p.s != s.site { - panic(fmt.Sprintf("invalid page site: %v vs %v", p.s, s)) - } - - if s.partialBuild { - p.forceRender = true - s.site.replacePage(p) - } else { - s.site.addPage(p) - } - } - return nil - }) - - for i := 0; i < s.numWorkers; i++ { - g2.Go(func() error { - for { - select { - case f, ok := <-s.fileSinglesChan: - if !ok { - return nil - } - err := s.readAndConvertContentFile(f) - if err != nil { - return err - } - case <-ctx.Done(): - return ctx.Err() - } - } - }) - - g2.Go(func() error { - for { - select { - case files, ok := <-s.fileAssetsChan: - if !ok { - return nil - } - for _, file := range files { - f, err := s.site.BaseFs.Content.Fs.Open(file.Filename()) - if err != nil { - return _errors.Wrap(err, "failed to open assets file") - } - err = s.site.publish(&s.site.PathSpec.ProcessingStats.Files, file.Path(), f) - f.Close() - if err != nil { - return err - } - } - - case <-ctx.Done(): - return ctx.Err() - } - } - }) - - g2.Go(func() error { - for { - select { - case bundle, ok := <-s.fileBundlesChan: - if !ok { - return nil - } - err := s.readAndConvertContentBundle(bundle) - if err != nil { - return err - } - case <-ctx.Done(): - return ctx.Err() - } - } - }) - } - - err := g2.Wait() - - close(s.pagesChan) - - if err != nil { - return err - } - - if err := g1.Wait(); err != nil { - return err - } - - s.site.rawAllPages.sort() - - return nil - -} - -func (s *siteContentProcessor) readAndConvertContentFile(file *fileInfo) error { - ctx := &handlerContext{source: file, pages: s.pagesChan} - return s.handleContent(ctx).err -} - -func (s *siteContentProcessor) readAndConvertContentBundle(bundle *bundleDir) error { - ctx := &handlerContext{bundle: bundle, pages: s.pagesChan} - return s.handleContent(ctx).err -} diff --git a/hugolib/page_bundler_capture.go b/hugolib/page_bundler_capture.go deleted file mode 100644 index c152262c..00000000 --- a/hugolib/page_bundler_capture.go +++ /dev/null @@ -1,775 +0,0 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "errors" - "fmt" - "os" - "path" - "path/filepath" - "runtime" - - "github.com/gohugoio/hugo/common/loggers" - _errors "github.com/pkg/errors" - - "sort" - "strings" - "sync" - - "github.com/spf13/afero" - - "github.com/gohugoio/hugo/hugofs" - - "github.com/gohugoio/hugo/helpers" - - "golang.org/x/sync/errgroup" - - "github.com/gohugoio/hugo/source" -) - -var errSkipCyclicDir = errors.New("skip potential cyclic dir") - -type capturer struct { - // To prevent symbolic link cycles: Visit same folder only once. - seen map[string]bool - seenMu sync.Mutex - - handler captureResultHandler - - sourceSpec *source.SourceSpec - fs afero.Fs - logger *loggers.Logger - - // Filenames limits the content to process to a list of filenames/directories. - // This is used for partial building in server mode. - filenames []string - - // Used to determine how to handle content changes in server mode. - contentChanges *contentChangeMap - - // Semaphore used to throttle the concurrent sub directory handling. - sem chan bool -} - -func newCapturer( - logger *loggers.Logger, - sourceSpec *source.SourceSpec, - handler captureResultHandler, - contentChanges *contentChangeMap, - filenames ...string) *capturer { - - numWorkers := 4 - if n := runtime.NumCPU(); n > numWorkers { - numWorkers = n - } - - // TODO(bep) the "index" vs "_index" check/strings should be moved in one place. - isBundleHeader := func(filename string) bool { - base := filepath.Base(filename) - name := helpers.Filename(base) - return IsContentFile(base) && (name == "index" || name == "_index") - } - - // Make sure that any bundle header files are processed before the others. This makes - // sure that any bundle head is processed before its resources. - sort.Slice(filenames, func(i, j int) bool { - a, b := filenames[i], filenames[j] - ac, bc := isBundleHeader(a), isBundleHeader(b) - - if ac { - return true - } - - if bc { - return false - } - - return a < b - }) - - c := &capturer{ - sem: make(chan bool, numWorkers), - handler: handler, - sourceSpec: sourceSpec, - fs: sourceSpec.SourceFs, - logger: logger, - contentChanges: contentChanges, - seen: make(map[string]bool), - filenames: filenames} - - return c -} - -// Captured files and bundles ready to be processed will be passed on to -// these channels. -type captureResultHandler interface { - handleSingles(fis ...*fileInfo) - handleCopyFiles(fis ...pathLangFile) - captureBundlesHandler -} - -type captureBundlesHandler interface { - handleBundles(b *bundleDirs) -} - -type captureResultHandlerChain struct { - handlers []captureBundlesHandler -} - -func (c *captureResultHandlerChain) handleSingles(fis ...*fileInfo) { - for _, h := range c.handlers { - if hh, ok := h.(captureResultHandler); ok { - hh.handleSingles(fis...) - } - } -} -func (c *captureResultHandlerChain) handleBundles(b *bundleDirs) { - for _, h := range c.handlers { - h.handleBundles(b) - } -} - -func (c *captureResultHandlerChain) handleCopyFiles(files ...pathLangFile) { - for _, h := range c.handlers { - if hh, ok := h.(captureResultHandler); ok { - hh.handleCopyFiles(files...) - } - } -} - -func (c *capturer) capturePartial(filenames ...string) error { - handled := make(map[string]bool) - - for _, filename := range filenames { - dir, resolvedFilename, tp := c.contentChanges.resolveAndRemove(filename) - if handled[resolvedFilename] { - continue - } - - handled[resolvedFilename] = true - - switch tp { - case bundleLeaf: - if err := c.handleDir(resolvedFilename); err != nil { - // Directory may have been deleted. - if !os.IsNotExist(err) { - return err - } - } - case bundleBranch: - if err := c.handleBranchDir(resolvedFilename); err != nil { - // Directory may have been deleted. - if !os.IsNotExist(err) { - return err - } - } - default: - fi, err := c.resolveRealPath(resolvedFilename) - if os.IsNotExist(err) { - // File has been deleted. - continue - } - - // Just in case the owning dir is a new symlink -- this will - // create the proper mapping for it. - c.resolveRealPath(dir) - - f, active := c.newFileInfo(fi, tp) - if active { - c.copyOrHandleSingle(f) - } - } - } - - return nil -} - -func (c *capturer) capture() error { - if len(c.filenames) > 0 { - return c.capturePartial(c.filenames...) - } - - err := c.handleDir(helpers.FilePathSeparator) - if err != nil { - return err - } - - return nil -} - -func (c *capturer) handleNestedDir(dirname string) error { - select { - case c.sem <- true: - var g errgroup.Group - - g.Go(func() error { - defer func() { - <-c.sem - }() - return c.handleDir(dirname) - }) - return g.Wait() - default: - // For deeply nested file trees, waiting for a semaphore wil deadlock. - return c.handleDir(dirname) - } -} - -// This handles a bundle branch and its resources only. This is used -// in server mode on changes. If this dir does not (anymore) represent a bundle -// branch, the handling is upgraded to the full handleDir method. -func (c *capturer) handleBranchDir(dirname string) error { - files, err := c.readDir(dirname) - if err != nil { - - return err - } - - var ( - dirType bundleDirType - ) - - for _, fi := range files { - if !fi.IsDir() { - tp, _ := classifyBundledFile(fi.RealName()) - if dirType == bundleNot { - dirType = tp - } - - if dirType == bundleLeaf { - return c.handleDir(dirname) - } - } - } - - if dirType != bundleBranch { - return c.handleDir(dirname) - } - - dirs := newBundleDirs(bundleBranch, c) - - var secondPass []*fileInfo - - // Handle potential bundle headers first. - for _, fi := range files { - if fi.IsDir() { - continue - } - - tp, isContent := classifyBundledFile(fi.RealName()) - - f, active := c.newFileInfo(fi, tp) - - if !active { - continue - } - - if !f.isOwner() { - if !isContent { - // This is a partial update -- we only care about the files that - // is in this bundle. - secondPass = append(secondPass, f) - } - continue - } - dirs.addBundleHeader(f) - } - - for _, f := range secondPass { - dirs.addBundleFiles(f) - } - - c.handler.handleBundles(dirs) - - return nil - -} - -func (c *capturer) handleDir(dirname string) error { - - files, err := c.readDir(dirname) - if err != nil { - return err - } - - type dirState int - - const ( - dirStateDefault dirState = iota - - dirStateAssetsOnly - dirStateSinglesOnly - ) - - var ( - fileBundleTypes = make([]bundleDirType, len(files)) - - // Start with the assumption that this dir contains only non-content assets (images etc.) - // If that is still true after we had a first look at the list of files, we - // can just copy the files to destination. We will still have to look at the - // sub-folders for potential bundles. - state = dirStateAssetsOnly - - // Start with the assumption that this dir is not a bundle. - // A directory is a bundle if it contains a index content file, - // e.g. index.md (a leaf bundle) or a _index.md (a branch bundle). - bundleType = bundleNot - ) - - /* First check for any content files. - - If there are none, then this is a assets folder only (images etc.) - and we can just plainly copy them to - destination. - - If this is a section with no image etc. or similar, we can just handle it - as it was a single content file. - */ - var hasNonContent, isBranch bool - - for i, fi := range files { - if !fi.IsDir() { - tp, isContent := classifyBundledFile(fi.RealName()) - - fileBundleTypes[i] = tp - if !isBranch { - isBranch = tp == bundleBranch - } - - if isContent { - // This is not a assets-only folder. - state = dirStateDefault - } else { - hasNonContent = true - } - } - } - - if isBranch && !hasNonContent { - // This is a section or similar with no need for any bundle handling. - state = dirStateSinglesOnly - } - - if state > dirStateDefault { - return c.handleNonBundle(dirname, files, state == dirStateSinglesOnly) - } - - var fileInfos = make([]*fileInfo, 0, len(files)) - - for i, fi := range files { - - currentType := bundleNot - - if !fi.IsDir() { - currentType = fileBundleTypes[i] - if bundleType == bundleNot && currentType != bundleNot { - bundleType = currentType - } - } - - if bundleType == bundleNot && currentType != bundleNot { - bundleType = currentType - } - - f, active := c.newFileInfo(fi, currentType) - - if !active { - continue - } - - fileInfos = append(fileInfos, f) - } - - var todo []*fileInfo - - if bundleType != bundleLeaf { - for _, fi := range fileInfos { - if fi.FileInfo().IsDir() { - // Handle potential nested bundles. - if err := c.handleNestedDir(fi.Path()); err != nil { - return err - } - } else if bundleType == bundleNot || (!fi.isOwner() && fi.isContentFile()) { - // Not in a bundle. - c.copyOrHandleSingle(fi) - } else { - // This is a section folder or similar with non-content files in it. - todo = append(todo, fi) - } - } - } else { - todo = fileInfos - } - - if len(todo) == 0 { - return nil - } - - dirs, err := c.createBundleDirs(todo, bundleType) - if err != nil { - return err - } - - // Send the bundle to the next step in the processor chain. - c.handler.handleBundles(dirs) - - return nil -} - -func (c *capturer) handleNonBundle( - dirname string, - fileInfos pathLangFileFis, - singlesOnly bool) error { - - for _, fi := range fileInfos { - if fi.IsDir() { - if err := c.handleNestedDir(fi.Filename()); err != nil { - return err - } - } else { - if singlesOnly { - f, active := c.newFileInfo(fi, bundleNot) - if !active { - continue - } - c.handler.handleSingles(f) - } else { - c.handler.handleCopyFiles(fi) - } - } - } - - return nil -} - -func (c *capturer) copyOrHandleSingle(fi *fileInfo) { - if fi.isContentFile() { - c.handler.handleSingles(fi) - } else { - // These do not currently need any further processing. - c.handler.handleCopyFiles(fi) - } -} - -func (c *capturer) createBundleDirs(fileInfos []*fileInfo, bundleType bundleDirType) (*bundleDirs, error) { - dirs := newBundleDirs(bundleType, c) - - for _, fi := range fileInfos { - if fi.FileInfo().IsDir() { - var collector func(fis ...*fileInfo) - - if bundleType == bundleBranch { - // All files in the current directory are part of this bundle. - // Trying to include sub folders in these bundles are filled with ambiguity. - collector = func(fis ...*fileInfo) { - for _, fi := range fis { - c.copyOrHandleSingle(fi) - } - } - } else { - // All nested files and directories are part of this bundle. - collector = func(fis ...*fileInfo) { - fileInfos = append(fileInfos, fis...) - } - } - err := c.collectFiles(fi.Path(), collector) - if err != nil { - return nil, err - } - - } else if fi.isOwner() { - // There can be more than one language, so: - // 1. Content files must be attached to its language's bundle. - // 2. Other files must be attached to all languages. - // 3. Every content file needs a bundle header. - dirs.addBundleHeader(fi) - } - } - - for _, fi := range fileInfos { - if fi.FileInfo().IsDir() || fi.isOwner() { - continue - } - - if fi.isContentFile() { - if bundleType != bundleBranch { - dirs.addBundleContentFile(fi) - } - } else { - dirs.addBundleFiles(fi) - } - } - - return dirs, nil -} - -func (c *capturer) collectFiles(dirname string, handleFiles func(fis ...*fileInfo)) error { - - filesInDir, err := c.readDir(dirname) - if err != nil { - return err - } - - for _, fi := range filesInDir { - if fi.IsDir() { - err := c.collectFiles(fi.Filename(), handleFiles) - if err != nil { - return err - } - } else { - f, active := c.newFileInfo(fi, bundleNot) - if active { - handleFiles(f) - } - } - } - - return nil -} - -func (c *capturer) readDir(dirname string) (pathLangFileFis, error) { - if c.sourceSpec.IgnoreFile(dirname) { - return nil, nil - } - - dir, err := c.fs.Open(dirname) - if err != nil { - return nil, err - } - defer dir.Close() - fis, err := dir.Readdir(-1) - if err != nil { - return nil, err - } - - pfis := make(pathLangFileFis, 0, len(fis)) - - for _, fi := range fis { - fip := fi.(pathLangFileFi) - - if !c.sourceSpec.IgnoreFile(fip.Filename()) { - - err := c.resolveRealPathIn(fip) - - if err != nil { - // It may have been deleted in the meantime. - if err == errSkipCyclicDir || os.IsNotExist(err) { - continue - } - return nil, err - } - - pfis = append(pfis, fip) - } - } - - return pfis, nil -} - -func (c *capturer) newFileInfo(fi pathLangFileFi, tp bundleDirType) (*fileInfo, bool) { - f := newFileInfo(c.sourceSpec, "", "", fi, tp) - return f, !f.disabled -} - -type pathLangFile interface { - hugofs.LanguageAnnouncer - hugofs.FilePather -} - -type pathLangFileFi interface { - os.FileInfo - pathLangFile -} - -type pathLangFileFis []pathLangFileFi - -type bundleDirs struct { - tp bundleDirType - // Maps languages to bundles. - bundles map[string]*bundleDir - - // Keeps track of language overrides for non-content files, e.g. logo.en.png. - langOverrides map[string]bool - - c *capturer -} - -func newBundleDirs(tp bundleDirType, c *capturer) *bundleDirs { - return &bundleDirs{tp: tp, bundles: make(map[string]*bundleDir), langOverrides: make(map[string]bool), c: c} -} - -type bundleDir struct { - tp bundleDirType - fi *fileInfo - - resources map[string]*fileInfo -} - -func (b bundleDir) clone() *bundleDir { - b.resources = make(map[string]*fileInfo) - fic := *b.fi - b.fi = &fic - return &b -} - -func newBundleDir(fi *fileInfo, bundleType bundleDirType) *bundleDir { - return &bundleDir{fi: fi, tp: bundleType, resources: make(map[string]*fileInfo)} -} - -func (b *bundleDirs) addBundleContentFile(fi *fileInfo) { - dir, found := b.bundles[fi.Lang()] - if !found { - // Every bundled content file needs a bundle header. - // If one does not exist in its language, we pick the default - // language version, or a random one if that doesn't exist, either. - tl := b.c.sourceSpec.DefaultContentLanguage - ldir, found := b.bundles[tl] - if !found { - // Just pick one. - for _, v := range b.bundles { - ldir = v - break - } - } - - if ldir == nil { - panic(fmt.Sprintf("bundle not found for file %q", fi.Filename())) - } - - dir = ldir.clone() - dir.fi.overriddenLang = fi.Lang() - b.bundles[fi.Lang()] = dir - } - - dir.resources[fi.Path()] = fi -} - -func (b *bundleDirs) addBundleFiles(fi *fileInfo) { - dir := filepath.ToSlash(fi.Dir()) - p := dir + fi.TranslationBaseName() + "." + fi.Ext() - for lang, bdir := range b.bundles { - key := path.Join(lang, p) - - // Given mypage.de.md (German translation) and mypage.md we pick the most - // specific for that language. - if fi.Lang() == lang || !b.langOverrides[key] { - bdir.resources[key] = fi - } - b.langOverrides[key] = true - } -} - -func (b *bundleDirs) addBundleHeader(fi *fileInfo) { - b.bundles[fi.Lang()] = newBundleDir(fi, b.tp) -} - -func (c *capturer) isSeen(dirname string) bool { - c.seenMu.Lock() - defer c.seenMu.Unlock() - seen := c.seen[dirname] - c.seen[dirname] = true - if seen { - c.logger.WARN.Printf("Content dir %q already processed; skipped to avoid infinite recursion.", dirname) - return true - - } - return false -} - -func (c *capturer) resolveRealPath(path string) (pathLangFileFi, error) { - fileInfo, err := c.lstatIfPossible(path) - if err != nil { - return nil, err - } - return fileInfo, c.resolveRealPathIn(fileInfo) -} - -func (c *capturer) resolveRealPathIn(fileInfo pathLangFileFi) error { - - basePath := fileInfo.BaseDir() - path := fileInfo.Filename() - - realPath := path - - if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink { - link, err := filepath.EvalSymlinks(path) - if err != nil { - return _errors.Wrapf(err, "Cannot read symbolic link %q, error was:", path) - } - - // This is a file on the outside of any base fs, so we have to use the os package. - sfi, err := os.Stat(link) - if err != nil { - return _errors.Wrapf(err, "Cannot stat %q, error was:", link) - } - - // TODO(bep) improve all of this. - if a, ok := fileInfo.(*hugofs.LanguageFileInfo); ok { - a.FileInfo = sfi - } - - realPath = link - - if realPath != path && sfi.IsDir() && c.isSeen(realPath) { - // Avoid cyclic symlinks. - // Note that this may prevent some uses that isn't cyclic and also - // potential useful, but this implementation is both robust and simple: - // We stop at the first directory that we have seen before, e.g. - // /content/blog will only be processed once. - return errSkipCyclicDir - } - - if c.contentChanges != nil { - // Keep track of symbolic links in watch mode. - var from, to string - if sfi.IsDir() { - from = realPath - to = path - - if !strings.HasSuffix(to, helpers.FilePathSeparator) { - to = to + helpers.FilePathSeparator - } - if !strings.HasSuffix(from, helpers.FilePathSeparator) { - from = from + helpers.FilePathSeparator - } - - if !strings.HasSuffix(basePath, helpers.FilePathSeparator) { - basePath = basePath + helpers.FilePathSeparator - } - - if strings.HasPrefix(from, basePath) { - // With symbolic links inside /content we need to keep - // a reference to both. This may be confusing with --navigateToChanged - // but the user has chosen this him or herself. - c.contentChanges.addSymbolicLinkMapping(from, from) - } - - } else { - from = realPath - to = path - } - - c.contentChanges.addSymbolicLinkMapping(from, to) - } - } - - return nil -} - -func (c *capturer) lstatIfPossible(path string) (pathLangFileFi, error) { - fi, err := helpers.LstatIfPossible(c.fs, path) - if err != nil { - return nil, err - } - return fi.(pathLangFileFi), nil -} diff --git a/hugolib/page_bundler_capture_test.go b/hugolib/page_bundler_capture_test.go deleted file mode 100644 index d6128352..00000000 --- a/hugolib/page_bundler_capture_test.go +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "fmt" - "os" - "path" - "path/filepath" - "sort" - - "github.com/gohugoio/hugo/common/loggers" - - "runtime" - "strings" - "sync" - "testing" - - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/source" - "github.com/stretchr/testify/require" -) - -type storeFilenames struct { - sync.Mutex - filenames []string - copyNames []string - dirKeys []string -} - -func (s *storeFilenames) handleSingles(fis ...*fileInfo) { - s.Lock() - defer s.Unlock() - for _, fi := range fis { - s.filenames = append(s.filenames, filepath.ToSlash(fi.Filename())) - } -} - -func (s *storeFilenames) handleBundles(d *bundleDirs) { - s.Lock() - defer s.Unlock() - var keys []string - for _, b := range d.bundles { - res := make([]string, len(b.resources)) - i := 0 - for _, r := range b.resources { - res[i] = path.Join(r.Lang(), filepath.ToSlash(r.Filename())) - i++ - } - sort.Strings(res) - keys = append(keys, path.Join("__bundle", b.fi.Lang(), filepath.ToSlash(b.fi.Filename()), "resources", strings.Join(res, "|"))) - } - s.dirKeys = append(s.dirKeys, keys...) -} - -func (s *storeFilenames) handleCopyFiles(files ...pathLangFile) { - s.Lock() - defer s.Unlock() - for _, file := range files { - s.copyNames = append(s.copyNames, filepath.ToSlash(file.Filename())) - } -} - -func (s *storeFilenames) sortedStr() string { - s.Lock() - defer s.Unlock() - sort.Strings(s.filenames) - sort.Strings(s.dirKeys) - sort.Strings(s.copyNames) - return "\nF:\n" + strings.Join(s.filenames, "\n") + "\nD:\n" + strings.Join(s.dirKeys, "\n") + - "\nC:\n" + strings.Join(s.copyNames, "\n") + "\n" -} - -func TestPageBundlerCaptureSymlinks(t *testing.T) { - if runtime.GOOS == "windows" && os.Getenv("CI") == "" { - t.Skip("Skip TestPageBundlerCaptureSymlinks as os.Symlink needs administrator rights on Windows") - } - - assert := require.New(t) - ps, clean, workDir := newTestBundleSymbolicSources(t) - sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) - defer clean() - - fileStore := &storeFilenames{} - logger := loggers.NewErrorLogger() - c := newCapturer(logger, sourceSpec, fileStore, nil) - - assert.NoError(c.capture()) - - expected := ` -F: -/base/a/page_s.md -/base/a/regular.md -/base/symbolic1/s1.md -/base/symbolic1/s2.md -/base/symbolic3/circus/a/page_s.md -/base/symbolic3/circus/a/regular.md -D: -__bundle/en/base/symbolic2/a1/index.md/resources/en/base/symbolic2/a1/logo.png|en/base/symbolic2/a1/page.md -C: -/base/symbolic3/s1.png -/base/symbolic3/s2.png -` - - got := strings.Replace(fileStore.sortedStr(), filepath.ToSlash(workDir), "", -1) - got = strings.Replace(got, "//", "/", -1) - - if expected != got { - diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got)) - t.Log(got) - t.Fatalf("Failed:\n%s", diff) - } -} - -func TestPageBundlerCaptureBasic(t *testing.T) { - t.Parallel() - - assert := require.New(t) - fs, cfg := newTestBundleSources(t) - assert.NoError(loadDefaultSettingsFor(cfg)) - assert.NoError(loadLanguageSettings(cfg, nil)) - ps, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) - - sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) - - fileStore := &storeFilenames{} - - c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil) - - assert.NoError(c.capture()) - - expected := ` -F: -/work/base/_1.md -/work/base/a/1.md -/work/base/a/2.md -/work/base/assets/pages/mypage.md -D: -__bundle/en/work/base/_index.md/resources/en/work/base/_1.png -__bundle/en/work/base/a/b/index.md/resources/en/work/base/a/b/ab1.md -__bundle/en/work/base/b/my-bundle/index.md/resources/en/work/base/b/my-bundle/1.md|en/work/base/b/my-bundle/2.md|en/work/base/b/my-bundle/c/logo.png|en/work/base/b/my-bundle/custom-mime.bep|en/work/base/b/my-bundle/sunset1.jpg|en/work/base/b/my-bundle/sunset2.jpg -__bundle/en/work/base/c/bundle/index.md/resources/en/work/base/c/bundle/logo-은행.png -__bundle/en/work/base/root/index.md/resources/en/work/base/root/1.md|en/work/base/root/c/logo.png -C: -/work/base/assets/pic1.png -/work/base/assets/pic2.png -/work/base/images/hugo-logo.png -` - - got := fileStore.sortedStr() - - if expected != got { - diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got)) - t.Log(got) - t.Fatalf("Failed:\n%s", diff) - } -} - -func TestPageBundlerCaptureMultilingual(t *testing.T) { - t.Parallel() - - assert := require.New(t) - fs, cfg := newTestBundleSourcesMultilingual(t) - assert.NoError(loadDefaultSettingsFor(cfg)) - assert.NoError(loadLanguageSettings(cfg, nil)) - - ps, err := helpers.NewPathSpec(fs, cfg) - assert.NoError(err) - - sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) - fileStore := &storeFilenames{} - c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil) - - assert.NoError(c.capture()) - - expected := ` -F: -/work/base/1s/mypage.md -/work/base/1s/mypage.nn.md -/work/base/bb/_1.md -/work/base/bb/_1.nn.md -/work/base/bb/en.md -/work/base/bc/page.md -/work/base/bc/page.nn.md -/work/base/be/_index.md -/work/base/be/page.md -/work/base/be/page.nn.md -D: -__bundle/en/work/base/bb/_index.md/resources/en/work/base/bb/a.png|en/work/base/bb/b.png|nn/work/base/bb/c.nn.png -__bundle/en/work/base/bc/_index.md/resources/en/work/base/bc/logo-bc.png -__bundle/en/work/base/bd/index.md/resources/en/work/base/bd/page.md -__bundle/en/work/base/bf/my-bf-bundle/index.md/resources/en/work/base/bf/my-bf-bundle/page.md -__bundle/en/work/base/lb/index.md/resources/en/work/base/lb/1.md|en/work/base/lb/2.md|en/work/base/lb/c/d/deep.png|en/work/base/lb/c/logo.png|en/work/base/lb/c/one.png|en/work/base/lb/c/page.md -__bundle/nn/work/base/bb/_index.nn.md/resources/en/work/base/bb/a.png|nn/work/base/bb/b.nn.png|nn/work/base/bb/c.nn.png -__bundle/nn/work/base/bd/index.md/resources/nn/work/base/bd/page.nn.md -__bundle/nn/work/base/bf/my-bf-bundle/index.nn.md/resources -__bundle/nn/work/base/lb/index.nn.md/resources/en/work/base/lb/c/d/deep.png|en/work/base/lb/c/one.png|nn/work/base/lb/2.nn.md|nn/work/base/lb/c/logo.nn.png -C: -/work/base/1s/mylogo.png -/work/base/bb/b/d.nn.png -` - - got := fileStore.sortedStr() - - if expected != got { - diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got)) - t.Log(got) - t.Fatalf("Failed:\n%s", strings.Join(diff, "\n")) - } - -} - -type noOpFileStore int - -func (noOpFileStore) handleSingles(fis ...*fileInfo) {} -func (noOpFileStore) handleBundles(b *bundleDirs) {} -func (noOpFileStore) handleCopyFiles(files ...pathLangFile) {} - -func BenchmarkPageBundlerCapture(b *testing.B) { - capturers := make([]*capturer, b.N) - - for i := 0; i < b.N; i++ { - cfg, fs := newTestCfg() - ps, _ := helpers.NewPathSpec(fs, cfg) - sourceSpec := source.NewSourceSpec(ps, fs.Source) - - base := fmt.Sprintf("base%d", i) - for j := 1; j <= 5; j++ { - js := fmt.Sprintf("j%d", j) - writeSource(b, fs, filepath.Join(base, js, "index.md"), "content") - writeSource(b, fs, filepath.Join(base, js, "logo1.png"), "content") - writeSource(b, fs, filepath.Join(base, js, "sub", "logo2.png"), "content") - writeSource(b, fs, filepath.Join(base, js, "section", "_index.md"), "content") - writeSource(b, fs, filepath.Join(base, js, "section", "logo.png"), "content") - writeSource(b, fs, filepath.Join(base, js, "section", "sub", "logo.png"), "content") - - for k := 1; k <= 5; k++ { - ks := fmt.Sprintf("k%d", k) - writeSource(b, fs, filepath.Join(base, js, ks, "logo1.png"), "content") - writeSource(b, fs, filepath.Join(base, js, "section", ks, "logo.png"), "content") - } - } - - for i := 1; i <= 5; i++ { - writeSource(b, fs, filepath.Join(base, "assetsonly", fmt.Sprintf("image%d.png", i)), "image") - } - - for i := 1; i <= 5; i++ { - writeSource(b, fs, filepath.Join(base, "contentonly", fmt.Sprintf("c%d.md", i)), "content") - } - - capturers[i] = newCapturer(loggers.NewErrorLogger(), sourceSpec, new(noOpFileStore), nil, base) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - err := capturers[i].capture() - if err != nil { - b.Fatal(err) - } - } -} diff --git a/hugolib/page_bundler_handlers.go b/hugolib/page_bundler_handlers.go deleted file mode 100644 index 2ab0ebaf..00000000 --- a/hugolib/page_bundler_handlers.go +++ /dev/null @@ -1,345 +0,0 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "errors" - "fmt" - "path/filepath" - "sort" - - "strings" - - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/resource" -) - -var ( - // This should be the only list of valid extensions for content files. - contentFileExtensions = []string{ - "html", "htm", - "mdown", "markdown", "md", - "asciidoc", "adoc", "ad", - "rest", "rst", - "mmark", - "org", - "pandoc", "pdc"} - - contentFileExtensionsSet map[string]bool -) - -func init() { - contentFileExtensionsSet = make(map[string]bool) - for _, ext := range contentFileExtensions { - contentFileExtensionsSet[ext] = true - } -} - -func newHandlerChain(s *Site) contentHandler { - c := &contentHandlers{s: s} - - contentFlow := c.parsePage(c.processFirstMatch( - // Handles all files with a content file extension. See above. - c.handlePageContent(), - - // Every HTML file without front matter will be passed on to this handler. - c.handleHTMLContent(), - )) - - c.rootHandler = c.processFirstMatch( - contentFlow, - - // Creates a file resource (image, CSS etc.) if there is a parent - // page set on the current context. - c.createResource(), - - // Everything that isn't handled above, will just be copied - // to destination. - c.copyFile(), - ) - - return c.rootHandler - -} - -type contentHandlers struct { - s *Site - rootHandler contentHandler -} - -func (c *contentHandlers) processFirstMatch(handlers ...contentHandler) func(ctx *handlerContext) handlerResult { - return func(ctx *handlerContext) handlerResult { - for _, h := range handlers { - res := h(ctx) - if res.handled || res.err != nil { - return res - } - } - return handlerResult{err: errors.New("no matching handler found")} - } -} - -type handlerContext struct { - // These are the pages stored in Site. - pages chan<- *Page - - doNotAddToSiteCollections bool - - currentPage *Page - parentPage *Page - - bundle *bundleDir - - source *fileInfo - - // Relative path to the target. - target string -} - -func (c *handlerContext) ext() string { - if c.currentPage != nil { - if c.currentPage.Markup != "" { - return c.currentPage.Markup - } - return c.currentPage.Ext() - } - - if c.bundle != nil { - return c.bundle.fi.Ext() - } else { - return c.source.Ext() - } -} - -func (c *handlerContext) targetPath() string { - if c.target != "" { - return c.target - } - - return c.source.Filename() -} - -func (c *handlerContext) file() *fileInfo { - if c.bundle != nil { - return c.bundle.fi - } - - return c.source -} - -// Create a copy with the current context as its parent. -func (c handlerContext) childCtx(fi *fileInfo) *handlerContext { - if c.currentPage == nil { - panic("Need a Page to create a child context") - } - - c.target = strings.TrimPrefix(fi.Path(), c.bundle.fi.Dir()) - c.source = fi - - c.doNotAddToSiteCollections = c.bundle != nil && c.bundle.tp != bundleBranch - - c.bundle = nil - - c.parentPage = c.currentPage - c.currentPage = nil - - return &c -} - -func (c *handlerContext) supports(exts ...string) bool { - ext := c.ext() - for _, s := range exts { - if s == ext { - return true - } - } - - return false -} - -func (c *handlerContext) isContentFile() bool { - return contentFileExtensionsSet[c.ext()] -} - -type ( - handlerResult struct { - err error - handled bool - resource resource.Resource - } - - contentHandler func(ctx *handlerContext) handlerResult -) - -var ( - notHandled handlerResult -) - -func (c *contentHandlers) parsePage(h contentHandler) contentHandler { - return func(ctx *handlerContext) handlerResult { - if !ctx.isContentFile() { - return notHandled - } - - result := handlerResult{handled: true} - fi := ctx.file() - - f, err := fi.Open() - if err != nil { - return handlerResult{err: fmt.Errorf("(%s) failed to open content file: %s", fi.Filename(), err)} - } - defer f.Close() - - p := c.s.newPageFromFile(fi) - - _, err = p.ReadFrom(f) - if err != nil { - return handlerResult{err: err} - } - - if !p.shouldBuild() { - if !ctx.doNotAddToSiteCollections { - ctx.pages <- p - } - return result - } - - ctx.currentPage = p - - if ctx.bundle != nil { - // Add the bundled files - for _, fi := range ctx.bundle.resources { - childCtx := ctx.childCtx(fi) - res := c.rootHandler(childCtx) - if res.err != nil { - return res - } - if res.resource != nil { - if pageResource, ok := res.resource.(*Page); ok { - pageResource.resourcePath = filepath.ToSlash(childCtx.target) - pageResource.parent = p - } - p.Resources = append(p.Resources, res.resource) - } - } - - sort.SliceStable(p.Resources, func(i, j int) bool { - if p.Resources[i].ResourceType() < p.Resources[j].ResourceType() { - return true - } - - p1, ok1 := p.Resources[i].(*Page) - p2, ok2 := p.Resources[j].(*Page) - - if ok1 != ok2 { - return ok2 - } - - if ok1 { - return defaultPageSort(p1, p2) - } - - return p.Resources[i].RelPermalink() < p.Resources[j].RelPermalink() - }) - - // Assign metadata from front matter if set - if len(p.resourcesMetadata) > 0 { - resource.AssignMetadata(p.resourcesMetadata, p.Resources...) - } - - } - - return h(ctx) - } -} - -func (c *contentHandlers) handlePageContent() contentHandler { - return func(ctx *handlerContext) handlerResult { - if ctx.supports("html", "htm") { - return notHandled - } - - p := ctx.currentPage - - if c.s.Cfg.GetBool("enableEmoji") { - p.workContent = helpers.Emojify(p.workContent) - } - - p.workContent = p.renderContent(p.workContent) - - tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.workContent) - p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents) - p.workContent = tmpContent - - if !ctx.doNotAddToSiteCollections { - ctx.pages <- p - } - - return handlerResult{handled: true, resource: p} - } -} - -func (c *contentHandlers) handleHTMLContent() contentHandler { - return func(ctx *handlerContext) handlerResult { - if !ctx.supports("html", "htm") { - return notHandled - } - - p := ctx.currentPage - - if !ctx.doNotAddToSiteCollections { - ctx.pages <- p - } - - return handlerResult{handled: true, resource: p} - } -} - -func (c *contentHandlers) createResource() contentHandler { - return func(ctx *handlerContext) handlerResult { - if ctx.parentPage == nil { - return notHandled - } - - resource, err := c.s.ResourceSpec.New( - resource.ResourceSourceDescriptor{ - TargetPathBuilder: ctx.parentPage.subResourceTargetPathFactory, - SourceFile: ctx.source, - RelTargetFilename: ctx.target, - URLBase: c.s.GetURLLanguageBasePath(), - TargetBasePaths: []string{c.s.GetTargetLanguageBasePath()}, - }) - - return handlerResult{err: err, handled: true, resource: resource} - } -} - -func (c *contentHandlers) copyFile() contentHandler { - return func(ctx *handlerContext) handlerResult { - f, err := c.s.BaseFs.Content.Fs.Open(ctx.source.Filename()) - if err != nil { - err := fmt.Errorf("failed to open file in copyFile: %s", err) - return handlerResult{err: err} - } - - target := ctx.targetPath() - - defer f.Close() - if err := c.s.publish(&c.s.PathSpec.ProcessingStats.Files, target, f); err != nil { - return handlerResult{err: err} - } - - return handlerResult{handled: true} - } -} diff --git a/hugolib/page_bundler_test.go b/hugolib/page_bundler_test.go deleted file mode 100644 index 1eb5aacd..00000000 --- a/hugolib/page_bundler_test.go +++ /dev/null @@ -1,751 +0,0 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "github.com/gohugoio/hugo/common/loggers" - - "os" - "runtime" - "testing" - - "github.com/gohugoio/hugo/helpers" - - "io" - - "github.com/spf13/afero" - - "github.com/gohugoio/hugo/media" - - "path/filepath" - - "fmt" - - "github.com/gohugoio/hugo/deps" - "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/viper" - - "github.com/stretchr/testify/require" -) - -func TestPageBundlerSiteRegular(t *testing.T) { - t.Parallel() - - for _, ugly := range []bool{false, true} { - t.Run(fmt.Sprintf("ugly=%t", ugly), - func(t *testing.T) { - - assert := require.New(t) - fs, cfg := newTestBundleSources(t) - assert.NoError(loadDefaultSettingsFor(cfg)) - assert.NoError(loadLanguageSettings(cfg, nil)) - - cfg.Set("permalinks", map[string]string{ - "a": ":sections/:filename", - "b": ":year/:slug/", - "c": ":sections/:slug", - "": ":filename/", - }) - - cfg.Set("outputFormats", map[string]interface{}{ - "CUSTOMO": map[string]interface{}{ - "mediaType": media.HTMLType, - "baseName": "cindex", - "path": "cpath", - }, - }) - - cfg.Set("outputs", map[string]interface{}{ - "home": []string{"HTML", "CUSTOMO"}, - "page": []string{"HTML", "CUSTOMO"}, - "section": []string{"HTML", "CUSTOMO"}, - }) - - cfg.Set("uglyURLs", ugly) - - s := buildSingleSite(t, deps.DepsCfg{Logger: loggers.NewWarningLogger(), Fs: fs, Cfg: cfg}, BuildCfg{}) - - th := testHelper{s.Cfg, s.Fs, t} - - assert.Len(s.RegularPages, 8) - - singlePage := s.getPage(KindPage, "a/1.md") - assert.Equal("", singlePage.BundleType()) - - assert.NotNil(singlePage) - assert.Equal(singlePage, s.getPage("page", "a/1")) - assert.Equal(singlePage, s.getPage("page", "1")) - - assert.Contains(singlePage.content(), "TheContent") - - if ugly { - assert.Equal("/a/1.html", singlePage.RelPermalink()) - th.assertFileContent(filepath.FromSlash("/work/public/a/1.html"), "TheContent") - - } else { - assert.Equal("/a/1/", singlePage.RelPermalink()) - th.assertFileContent(filepath.FromSlash("/work/public/a/1/index.html"), "TheContent") - } - - th.assertFileContent(filepath.FromSlash("/work/public/images/hugo-logo.png"), "content") - - // This should be just copied to destination. - th.assertFileContent(filepath.FromSlash("/work/public/assets/pic1.png"), "content") - - leafBundle1 := s.getPage(KindPage, "b/my-bundle/index.md") - assert.NotNil(leafBundle1) - assert.Equal("leaf", leafBundle1.BundleType()) - assert.Equal("b", leafBundle1.Section()) - sectionB := s.getPage(KindSection, "b") - assert.NotNil(sectionB) - home, _ := s.Info.Home() - assert.Equal("branch", home.BundleType()) - - // This is a root bundle and should live in the "home section" - // See https://github.com/gohugoio/hugo/issues/4332 - rootBundle := s.getPage(KindPage, "root") - assert.NotNil(rootBundle) - assert.True(rootBundle.Parent().IsHome()) - if ugly { - assert.Equal("/root.html", rootBundle.RelPermalink()) - } else { - assert.Equal("/root/", rootBundle.RelPermalink()) - } - - leafBundle2 := s.getPage(KindPage, "a/b/index.md") - assert.NotNil(leafBundle2) - unicodeBundle := s.getPage(KindPage, "c/bundle/index.md") - assert.NotNil(unicodeBundle) - - pageResources := leafBundle1.Resources.ByType(pageResourceType) - assert.Len(pageResources, 2) - firstPage := pageResources[0].(*Page) - secondPage := pageResources[1].(*Page) - assert.Equal(filepath.FromSlash("/work/base/b/my-bundle/1.md"), firstPage.pathOrTitle(), secondPage.pathOrTitle()) - assert.Contains(firstPage.content(), "TheContent") - assert.Equal(6, len(leafBundle1.Resources)) - - // Verify shortcode in bundled page - assert.Contains(secondPage.content(), filepath.FromSlash("MyShort in b/my-bundle/2.md")) - - // https://github.com/gohugoio/hugo/issues/4582 - assert.Equal(leafBundle1, firstPage.Parent()) - assert.Equal(leafBundle1, secondPage.Parent()) - - assert.Equal(firstPage, pageResources.GetMatch("1*")) - assert.Equal(secondPage, pageResources.GetMatch("2*")) - assert.Nil(pageResources.GetMatch("doesnotexist*")) - - imageResources := leafBundle1.Resources.ByType("image") - assert.Equal(3, len(imageResources)) - image := imageResources[0] - - altFormat := leafBundle1.OutputFormats().Get("CUSTOMO") - assert.NotNil(altFormat) - - assert.Equal("https://example.com/2017/pageslug/c/logo.png", image.Permalink()) - - th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content") - th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/c/logo.png"), "content") - - // Custom media type defined in site config. - assert.Len(leafBundle1.Resources.ByType("bepsays"), 1) - - if ugly { - assert.Equal("/2017/pageslug.html", leafBundle1.RelPermalink()) - th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug.html"), - "TheContent", - "Sunset RelPermalink: /2017/pageslug/sunset1.jpg", - "Thumb Width: 123", - "Thumb Name: my-sunset-1", - "Short Sunset RelPermalink: /2017/pageslug/sunset2.jpg", - "Short Thumb Width: 56", - "1: Image Title: Sunset Galore 1", - "1: Image Params: map[myparam:My Sunny Param]", - "2: Image Title: Sunset Galore 2", - "2: Image Params: map[myparam:My Sunny Param]", - "1: Image myParam: Lower: My Sunny Param Caps: My Sunny Param", - ) - th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug.html"), "TheContent") - - assert.Equal("/a/b.html", leafBundle2.RelPermalink()) - - // 은행 - assert.Equal("/c/%EC%9D%80%ED%96%89.html", unicodeBundle.RelPermalink()) - th.assertFileContent(filepath.FromSlash("/work/public/c/은행.html"), "Content for 은행") - th.assertFileContent(filepath.FromSlash("/work/public/c/은행/logo-은행.png"), "은행 PNG") - - } else { - assert.Equal("/2017/pageslug/", leafBundle1.RelPermalink()) - th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "TheContent") - th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/cindex.html"), "TheContent") - th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "Single Title") - th.assertFileContent(filepath.FromSlash("/work/public/root/index.html"), "Single Title") - - assert.Equal("/a/b/", leafBundle2.RelPermalink()) - - } - - }) - } - -} - -func TestPageBundlerSiteMultilingual(t *testing.T) { - t.Parallel() - - for _, ugly := range []bool{false, true} { - t.Run(fmt.Sprintf("ugly=%t", ugly), - func(t *testing.T) { - - assert := require.New(t) - fs, cfg := newTestBundleSourcesMultilingual(t) - cfg.Set("uglyURLs", ugly) - - assert.NoError(loadDefaultSettingsFor(cfg)) - assert.NoError(loadLanguageSettings(cfg, nil)) - sites, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg}) - assert.NoError(err) - assert.Equal(2, len(sites.Sites)) - - assert.NoError(sites.Build(BuildCfg{})) - - s := sites.Sites[0] - - assert.Equal(8, len(s.RegularPages)) - assert.Equal(16, len(s.Pages)) - assert.Equal(31, len(s.AllPages)) - - bundleWithSubPath := s.getPage(KindPage, "lb/index") - assert.NotNil(bundleWithSubPath) - - // See https://github.com/gohugoio/hugo/issues/4312 - // Before that issue: - // A bundle in a/b/index.en.md - // a/b/index.en.md => OK - // a/b/index => OK - // index.en.md => ambigous, but OK. - // With bundles, the file name has little meaning, the folder it lives in does. So this should also work: - // a/b - // and probably also just b (aka "my-bundle") - // These may also be translated, so we also need to test that. - // "bf", "my-bf-bundle", "index.md + nn - bfBundle := s.getPage(KindPage, "bf/my-bf-bundle/index") - assert.NotNil(bfBundle) - assert.Equal("en", bfBundle.Lang()) - assert.Equal(bfBundle, s.getPage(KindPage, "bf/my-bf-bundle/index.md")) - assert.Equal(bfBundle, s.getPage(KindPage, "bf/my-bf-bundle")) - assert.Equal(bfBundle, s.getPage(KindPage, "my-bf-bundle")) - - nnSite := sites.Sites[1] - assert.Equal(7, len(nnSite.RegularPages)) - - bfBundleNN := nnSite.getPage(KindPage, "bf/my-bf-bundle/index") - assert.NotNil(bfBundleNN) - assert.Equal("nn", bfBundleNN.Lang()) - assert.Equal(bfBundleNN, nnSite.getPage(KindPage, "bf/my-bf-bundle/index.nn.md")) - assert.Equal(bfBundleNN, nnSite.getPage(KindPage, "bf/my-bf-bundle")) - assert.Equal(bfBundleNN, nnSite.getPage(KindPage, "my-bf-bundle")) - - // See https://github.com/gohugoio/hugo/issues/4295 - // Every resource should have its Name prefixed with its base folder. - cBundleResources := bundleWithSubPath.Resources.Match("c/**") - assert.Equal(4, len(cBundleResources)) - bundlePage := bundleWithSubPath.Resources.GetMatch("c/page*") - assert.NotNil(bundlePage) - assert.IsType(&Page{}, bundlePage) - - }) - } -} - -func TestMultilingualDisableDefaultLanguage(t *testing.T) { - t.Parallel() - - assert := require.New(t) - _, cfg := newTestBundleSourcesMultilingual(t) - - cfg.Set("disableLanguages", []string{"en"}) - - err := loadDefaultSettingsFor(cfg) - assert.NoError(err) - err = loadLanguageSettings(cfg, nil) - assert.Error(err) - assert.Contains(err.Error(), "cannot disable default language") -} - -func TestMultilingualDisableLanguage(t *testing.T) { - t.Parallel() - - assert := require.New(t) - fs, cfg := newTestBundleSourcesMultilingual(t) - cfg.Set("disableLanguages", []string{"nn"}) - - assert.NoError(loadDefaultSettingsFor(cfg)) - assert.NoError(loadLanguageSettings(cfg, nil)) - - sites, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg}) - assert.NoError(err) - assert.Equal(1, len(sites.Sites)) - - assert.NoError(sites.Build(BuildCfg{})) - - s := sites.Sites[0] - - assert.Equal(8, len(s.RegularPages)) - assert.Equal(16, len(s.Pages)) - // No nn pages - assert.Equal(16, len(s.AllPages)) - for _, p := range s.rawAllPages { - assert.True(p.Lang() != "nn") - } - for _, p := range s.AllPages { - assert.True(p.Lang() != "nn") - } - -} - -func TestPageBundlerSiteWitSymbolicLinksInContent(t *testing.T) { - if runtime.GOOS == "windows" && os.Getenv("CI") == "" { - t.Skip("Skip TestPageBundlerSiteWitSymbolicLinksInContent as os.Symlink needs administrator rights on Windows") - } - - assert := require.New(t) - ps, clean, workDir := newTestBundleSymbolicSources(t) - defer clean() - - cfg := ps.Cfg - fs := ps.Fs - - s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: loggers.NewErrorLogger()}, BuildCfg{}) - - th := testHelper{s.Cfg, s.Fs, t} - - assert.Equal(7, len(s.RegularPages)) - a1Bundle := s.getPage(KindPage, "symbolic2/a1/index.md") - assert.NotNil(a1Bundle) - assert.Equal(2, len(a1Bundle.Resources)) - assert.Equal(1, len(a1Bundle.Resources.ByType(pageResourceType))) - - th.assertFileContent(filepath.FromSlash(workDir+"/public/a/page/index.html"), "TheContent") - th.assertFileContent(filepath.FromSlash(workDir+"/public/symbolic1/s1/index.html"), "TheContent") - th.assertFileContent(filepath.FromSlash(workDir+"/public/symbolic2/a1/index.html"), "TheContent") - -} - -func TestPageBundlerHeadless(t *testing.T) { - t.Parallel() - - cfg, fs := newTestCfg() - assert := require.New(t) - - workDir := "/work" - cfg.Set("workingDir", workDir) - cfg.Set("contentDir", "base") - cfg.Set("baseURL", "https://example.com") - - pageContent := `--- -title: "Bundle Galore" -slug: s1 -date: 2017-01-23 ---- - -TheContent. - -{{< myShort >}} -` - - writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "single.html"), "single {{ .Content }}") - writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "list.html"), "list") - writeSource(t, fs, filepath.Join(workDir, "layouts", "shortcodes", "myShort.html"), "SHORTCODE") - - writeSource(t, fs, filepath.Join(workDir, "base", "a", "index.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "a", "l1.png"), "PNG image") - writeSource(t, fs, filepath.Join(workDir, "base", "a", "l2.png"), "PNG image") - - writeSource(t, fs, filepath.Join(workDir, "base", "b", "index.md"), `--- -title: "Headless Bundle in Topless Bar" -slug: s2 -headless: true -date: 2017-01-23 ---- - -TheContent. -HEADLESS {{< myShort >}} -`) - writeSource(t, fs, filepath.Join(workDir, "base", "b", "l1.png"), "PNG image") - writeSource(t, fs, filepath.Join(workDir, "base", "b", "l2.png"), "PNG image") - writeSource(t, fs, filepath.Join(workDir, "base", "b", "p1.md"), pageContent) - - s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) - - assert.Equal(1, len(s.RegularPages)) - assert.Equal(1, len(s.headlessPages)) - - regular := s.getPage(KindPage, "a/index") - assert.Equal("/a/s1/", regular.RelPermalink()) - - headless := s.getPage(KindPage, "b/index") - assert.NotNil(headless) - assert.True(headless.headless) - assert.Equal("Headless Bundle in Topless Bar", headless.Title()) - assert.Equal("", headless.RelPermalink()) - assert.Equal("", headless.Permalink()) - assert.Contains(headless.content(), "HEADLESS SHORTCODE") - - headlessResources := headless.Resources - assert.Equal(3, len(headlessResources)) - assert.Equal(2, len(headlessResources.Match("l*"))) - pageResource := headlessResources.GetMatch("p*") - assert.NotNil(pageResource) - assert.IsType(&Page{}, pageResource) - p := pageResource.(*Page) - assert.Contains(p.content(), "SHORTCODE") - assert.Equal("p1.md", p.Name()) - - th := testHelper{s.Cfg, s.Fs, t} - - th.assertFileContent(filepath.FromSlash(workDir+"/public/a/s1/index.html"), "TheContent") - th.assertFileContent(filepath.FromSlash(workDir+"/public/a/s1/l1.png"), "PNG") - - th.assertFileNotExist(workDir + "/public/b/s2/index.html") - // But the bundled resources needs to be published - th.assertFileContent(filepath.FromSlash(workDir+"/public/b/s2/l1.png"), "PNG") - -} - -func newTestBundleSources(t *testing.T) (*hugofs.Fs, *viper.Viper) { - cfg, fs := newTestCfg() - assert := require.New(t) - - workDir := "/work" - cfg.Set("workingDir", workDir) - cfg.Set("contentDir", "base") - cfg.Set("baseURL", "https://example.com") - cfg.Set("mediaTypes", map[string]interface{}{ - "text/bepsays": map[string]interface{}{ - "suffixes": []string{"bep"}, - }, - }) - - pageContent := `--- -title: "Bundle Galore" -slug: pageslug -date: 2017-10-09 ---- - -TheContent. -` - - pageContentShortcode := `--- -title: "Bundle Galore" -slug: pageslug -date: 2017-10-09 ---- - -TheContent. - -{{< myShort >}} -` - - pageWithImageShortcodeAndResourceMetadataContent := `--- -title: "Bundle Galore" -slug: pageslug -date: 2017-10-09 -resources: -- src: "*.jpg" - name: "my-sunset-:counter" - title: "Sunset Galore :counter" - params: - myParam: "My Sunny Param" ---- - -TheContent. - -{{< myShort >}} -` - - pageContentNoSlug := `--- -title: "Bundle Galore #2" -date: 2017-10-09 ---- - -TheContent. -` - - singleLayout := ` -Single Title: {{ .Title }} -Content: {{ .Content }} -{{ $sunset := .Resources.GetMatch "my-sunset-1*" }} -{{ with $sunset }} -Sunset RelPermalink: {{ .RelPermalink }} -{{ $thumb := .Fill "123x123" }} -Thumb Width: {{ $thumb.Width }} -Thumb Name: {{ $thumb.Name }} -Thumb Title: {{ $thumb.Title }} -Thumb RelPermalink: {{ $thumb.RelPermalink }} -{{ end }} -{{ range $i, $e := .Resources.ByType "image" }} -{{ $i }}: Image Title: {{ .Title }} -{{ $i }}: Image Name: {{ .Name }} -{{ $i }}: Image Params: {{ printf "%v" .Params }} -{{ $i }}: Image myParam: Lower: {{ .Params.myparam }} Caps: {{ .Params.MYPARAM }} -{{ end }} -` - - myShort := ` -MyShort in {{ .Page.Path }}: -{{ $sunset := .Page.Resources.GetMatch "my-sunset-2*" }} -{{ with $sunset }} -Short Sunset RelPermalink: {{ .RelPermalink }} -{{ $thumb := .Fill "56x56" }} -Short Thumb Width: {{ $thumb.Width }} -{{ end }} -` - - listLayout := `{{ .Title }}|{{ .Content }}` - - writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "single.html"), singleLayout) - writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "list.html"), listLayout) - writeSource(t, fs, filepath.Join(workDir, "layouts", "shortcodes", "myShort.html"), myShort) - - writeSource(t, fs, filepath.Join(workDir, "base", "_index.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "_1.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "_1.png"), pageContent) - - writeSource(t, fs, filepath.Join(workDir, "base", "images", "hugo-logo.png"), "content") - writeSource(t, fs, filepath.Join(workDir, "base", "a", "2.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "a", "1.md"), pageContent) - - writeSource(t, fs, filepath.Join(workDir, "base", "a", "b", "index.md"), pageContentNoSlug) - writeSource(t, fs, filepath.Join(workDir, "base", "a", "b", "ab1.md"), pageContentNoSlug) - - // Mostly plain static assets in a folder with a page in a sub folder thrown in. - writeSource(t, fs, filepath.Join(workDir, "base", "assets", "pic1.png"), "content") - writeSource(t, fs, filepath.Join(workDir, "base", "assets", "pic2.png"), "content") - writeSource(t, fs, filepath.Join(workDir, "base", "assets", "pages", "mypage.md"), pageContent) - - // Bundle - writeSource(t, fs, filepath.Join(workDir, "base", "b", "my-bundle", "index.md"), pageWithImageShortcodeAndResourceMetadataContent) - writeSource(t, fs, filepath.Join(workDir, "base", "b", "my-bundle", "1.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "b", "my-bundle", "2.md"), pageContentShortcode) - writeSource(t, fs, filepath.Join(workDir, "base", "b", "my-bundle", "custom-mime.bep"), "bepsays") - writeSource(t, fs, filepath.Join(workDir, "base", "b", "my-bundle", "c", "logo.png"), "content") - - // Bundle with 은행 slug - // See https://github.com/gohugoio/hugo/issues/4241 - writeSource(t, fs, filepath.Join(workDir, "base", "c", "bundle", "index.md"), `--- -title: "은행 은행" -slug: 은행 -date: 2017-10-09 ---- - -Content for 은행. -`) - - // Bundle in root - writeSource(t, fs, filepath.Join(workDir, "base", "root", "index.md"), pageWithImageShortcodeAndResourceMetadataContent) - writeSource(t, fs, filepath.Join(workDir, "base", "root", "1.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "root", "c", "logo.png"), "content") - - writeSource(t, fs, filepath.Join(workDir, "base", "c", "bundle", "logo-은행.png"), "은행 PNG") - - // Write a real image into one of the bundle above. - src, err := os.Open("testdata/sunset.jpg") - assert.NoError(err) - - // We need 2 to test https://github.com/gohugoio/hugo/issues/4202 - out, err := fs.Source.Create(filepath.Join(workDir, "base", "b", "my-bundle", "sunset1.jpg")) - assert.NoError(err) - out2, err := fs.Source.Create(filepath.Join(workDir, "base", "b", "my-bundle", "sunset2.jpg")) - assert.NoError(err) - - _, err = io.Copy(out, src) - out.Close() - src.Seek(0, 0) - _, err = io.Copy(out2, src) - out2.Close() - src.Close() - assert.NoError(err) - - return fs, cfg - -} - -func newTestBundleSourcesMultilingual(t *testing.T) (*hugofs.Fs, *viper.Viper) { - cfg, fs := newTestCfg() - - workDir := "/work" - cfg.Set("workingDir", workDir) - cfg.Set("contentDir", "base") - cfg.Set("baseURL", "https://example.com") - cfg.Set("defaultContentLanguage", "en") - - langConfig := map[string]interface{}{ - "en": map[string]interface{}{ - "weight": 1, - "languageName": "English", - }, - "nn": map[string]interface{}{ - "weight": 2, - "languageName": "Nynorsk", - }, - } - - cfg.Set("languages", langConfig) - - pageContent := `--- -slug: pageslug -date: 2017-10-09 ---- - -TheContent. -` - - layout := `{{ .Title }}|{{ .Content }}|Lang: {{ .Site.Language.Lang }}` - - writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "single.html"), layout) - writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "list.html"), layout) - - writeSource(t, fs, filepath.Join(workDir, "base", "1s", "mypage.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "1s", "mypage.nn.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "1s", "mylogo.png"), "content") - - writeSource(t, fs, filepath.Join(workDir, "base", "bb", "_index.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "bb", "_index.nn.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "bb", "en.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "bb", "_1.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "bb", "_1.nn.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "bb", "a.png"), "content") - writeSource(t, fs, filepath.Join(workDir, "base", "bb", "b.png"), "content") - writeSource(t, fs, filepath.Join(workDir, "base", "bb", "b.nn.png"), "content") - writeSource(t, fs, filepath.Join(workDir, "base", "bb", "c.nn.png"), "content") - writeSource(t, fs, filepath.Join(workDir, "base", "bb", "b", "d.nn.png"), "content") - - writeSource(t, fs, filepath.Join(workDir, "base", "bc", "_index.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "bc", "page.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "bc", "logo-bc.png"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "bc", "page.nn.md"), pageContent) - - writeSource(t, fs, filepath.Join(workDir, "base", "bd", "index.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "bd", "page.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "bd", "page.nn.md"), pageContent) - - writeSource(t, fs, filepath.Join(workDir, "base", "be", "_index.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "be", "page.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "be", "page.nn.md"), pageContent) - - // Bundle leaf, multilingual - writeSource(t, fs, filepath.Join(workDir, "base", "lb", "index.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "lb", "index.nn.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "lb", "1.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "lb", "2.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "lb", "2.nn.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "lb", "c", "page.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "lb", "c", "logo.png"), "content") - writeSource(t, fs, filepath.Join(workDir, "base", "lb", "c", "logo.nn.png"), "content") - writeSource(t, fs, filepath.Join(workDir, "base", "lb", "c", "one.png"), "content") - writeSource(t, fs, filepath.Join(workDir, "base", "lb", "c", "d", "deep.png"), "content") - - //Translated bundle in some sensible sub path. - writeSource(t, fs, filepath.Join(workDir, "base", "bf", "my-bf-bundle", "index.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "bf", "my-bf-bundle", "index.nn.md"), pageContent) - writeSource(t, fs, filepath.Join(workDir, "base", "bf", "my-bf-bundle", "page.md"), pageContent) - - return fs, cfg -} - -func newTestBundleSymbolicSources(t *testing.T) (*helpers.PathSpec, func(), string) { - assert := require.New(t) - // We need to use the OS fs for this. - cfg := viper.New() - fs := hugofs.NewFrom(hugofs.Os, cfg) - fs.Destination = &afero.MemMapFs{} - loadDefaultSettingsFor(cfg) - - workDir, clean, err := createTempDir("hugosym") - assert.NoError(err) - - contentDir := "base" - cfg.Set("workingDir", workDir) - cfg.Set("contentDir", contentDir) - cfg.Set("baseURL", "https://example.com") - - if err := loadLanguageSettings(cfg, nil); err != nil { - t.Fatal(err) - } - - layout := `{{ .Title }}|{{ .Content }}` - pageContent := `--- -slug: %s -date: 2017-10-09 ---- - -TheContent. -` - - fs.Source.MkdirAll(filepath.Join(workDir, "layouts", "_default"), 0777) - fs.Source.MkdirAll(filepath.Join(workDir, contentDir), 0777) - fs.Source.MkdirAll(filepath.Join(workDir, contentDir, "a"), 0777) - for i := 1; i <= 3; i++ { - fs.Source.MkdirAll(filepath.Join(workDir, fmt.Sprintf("symcontent%d", i)), 0777) - - } - fs.Source.MkdirAll(filepath.Join(workDir, "symcontent2", "a1"), 0777) - - writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "single.html"), layout) - writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "list.html"), layout) - - writeSource(t, fs, filepath.Join(workDir, contentDir, "a", "regular.md"), fmt.Sprintf(pageContent, "a1")) - - // Regular files inside symlinked folder. - writeSource(t, fs, filepath.Join(workDir, "symcontent1", "s1.md"), fmt.Sprintf(pageContent, "s1")) - writeSource(t, fs, filepath.Join(workDir, "symcontent1", "s2.md"), fmt.Sprintf(pageContent, "s2")) - - // A bundle - writeSource(t, fs, filepath.Join(workDir, "symcontent2", "a1", "index.md"), fmt.Sprintf(pageContent, "")) - writeSource(t, fs, filepath.Join(workDir, "symcontent2", "a1", "page.md"), fmt.Sprintf(pageContent, "page")) - writeSource(t, fs, filepath.Join(workDir, "symcontent2", "a1", "logo.png"), "image") - - // Assets - writeSource(t, fs, filepath.Join(workDir, "symcontent3", "s1.png"), "image") - writeSource(t, fs, filepath.Join(workDir, "symcontent3", "s2.png"), "image") - - wd, _ := os.Getwd() - defer func() { - os.Chdir(wd) - }() - // Symlinked sections inside content. - os.Chdir(filepath.Join(workDir, contentDir)) - for i := 1; i <= 3; i++ { - assert.NoError(os.Symlink(filepath.FromSlash(fmt.Sprintf(("../symcontent%d"), i)), fmt.Sprintf("symbolic%d", i))) - } - - os.Chdir(filepath.Join(workDir, contentDir, "a")) - - // Create a symlink to one single content file - assert.NoError(os.Symlink(filepath.FromSlash("../../symcontent2/a1/page.md"), "page_s.md")) - - os.Chdir(filepath.FromSlash("../../symcontent3")) - - // Create a circular symlink. Will print some warnings. - assert.NoError(os.Symlink(filepath.Join("..", contentDir), filepath.FromSlash("circus"))) - - os.Chdir(workDir) - assert.NoError(err) - - ps, _ := helpers.NewPathSpec(fs, cfg) - - return ps, clean, workDir -} diff --git a/hugolib/page_collections.go b/hugolib/page_collections.go deleted file mode 100644 index 78325344..00000000 --- a/hugolib/page_collections.go +++ /dev/null @@ -1,341 +0,0 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "fmt" - "path" - "path/filepath" - "strings" - - "github.com/gohugoio/hugo/cache" - "github.com/gohugoio/hugo/helpers" -) - -// PageCollections contains the page collections for a site. -type PageCollections struct { - // Includes only pages of all types, and only pages in the current language. - Pages Pages - - // Includes all pages in all languages, including the current one. - // Includes pages of all types. - AllPages Pages - - // A convenience cache for the traditional index types, taxonomies, home page etc. - // This is for the current language only. - indexPages Pages - - // A convenience cache for the regular pages. - // This is for the current language only. - RegularPages Pages - - // A convenience cache for the all the regular pages. - AllRegularPages Pages - - // Includes absolute all pages (of all types), including drafts etc. - rawAllPages Pages - - // Includes headless bundles, i.e. bundles that produce no output for its content page. - headlessPages Pages - - pageIndex *cache.Lazy -} - -// Get initializes the index if not already done so, then -// looks up the given page ref, returns nil if no value found. -func (c *PageCollections) getFromCache(ref string) (*Page, error) { - v, found, err := c.pageIndex.Get(ref) - if err != nil { - return nil, err - } - if !found { - return nil, nil - } - - p := v.(*Page) - - if p != ambiguityFlag { - return p, nil - } - return nil, fmt.Errorf("page reference %q is ambiguous", ref) -} - -var ambiguityFlag = &Page{Kind: kindUnknown, title: "ambiguity flag"} - -func (c *PageCollections) refreshPageCaches() { - c.indexPages = c.findPagesByKindNotIn(KindPage, c.Pages) - c.RegularPages = c.findPagesByKindIn(KindPage, c.Pages) - c.AllRegularPages = c.findPagesByKindIn(KindPage, c.AllPages) - - indexLoader := func() (map[string]interface{}, error) { - index := make(map[string]interface{}) - - add := func(ref string, p *Page) { - existing := index[ref] - if existing == nil { - index[ref] = p - } else if existing != ambiguityFlag && existing != p { - index[ref] = ambiguityFlag - } - } - - for _, pageCollection := range []Pages{c.RegularPages, c.headlessPages} { - for _, p := range pageCollection { - sourceRef := p.absoluteSourceRef() - - if sourceRef != "" { - // index the canonical ref - // e.g. /section/article.md - add(sourceRef, p) - } - - // Ref/Relref supports this potentially ambiguous lookup. - add(p.LogicalName(), p) - - translationBaseName := p.TranslationBaseName() - - dir, _ := path.Split(sourceRef) - dir = strings.TrimSuffix(dir, "/") - - if translationBaseName == "index" { - add(dir, p) - add(path.Base(dir), p) - } else { - add(translationBaseName, p) - } - - // We need a way to get to the current language version. - pathWithNoExtensions := path.Join(dir, translationBaseName) - add(pathWithNoExtensions, p) - } - } - - for _, p := range c.indexPages { - // index the canonical, unambiguous ref for any backing file - // e.g. /section/_index.md - sourceRef := p.absoluteSourceRef() - if sourceRef != "" { - add(sourceRef, p) - } - - ref := path.Join(p.sections...) - - // index the canonical, unambiguous virtual ref - // e.g. /section - // (this may already have been indexed above) - add("/"+ref, p) - } - - return index, nil - } - - c.pageIndex = cache.NewLazy(indexLoader) -} - -func newPageCollections() *PageCollections { - return &PageCollections{} -} - -func newPageCollectionsFromPages(pages Pages) *PageCollections { - return &PageCollections{rawAllPages: pages} -} - -// This is an adapter func for the old API with Kind as first argument. -// This is invoked when you do .Site.GetPage. We drop the Kind and fails -// if there are more than 2 arguments, which would be ambigous. -func (c *PageCollections) getPageOldVersion(ref ...string) (*Page, error) { - var refs []string - for _, r := range ref { - // A common construct in the wild is - // .Site.GetPage "home" "" or - // .Site.GetPage "home" "/" - if r != "" && r != "/" { - refs = append(refs, r) - } - } - - var key string - - if len(refs) > 2 { - // This was allowed in Hugo <= 0.44, but we cannot support this with the - // new API. This should be the most unusual case. - return nil, fmt.Errorf(`too many arguments to .Site.GetPage: %v. Use lookups on the form {{ .Site.GetPage "/posts/mypage-md" }}`, ref) - } - - if len(refs) == 0 || refs[0] == KindHome { - key = "/" - } else if len(refs) == 1 { - if len(ref) == 2 && refs[0] == KindSection { - // This is an old style reference to the "Home Page section". - // Typically fetched via {{ .Site.GetPage "section" .Section }} - // See https://github.com/gohugoio/hugo/issues/4989 - key = "/" - } else { - key = refs[0] - } - } else { - key = refs[1] - } - - key = filepath.ToSlash(key) - if !strings.HasPrefix(key, "/") { - key = "/" + key - } - - return c.getPageNew(nil, key) -} - -// Only used in tests. -func (c *PageCollections) getPage(typ string, sections ...string) *Page { - refs := append([]string{typ}, path.Join(sections...)) - p, _ := c.getPageOldVersion(refs...) - return p -} - -// Ref is either unix-style paths (i.e. callers responsible for -// calling filepath.ToSlash as necessary) or shorthand refs. -func (c *PageCollections) getPageNew(context *Page, ref string) (*Page, error) { - var anError error - - // Absolute (content root relative) reference. - if strings.HasPrefix(ref, "/") { - p, err := c.getFromCache(ref) - if err == nil && p != nil { - return p, nil - } - if err != nil { - anError = err - } - - } else if context != nil { - // Try the page-relative path. - ppath := path.Join("/", strings.Join(context.sections, "/"), ref) - p, err := c.getFromCache(ppath) - if err == nil && p != nil { - return p, nil - } - if err != nil { - anError = err - } - } - - if !strings.HasPrefix(ref, "/") { - // Many people will have "post/foo.md" in their content files. - p, err := c.getFromCache("/" + ref) - if err == nil && p != nil { - if context != nil { - // TODO(bep) remove this case and the message below when the storm has passed - helpers.DistinctFeedbackLog.Printf(`WARNING: make non-relative ref/relref page reference(s) in page %q absolute, e.g. {{< ref "/blog/my-post.md" >}}`, context.absoluteSourceRef()) - } - return p, nil - } - if err != nil { - anError = err - } - } - - // Last try. - ref = strings.TrimPrefix(ref, "/") - p, err := c.getFromCache(ref) - if err != nil { - anError = err - } - - if p == nil && anError != nil { - if context != nil { - return nil, fmt.Errorf("failed to resolve path from page %q: %s", context.absoluteSourceRef(), anError) - } - return nil, fmt.Errorf("failed to resolve page: %s", anError) - } - - return p, nil -} - -func (*PageCollections) findPagesByKindIn(kind string, inPages Pages) Pages { - var pages Pages - for _, p := range inPages { - if p.Kind == kind { - pages = append(pages, p) - } - } - return pages -} - -func (*PageCollections) findFirstPageByKindIn(kind string, inPages Pages) *Page { - for _, p := range inPages { - if p.Kind == kind { - return p - } - } - return nil -} - -func (*PageCollections) findPagesByKindNotIn(kind string, inPages Pages) Pages { - var pages Pages - for _, p := range inPages { - if p.Kind != kind { - pages = append(pages, p) - } - } - return pages -} - -func (c *PageCollections) findPagesByKind(kind string) Pages { - return c.findPagesByKindIn(kind, c.Pages) -} - -func (c *PageCollections) addPage(page *Page) { - c.rawAllPages = append(c.rawAllPages, page) -} - -func (c *PageCollections) removePageFilename(filename string) { - if i := c.rawAllPages.findPagePosByFilename(filename); i >= 0 { - c.clearResourceCacheForPage(c.rawAllPages[i]) - c.rawAllPages = append(c.rawAllPages[:i], c.rawAllPages[i+1:]...) - } - -} - -func (c *PageCollections) removePage(page *Page) { - if i := c.rawAllPages.findPagePos(page); i >= 0 { - c.clearResourceCacheForPage(c.rawAllPages[i]) - c.rawAllPages = append(c.rawAllPages[:i], c.rawAllPages[i+1:]...) - } - -} - -func (c *PageCollections) findPagesByShortcode(shortcode string) Pages { - var pages Pages - - for _, p := range c.rawAllPages { - if p.shortcodeState != nil { - if _, ok := p.shortcodeState.nameSet[shortcode]; ok { - pages = append(pages, p) - } - } - } - return pages -} - -func (c *PageCollections) replacePage(page *Page) { - // will find existing page that matches filepath and remove it - c.removePage(page) - c.addPage(page) -} - -func (c *PageCollections) clearResourceCacheForPage(page *Page) { - if len(page.Resources) > 0 { - page.s.ResourceSpec.DeleteCacheByPrefix(page.relTargetPathBase) - } -} diff --git a/hugolib/page_collections_test.go b/hugolib/page_collections_test.go deleted file mode 100644 index 2f8b3149..00000000 --- a/hugolib/page_collections_test.go +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "fmt" - "math/rand" - "path" - "path/filepath" - "testing" - "time" - - "github.com/gohugoio/hugo/deps" - "github.com/stretchr/testify/require" -) - -const pageCollectionsPageTemplate = `--- -title: "%s" -categories: -- Hugo ---- -# Doc -` - -func BenchmarkGetPage(b *testing.B) { - var ( - cfg, fs = newTestCfg() - r = rand.New(rand.NewSource(time.Now().UnixNano())) - ) - - for i := 0; i < 10; i++ { - for j := 0; j < 100; j++ { - writeSource(b, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", j)), "CONTENT") - } - } - - s := buildSingleSite(b, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - - pagePaths := make([]string, b.N) - - for i := 0; i < b.N; i++ { - pagePaths[i] = fmt.Sprintf("sect%d", r.Intn(10)) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - home, _ := s.getPageNew(nil, "/") - if home == nil { - b.Fatal("Home is nil") - } - - p, _ := s.getPageNew(nil, pagePaths[i]) - if p == nil { - b.Fatal("Section is nil") - } - - } -} - -func BenchmarkGetPageRegular(b *testing.B) { - var ( - cfg, fs = newTestCfg() - r = rand.New(rand.NewSource(time.Now().UnixNano())) - ) - - for i := 0; i < 10; i++ { - for j := 0; j < 100; j++ { - content := fmt.Sprintf(pageCollectionsPageTemplate, fmt.Sprintf("Title%d_%d", i, j)) - writeSource(b, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", j)), content) - } - } - - s := buildSingleSite(b, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - - pagePaths := make([]string, b.N) - - for i := 0; i < b.N; i++ { - pagePaths[i] = path.Join(fmt.Sprintf("sect%d", r.Intn(10)), fmt.Sprintf("page%d.md", r.Intn(100))) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - page, _ := s.getPageNew(nil, pagePaths[i]) - require.NotNil(b, page) - } -} - -type testCase struct { - kind string - context *Page - path []string - expectedTitle string -} - -func (t *testCase) check(p *Page, err error, errorMsg string, assert *require.Assertions) { - switch t.kind { - case "Ambiguous": - assert.Error(err) - assert.Nil(p, errorMsg) - case "NoPage": - assert.NoError(err) - assert.Nil(p, errorMsg) - default: - assert.NoError(err, errorMsg) - assert.NotNil(p, errorMsg) - assert.Equal(t.kind, p.Kind, errorMsg) - assert.Equal(t.expectedTitle, p.title, errorMsg) - } -} - -func TestGetPage(t *testing.T) { - - var ( - assert = require.New(t) - cfg, fs = newTestCfg() - ) - - for i := 0; i < 10; i++ { - for j := 0; j < 10; j++ { - content := fmt.Sprintf(pageCollectionsPageTemplate, fmt.Sprintf("Title%d_%d", i, j)) - writeSource(t, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", j)), content) - } - } - - content := fmt.Sprintf(pageCollectionsPageTemplate, "home page") - writeSource(t, fs, filepath.Join("content", "_index.md"), content) - - content = fmt.Sprintf(pageCollectionsPageTemplate, "about page") - writeSource(t, fs, filepath.Join("content", "about.md"), content) - - content = fmt.Sprintf(pageCollectionsPageTemplate, "section 3") - writeSource(t, fs, filepath.Join("content", "sect3", "_index.md"), content) - - content = fmt.Sprintf(pageCollectionsPageTemplate, "UniqueBase") - writeSource(t, fs, filepath.Join("content", "sect3", "unique.md"), content) - - content = fmt.Sprintf(pageCollectionsPageTemplate, "another sect7") - writeSource(t, fs, filepath.Join("content", "sect3", "sect7", "_index.md"), content) - - content = fmt.Sprintf(pageCollectionsPageTemplate, "deep page") - writeSource(t, fs, filepath.Join("content", "sect3", "subsect", "deep.md"), content) - - s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - - sec3, err := s.getPageNew(nil, "/sect3") - assert.NoError(err, "error getting Page for /sec3") - assert.NotNil(sec3, "failed to get Page for /sec3") - - tests := []testCase{ - // legacy content root relative paths - {KindHome, nil, []string{}, "home page"}, - {KindPage, nil, []string{"about.md"}, "about page"}, - {KindSection, nil, []string{"sect3"}, "section 3"}, - {KindPage, nil, []string{"sect3/page1.md"}, "Title3_1"}, - {KindPage, nil, []string{"sect4/page2.md"}, "Title4_2"}, - {KindSection, nil, []string{"sect3/sect7"}, "another sect7"}, - {KindPage, nil, []string{"sect3/subsect/deep.md"}, "deep page"}, - {KindPage, nil, []string{filepath.FromSlash("sect5/page3.md")}, "Title5_3"}, //test OS-specific path - - // shorthand refs (potentially ambiguous) - {KindPage, nil, []string{"unique.md"}, "UniqueBase"}, - {"Ambiguous", nil, []string{"page1.md"}, ""}, - - // ISSUE: This is an ambiguous ref, but because we have to support the legacy - // content root relative paths without a leading slash, the lookup - // returns /sect7. This undermines ambiguity detection, but we have no choice. - //{"Ambiguous", nil, []string{"sect7"}, ""}, - {KindSection, nil, []string{"sect7"}, "Sect7s"}, - - // absolute paths - {KindHome, nil, []string{"/"}, "home page"}, - {KindPage, nil, []string{"/about.md"}, "about page"}, - {KindSection, nil, []string{"/sect3"}, "section 3"}, - {KindPage, nil, []string{"/sect3/page1.md"}, "Title3_1"}, - {KindPage, nil, []string{"/sect4/page2.md"}, "Title4_2"}, - {KindSection, nil, []string{"/sect3/sect7"}, "another sect7"}, - {KindPage, nil, []string{"/sect3/subsect/deep.md"}, "deep page"}, - {KindPage, nil, []string{filepath.FromSlash("/sect5/page3.md")}, "Title5_3"}, //test OS-specific path - {KindPage, nil, []string{"/sect3/unique.md"}, "UniqueBase"}, //next test depends on this page existing - // {"NoPage", nil, []string{"/unique.md"}, ""}, // ISSUE #4969: this is resolving to /sect3/unique.md - {"NoPage", nil, []string{"/missing-page.md"}, ""}, - {"NoPage", nil, []string{"/missing-section"}, ""}, - - // relative paths - {KindHome, sec3, []string{".."}, "home page"}, - {KindHome, sec3, []string{"../"}, "home page"}, - {KindPage, sec3, []string{"../about.md"}, "about page"}, - {KindSection, sec3, []string{"."}, "section 3"}, - {KindSection, sec3, []string{"./"}, "section 3"}, - {KindPage, sec3, []string{"page1.md"}, "Title3_1"}, - {KindPage, sec3, []string{"./page1.md"}, "Title3_1"}, - {KindPage, sec3, []string{"../sect4/page2.md"}, "Title4_2"}, - {KindSection, sec3, []string{"sect7"}, "another sect7"}, - {KindSection, sec3, []string{"./sect7"}, "another sect7"}, - {KindPage, sec3, []string{"./subsect/deep.md"}, "deep page"}, - {KindPage, sec3, []string{"./subsect/../../sect7/page9.md"}, "Title7_9"}, - {KindPage, sec3, []string{filepath.FromSlash("../sect5/page3.md")}, "Title5_3"}, //test OS-specific path - {KindPage, sec3, []string{"./unique.md"}, "UniqueBase"}, - {"NoPage", sec3, []string{"./sect2"}, ""}, - //{"NoPage", sec3, []string{"sect2"}, ""}, // ISSUE: /sect3 page relative query is resolving to /sect2 - - // absolute paths ignore context - {KindHome, sec3, []string{"/"}, "home page"}, - {KindPage, sec3, []string{"/about.md"}, "about page"}, - {KindPage, sec3, []string{"/sect4/page2.md"}, "Title4_2"}, - {KindPage, sec3, []string{"/sect3/subsect/deep.md"}, "deep page"}, //next test depends on this page existing - {"NoPage", sec3, []string{"/subsect/deep.md"}, ""}, - } - - for _, test := range tests { - errorMsg := fmt.Sprintf("Test case %s %v -> %s", test.context, test.path, test.expectedTitle) - - // test legacy public Site.GetPage (which does not support page context relative queries) - if test.context == nil { - args := append([]string{test.kind}, test.path...) - page, err := s.Info.GetPage(args...) - test.check(page, err, errorMsg, assert) - } - - // test new internal Site.getPageNew - var ref string - if len(test.path) == 1 { - ref = filepath.ToSlash(test.path[0]) - } else { - ref = path.Join(test.path...) - } - page2, err := s.getPageNew(test.context, ref) - test.check(page2, err, errorMsg, assert) - } - -} diff --git a/hugolib/pagebundler.go b/hugolib/pagebundler.go new file mode 100644 index 00000000..62ef2b52 --- /dev/null +++ b/hugolib/pagebundler.go @@ -0,0 +1,209 @@ +// Copyright 2017-present The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "context" + "fmt" + "math" + "runtime" + + _errors "github.com/pkg/errors" + + "golang.org/x/sync/errgroup" +) + +type siteContentProcessor struct { + site *Site + + handleContent contentHandler + + ctx context.Context + + // The input file bundles. + fileBundlesChan chan *bundleDir + + // The input file singles. + fileSinglesChan chan *fileInfo + + // These assets should be just copied to destination. + fileAssetsChan chan []pathLangFile + + numWorkers int + + // The output Pages + pagesChan chan *Page + + // Used for partial rebuilds (aka. live reload) + // Will signal replacement of pages in the site collection. + partialBuild bool +} + +func (s *siteContentProcessor) processBundle(b *bundleDir) { + select { + case s.fileBundlesChan <- b: + case <-s.ctx.Done(): + } +} + +func (s *siteContentProcessor) processSingle(fi *fileInfo) { + select { + case s.fileSinglesChan <- fi: + case <-s.ctx.Done(): + } +} + +func (s *siteContentProcessor) processAssets(assets []pathLangFile) { + select { + case s.fileAssetsChan <- assets: + case <-s.ctx.Done(): + } +} + +func newSiteContentProcessor(ctx context.Context, partialBuild bool, s *Site) *siteContentProcessor { + numWorkers := 12 + if n := runtime.NumCPU() * 3; n > numWorkers { + numWorkers = n + } + + numWorkers = int(math.Ceil(float64(numWorkers) / float64(len(s.owner.Sites)))) + + return &siteContentProcessor{ + ctx: ctx, + partialBuild: partialBuild, + site: s, + handleContent: newHandlerChain(s), + fileBundlesChan: make(chan *bundleDir, numWorkers), + fileSinglesChan: make(chan *fileInfo, numWorkers), + fileAssetsChan: make(chan []pathLangFile, numWorkers), + numWorkers: numWorkers, + pagesChan: make(chan *Page, numWorkers), + } +} + +func (s *siteContentProcessor) closeInput() { + close(s.fileSinglesChan) + close(s.fileBundlesChan) + close(s.fileAssetsChan) +} + +func (s *siteContentProcessor) process(ctx context.Context) error { + g1, ctx := errgroup.WithContext(ctx) + g2, ctx := errgroup.WithContext(ctx) + + // There can be only one of these per site. + g1.Go(func() error { + for p := range s.pagesChan { + if p.s != s.site { + panic(fmt.Sprintf("invalid page site: %v vs %v", p.s, s)) + } + + if s.partialBuild { + p.forceRender = true + s.site.replacePage(p) + } else { + s.site.addPage(p) + } + } + return nil + }) + + for i := 0; i < s.numWorkers; i++ { + g2.Go(func() error { + for { + select { + case f, ok := <-s.fileSinglesChan: + if !ok { + return nil + } + err := s.readAndConvertContentFile(f) + if err != nil { + return err + } + case <-ctx.Done(): + return ctx.Err() + } + } + }) + + g2.Go(func() error { + for { + select { + case files, ok := <-s.fileAssetsChan: + if !ok { + return nil + } + for _, file := range files { + f, err := s.site.BaseFs.Content.Fs.Open(file.Filename()) + if err != nil { + return _errors.Wrap(err, "failed to open assets file") + } + err = s.site.publish(&s.site.PathSpec.ProcessingStats.Files, file.Path(), f) + f.Close() + if err != nil { + return err + } + } + + case <-ctx.Done(): + return ctx.Err() + } + } + }) + + g2.Go(func() error { + for { + select { + case bundle, ok := <-s.fileBundlesChan: + if !ok { + return nil + } + err := s.readAndConvertContentBundle(bundle) + if err != nil { + return err + } + case <-ctx.Done(): + return ctx.Err() + } + } + }) + } + + err := g2.Wait() + + close(s.pagesChan) + + if err != nil { + return err + } + + if err := g1.Wait(); err != nil { + return err + } + + s.site.rawAllPages.sort() + + return nil + +} + +func (s *siteContentProcessor) readAndConvertContentFile(file *fileInfo) error { + ctx := &handlerContext{source: file, pages: s.pagesChan} + return s.handleContent(ctx).err +} + +func (s *siteContentProcessor) readAndConvertContentBundle(bundle *bundleDir) error { + ctx := &handlerContext{bundle: bundle, pages: s.pagesChan} + return s.handleContent(ctx).err +} diff --git a/hugolib/pagebundler_capture.go b/hugolib/pagebundler_capture.go new file mode 100644 index 00000000..c152262c --- /dev/null +++ b/hugolib/pagebundler_capture.go @@ -0,0 +1,775 @@ +// Copyright 2017-present The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "runtime" + + "github.com/gohugoio/hugo/common/loggers" + _errors "github.com/pkg/errors" + + "sort" + "strings" + "sync" + + "github.com/spf13/afero" + + "github.com/gohugoio/hugo/hugofs" + + "github.com/gohugoio/hugo/helpers" + + "golang.org/x/sync/errgroup" + + "github.com/gohugoio/hugo/source" +) + +var errSkipCyclicDir = errors.New("skip potential cyclic dir") + +type capturer struct { + // To prevent symbolic link cycles: Visit same folder only once. + seen map[string]bool + seenMu sync.Mutex + + handler captureResultHandler + + sourceSpec *source.SourceSpec + fs afero.Fs + logger *loggers.Logger + + // Filenames limits the content to process to a list of filenames/directories. + // This is used for partial building in server mode. + filenames []string + + // Used to determine how to handle content changes in server mode. + contentChanges *contentChangeMap + + // Semaphore used to throttle the concurrent sub directory handling. + sem chan bool +} + +func newCapturer( + logger *loggers.Logger, + sourceSpec *source.SourceSpec, + handler captureResultHandler, + contentChanges *contentChangeMap, + filenames ...string) *capturer { + + numWorkers := 4 + if n := runtime.NumCPU(); n > numWorkers { + numWorkers = n + } + + // TODO(bep) the "index" vs "_index" check/strings should be moved in one place. + isBundleHeader := func(filename string) bool { + base := filepath.Base(filename) + name := helpers.Filename(base) + return IsContentFile(base) && (name == "index" || name == "_index") + } + + // Make sure that any bundle header files are processed before the others. This makes + // sure that any bundle head is processed before its resources. + sort.Slice(filenames, func(i, j int) bool { + a, b := filenames[i], filenames[j] + ac, bc := isBundleHeader(a), isBundleHeader(b) + + if ac { + return true + } + + if bc { + return false + } + + return a < b + }) + + c := &capturer{ + sem: make(chan bool, numWorkers), + handler: handler, + sourceSpec: sourceSpec, + fs: sourceSpec.SourceFs, + logger: logger, + contentChanges: contentChanges, + seen: make(map[string]bool), + filenames: filenames} + + return c +} + +// Captured files and bundles ready to be processed will be passed on to +// these channels. +type captureResultHandler interface { + handleSingles(fis ...*fileInfo) + handleCopyFiles(fis ...pathLangFile) + captureBundlesHandler +} + +type captureBundlesHandler interface { + handleBundles(b *bundleDirs) +} + +type captureResultHandlerChain struct { + handlers []captureBundlesHandler +} + +func (c *captureResultHandlerChain) handleSingles(fis ...*fileInfo) { + for _, h := range c.handlers { + if hh, ok := h.(captureResultHandler); ok { + hh.handleSingles(fis...) + } + } +} +func (c *captureResultHandlerChain) handleBundles(b *bundleDirs) { + for _, h := range c.handlers { + h.handleBundles(b) + } +} + +func (c *captureResultHandlerChain) handleCopyFiles(files ...pathLangFile) { + for _, h := range c.handlers { + if hh, ok := h.(captureResultHandler); ok { + hh.handleCopyFiles(files...) + } + } +} + +func (c *capturer) capturePartial(filenames ...string) error { + handled := make(map[string]bool) + + for _, filename := range filenames { + dir, resolvedFilename, tp := c.contentChanges.resolveAndRemove(filename) + if handled[resolvedFilename] { + continue + } + + handled[resolvedFilename] = true + + switch tp { + case bundleLeaf: + if err := c.handleDir(resolvedFilename); err != nil { + // Directory may have been deleted. + if !os.IsNotExist(err) { + return err + } + } + case bundleBranch: + if err := c.handleBranchDir(resolvedFilename); err != nil { + // Directory may have been deleted. + if !os.IsNotExist(err) { + return err + } + } + default: + fi, err := c.resolveRealPath(resolvedFilename) + if os.IsNotExist(err) { + // File has been deleted. + continue + } + + // Just in case the owning dir is a new symlink -- this will + // create the proper mapping for it. + c.resolveRealPath(dir) + + f, active := c.newFileInfo(fi, tp) + if active { + c.copyOrHandleSingle(f) + } + } + } + + return nil +} + +func (c *capturer) capture() error { + if len(c.filenames) > 0 { + return c.capturePartial(c.filenames...) + } + + err := c.handleDir(helpers.FilePathSeparator) + if err != nil { + return err + } + + return nil +} + +func (c *capturer) handleNestedDir(dirname string) error { + select { + case c.sem <- true: + var g errgroup.Group + + g.Go(func() error { + defer func() { + <-c.sem + }() + return c.handleDir(dirname) + }) + return g.Wait() + default: + // For deeply nested file trees, waiting for a semaphore wil deadlock. + return c.handleDir(dirname) + } +} + +// This handles a bundle branch and its resources only. This is used +// in server mode on changes. If this dir does not (anymore) represent a bundle +// branch, the handling is upgraded to the full handleDir method. +func (c *capturer) handleBranchDir(dirname string) error { + files, err := c.readDir(dirname) + if err != nil { + + return err + } + + var ( + dirType bundleDirType + ) + + for _, fi := range files { + if !fi.IsDir() { + tp, _ := classifyBundledFile(fi.RealName()) + if dirType == bundleNot { + dirType = tp + } + + if dirType == bundleLeaf { + return c.handleDir(dirname) + } + } + } + + if dirType != bundleBranch { + return c.handleDir(dirname) + } + + dirs := newBundleDirs(bundleBranch, c) + + var secondPass []*fileInfo + + // Handle potential bundle headers first. + for _, fi := range files { + if fi.IsDir() { + continue + } + + tp, isContent := classifyBundledFile(fi.RealName()) + + f, active := c.newFileInfo(fi, tp) + + if !active { + continue + } + + if !f.isOwner() { + if !isContent { + // This is a partial update -- we only care about the files that + // is in this bundle. + secondPass = append(secondPass, f) + } + continue + } + dirs.addBundleHeader(f) + } + + for _, f := range secondPass { + dirs.addBundleFiles(f) + } + + c.handler.handleBundles(dirs) + + return nil + +} + +func (c *capturer) handleDir(dirname string) error { + + files, err := c.readDir(dirname) + if err != nil { + return err + } + + type dirState int + + const ( + dirStateDefault dirState = iota + + dirStateAssetsOnly + dirStateSinglesOnly + ) + + var ( + fileBundleTypes = make([]bundleDirType, len(files)) + + // Start with the assumption that this dir contains only non-content assets (images etc.) + // If that is still true after we had a first look at the list of files, we + // can just copy the files to destination. We will still have to look at the + // sub-folders for potential bundles. + state = dirStateAssetsOnly + + // Start with the assumption that this dir is not a bundle. + // A directory is a bundle if it contains a index content file, + // e.g. index.md (a leaf bundle) or a _index.md (a branch bundle). + bundleType = bundleNot + ) + + /* First check for any content files. + - If there are none, then this is a assets folder only (images etc.) + and we can just plainly copy them to + destination. + - If this is a section with no image etc. or similar, we can just handle it + as it was a single content file. + */ + var hasNonContent, isBranch bool + + for i, fi := range files { + if !fi.IsDir() { + tp, isContent := classifyBundledFile(fi.RealName()) + + fileBundleTypes[i] = tp + if !isBranch { + isBranch = tp == bundleBranch + } + + if isContent { + // This is not a assets-only folder. + state = dirStateDefault + } else { + hasNonContent = true + } + } + } + + if isBranch && !hasNonContent { + // This is a section or similar with no need for any bundle handling. + state = dirStateSinglesOnly + } + + if state > dirStateDefault { + return c.handleNonBundle(dirname, files, state == dirStateSinglesOnly) + } + + var fileInfos = make([]*fileInfo, 0, len(files)) + + for i, fi := range files { + + currentType := bundleNot + + if !fi.IsDir() { + currentType = fileBundleTypes[i] + if bundleType == bundleNot && currentType != bundleNot { + bundleType = currentType + } + } + + if bundleType == bundleNot && currentType != bundleNot { + bundleType = currentType + } + + f, active := c.newFileInfo(fi, currentType) + + if !active { + continue + } + + fileInfos = append(fileInfos, f) + } + + var todo []*fileInfo + + if bundleType != bundleLeaf { + for _, fi := range fileInfos { + if fi.FileInfo().IsDir() { + // Handle potential nested bundles. + if err := c.handleNestedDir(fi.Path()); err != nil { + return err + } + } else if bundleType == bundleNot || (!fi.isOwner() && fi.isContentFile()) { + // Not in a bundle. + c.copyOrHandleSingle(fi) + } else { + // This is a section folder or similar with non-content files in it. + todo = append(todo, fi) + } + } + } else { + todo = fileInfos + } + + if len(todo) == 0 { + return nil + } + + dirs, err := c.createBundleDirs(todo, bundleType) + if err != nil { + return err + } + + // Send the bundle to the next step in the processor chain. + c.handler.handleBundles(dirs) + + return nil +} + +func (c *capturer) handleNonBundle( + dirname string, + fileInfos pathLangFileFis, + singlesOnly bool) error { + + for _, fi := range fileInfos { + if fi.IsDir() { + if err := c.handleNestedDir(fi.Filename()); err != nil { + return err + } + } else { + if singlesOnly { + f, active := c.newFileInfo(fi, bundleNot) + if !active { + continue + } + c.handler.handleSingles(f) + } else { + c.handler.handleCopyFiles(fi) + } + } + } + + return nil +} + +func (c *capturer) copyOrHandleSingle(fi *fileInfo) { + if fi.isContentFile() { + c.handler.handleSingles(fi) + } else { + // These do not currently need any further processing. + c.handler.handleCopyFiles(fi) + } +} + +func (c *capturer) createBundleDirs(fileInfos []*fileInfo, bundleType bundleDirType) (*bundleDirs, error) { + dirs := newBundleDirs(bundleType, c) + + for _, fi := range fileInfos { + if fi.FileInfo().IsDir() { + var collector func(fis ...*fileInfo) + + if bundleType == bundleBranch { + // All files in the current directory are part of this bundle. + // Trying to include sub folders in these bundles are filled with ambiguity. + collector = func(fis ...*fileInfo) { + for _, fi := range fis { + c.copyOrHandleSingle(fi) + } + } + } else { + // All nested files and directories are part of this bundle. + collector = func(fis ...*fileInfo) { + fileInfos = append(fileInfos, fis...) + } + } + err := c.collectFiles(fi.Path(), collector) + if err != nil { + return nil, err + } + + } else if fi.isOwner() { + // There can be more than one language, so: + // 1. Content files must be attached to its language's bundle. + // 2. Other files must be attached to all languages. + // 3. Every content file needs a bundle header. + dirs.addBundleHeader(fi) + } + } + + for _, fi := range fileInfos { + if fi.FileInfo().IsDir() || fi.isOwner() { + continue + } + + if fi.isContentFile() { + if bundleType != bundleBranch { + dirs.addBundleContentFile(fi) + } + } else { + dirs.addBundleFiles(fi) + } + } + + return dirs, nil +} + +func (c *capturer) collectFiles(dirname string, handleFiles func(fis ...*fileInfo)) error { + + filesInDir, err := c.readDir(dirname) + if err != nil { + return err + } + + for _, fi := range filesInDir { + if fi.IsDir() { + err := c.collectFiles(fi.Filename(), handleFiles) + if err != nil { + return err + } + } else { + f, active := c.newFileInfo(fi, bundleNot) + if active { + handleFiles(f) + } + } + } + + return nil +} + +func (c *capturer) readDir(dirname string) (pathLangFileFis, error) { + if c.sourceSpec.IgnoreFile(dirname) { + return nil, nil + } + + dir, err := c.fs.Open(dirname) + if err != nil { + return nil, err + } + defer dir.Close() + fis, err := dir.Readdir(-1) + if err != nil { + return nil, err + } + + pfis := make(pathLangFileFis, 0, len(fis)) + + for _, fi := range fis { + fip := fi.(pathLangFileFi) + + if !c.sourceSpec.IgnoreFile(fip.Filename()) { + + err := c.resolveRealPathIn(fip) + + if err != nil { + // It may have been deleted in the meantime. + if err == errSkipCyclicDir || os.IsNotExist(err) { + continue + } + return nil, err + } + + pfis = append(pfis, fip) + } + } + + return pfis, nil +} + +func (c *capturer) newFileInfo(fi pathLangFileFi, tp bundleDirType) (*fileInfo, bool) { + f := newFileInfo(c.sourceSpec, "", "", fi, tp) + return f, !f.disabled +} + +type pathLangFile interface { + hugofs.LanguageAnnouncer + hugofs.FilePather +} + +type pathLangFileFi interface { + os.FileInfo + pathLangFile +} + +type pathLangFileFis []pathLangFileFi + +type bundleDirs struct { + tp bundleDirType + // Maps languages to bundles. + bundles map[string]*bundleDir + + // Keeps track of language overrides for non-content files, e.g. logo.en.png. + langOverrides map[string]bool + + c *capturer +} + +func newBundleDirs(tp bundleDirType, c *capturer) *bundleDirs { + return &bundleDirs{tp: tp, bundles: make(map[string]*bundleDir), langOverrides: make(map[string]bool), c: c} +} + +type bundleDir struct { + tp bundleDirType + fi *fileInfo + + resources map[string]*fileInfo +} + +func (b bundleDir) clone() *bundleDir { + b.resources = make(map[string]*fileInfo) + fic := *b.fi + b.fi = &fic + return &b +} + +func newBundleDir(fi *fileInfo, bundleType bundleDirType) *bundleDir { + return &bundleDir{fi: fi, tp: bundleType, resources: make(map[string]*fileInfo)} +} + +func (b *bundleDirs) addBundleContentFile(fi *fileInfo) { + dir, found := b.bundles[fi.Lang()] + if !found { + // Every bundled content file needs a bundle header. + // If one does not exist in its language, we pick the default + // language version, or a random one if that doesn't exist, either. + tl := b.c.sourceSpec.DefaultContentLanguage + ldir, found := b.bundles[tl] + if !found { + // Just pick one. + for _, v := range b.bundles { + ldir = v + break + } + } + + if ldir == nil { + panic(fmt.Sprintf("bundle not found for file %q", fi.Filename())) + } + + dir = ldir.clone() + dir.fi.overriddenLang = fi.Lang() + b.bundles[fi.Lang()] = dir + } + + dir.resources[fi.Path()] = fi +} + +func (b *bundleDirs) addBundleFiles(fi *fileInfo) { + dir := filepath.ToSlash(fi.Dir()) + p := dir + fi.TranslationBaseName() + "." + fi.Ext() + for lang, bdir := range b.bundles { + key := path.Join(lang, p) + + // Given mypage.de.md (German translation) and mypage.md we pick the most + // specific for that language. + if fi.Lang() == lang || !b.langOverrides[key] { + bdir.resources[key] = fi + } + b.langOverrides[key] = true + } +} + +func (b *bundleDirs) addBundleHeader(fi *fileInfo) { + b.bundles[fi.Lang()] = newBundleDir(fi, b.tp) +} + +func (c *capturer) isSeen(dirname string) bool { + c.seenMu.Lock() + defer c.seenMu.Unlock() + seen := c.seen[dirname] + c.seen[dirname] = true + if seen { + c.logger.WARN.Printf("Content dir %q already processed; skipped to avoid infinite recursion.", dirname) + return true + + } + return false +} + +func (c *capturer) resolveRealPath(path string) (pathLangFileFi, error) { + fileInfo, err := c.lstatIfPossible(path) + if err != nil { + return nil, err + } + return fileInfo, c.resolveRealPathIn(fileInfo) +} + +func (c *capturer) resolveRealPathIn(fileInfo pathLangFileFi) error { + + basePath := fileInfo.BaseDir() + path := fileInfo.Filename() + + realPath := path + + if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink { + link, err := filepath.EvalSymlinks(path) + if err != nil { + return _errors.Wrapf(err, "Cannot read symbolic link %q, error was:", path) + } + + // This is a file on the outside of any base fs, so we have to use the os package. + sfi, err := os.Stat(link) + if err != nil { + return _errors.Wrapf(err, "Cannot stat %q, error was:", link) + } + + // TODO(bep) improve all of this. + if a, ok := fileInfo.(*hugofs.LanguageFileInfo); ok { + a.FileInfo = sfi + } + + realPath = link + + if realPath != path && sfi.IsDir() && c.isSeen(realPath) { + // Avoid cyclic symlinks. + // Note that this may prevent some uses that isn't cyclic and also + // potential useful, but this implementation is both robust and simple: + // We stop at the first directory that we have seen before, e.g. + // /content/blog will only be processed once. + return errSkipCyclicDir + } + + if c.contentChanges != nil { + // Keep track of symbolic links in watch mode. + var from, to string + if sfi.IsDir() { + from = realPath + to = path + + if !strings.HasSuffix(to, helpers.FilePathSeparator) { + to = to + helpers.FilePathSeparator + } + if !strings.HasSuffix(from, helpers.FilePathSeparator) { + from = from + helpers.FilePathSeparator + } + + if !strings.HasSuffix(basePath, helpers.FilePathSeparator) { + basePath = basePath + helpers.FilePathSeparator + } + + if strings.HasPrefix(from, basePath) { + // With symbolic links inside /content we need to keep + // a reference to both. This may be confusing with --navigateToChanged + // but the user has chosen this him or herself. + c.contentChanges.addSymbolicLinkMapping(from, from) + } + + } else { + from = realPath + to = path + } + + c.contentChanges.addSymbolicLinkMapping(from, to) + } + } + + return nil +} + +func (c *capturer) lstatIfPossible(path string) (pathLangFileFi, error) { + fi, err := helpers.LstatIfPossible(c.fs, path) + if err != nil { + return nil, err + } + return fi.(pathLangFileFi), nil +} diff --git a/hugolib/pagebundler_capture_test.go b/hugolib/pagebundler_capture_test.go new file mode 100644 index 00000000..d6128352 --- /dev/null +++ b/hugolib/pagebundler_capture_test.go @@ -0,0 +1,274 @@ +// Copyright 2017-present The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "os" + "path" + "path/filepath" + "sort" + + "github.com/gohugoio/hugo/common/loggers" + + "runtime" + "strings" + "sync" + "testing" + + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/source" + "github.com/stretchr/testify/require" +) + +type storeFilenames struct { + sync.Mutex + filenames []string + copyNames []string + dirKeys []string +} + +func (s *storeFilenames) handleSingles(fis ...*fileInfo) { + s.Lock() + defer s.Unlock() + for _, fi := range fis { + s.filenames = append(s.filenames, filepath.ToSlash(fi.Filename())) + } +} + +func (s *storeFilenames) handleBundles(d *bundleDirs) { + s.Lock() + defer s.Unlock() + var keys []string + for _, b := range d.bundles { + res := make([]string, len(b.resources)) + i := 0 + for _, r := range b.resources { + res[i] = path.Join(r.Lang(), filepath.ToSlash(r.Filename())) + i++ + } + sort.Strings(res) + keys = append(keys, path.Join("__bundle", b.fi.Lang(), filepath.ToSlash(b.fi.Filename()), "resources", strings.Join(res, "|"))) + } + s.dirKeys = append(s.dirKeys, keys...) +} + +func (s *storeFilenames) handleCopyFiles(files ...pathLangFile) { + s.Lock() + defer s.Unlock() + for _, file := range files { + s.copyNames = append(s.copyNames, filepath.ToSlash(file.Filename())) + } +} + +func (s *storeFilenames) sortedStr() string { + s.Lock() + defer s.Unlock() + sort.Strings(s.filenames) + sort.Strings(s.dirKeys) + sort.Strings(s.copyNames) + return "\nF:\n" + strings.Join(s.filenames, "\n") + "\nD:\n" + strings.Join(s.dirKeys, "\n") + + "\nC:\n" + strings.Join(s.copyNames, "\n") + "\n" +} + +func TestPageBundlerCaptureSymlinks(t *testing.T) { + if runtime.GOOS == "windows" && os.Getenv("CI") == "" { + t.Skip("Skip TestPageBundlerCaptureSymlinks as os.Symlink needs administrator rights on Windows") + } + + assert := require.New(t) + ps, clean, workDir := newTestBundleSymbolicSources(t) + sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) + defer clean() + + fileStore := &storeFilenames{} + logger := loggers.NewErrorLogger() + c := newCapturer(logger, sourceSpec, fileStore, nil) + + assert.NoError(c.capture()) + + expected := ` +F: +/base/a/page_s.md +/base/a/regular.md +/base/symbolic1/s1.md +/base/symbolic1/s2.md +/base/symbolic3/circus/a/page_s.md +/base/symbolic3/circus/a/regular.md +D: +__bundle/en/base/symbolic2/a1/index.md/resources/en/base/symbolic2/a1/logo.png|en/base/symbolic2/a1/page.md +C: +/base/symbolic3/s1.png +/base/symbolic3/s2.png +` + + got := strings.Replace(fileStore.sortedStr(), filepath.ToSlash(workDir), "", -1) + got = strings.Replace(got, "//", "/", -1) + + if expected != got { + diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got)) + t.Log(got) + t.Fatalf("Failed:\n%s", diff) + } +} + +func TestPageBundlerCaptureBasic(t *testing.T) { + t.Parallel() + + assert := require.New(t) + fs, cfg := newTestBundleSources(t) + assert.NoError(loadDefaultSettingsFor(cfg)) + assert.NoError(loadLanguageSettings(cfg, nil)) + ps, err := helpers.NewPathSpec(fs, cfg) + assert.NoError(err) + + sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) + + fileStore := &storeFilenames{} + + c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil) + + assert.NoError(c.capture()) + + expected := ` +F: +/work/base/_1.md +/work/base/a/1.md +/work/base/a/2.md +/work/base/assets/pages/mypage.md +D: +__bundle/en/work/base/_index.md/resources/en/work/base/_1.png +__bundle/en/work/base/a/b/index.md/resources/en/work/base/a/b/ab1.md +__bundle/en/work/base/b/my-bundle/index.md/resources/en/work/base/b/my-bundle/1.md|en/work/base/b/my-bundle/2.md|en/work/base/b/my-bundle/c/logo.png|en/work/base/b/my-bundle/custom-mime.bep|en/work/base/b/my-bundle/sunset1.jpg|en/work/base/b/my-bundle/sunset2.jpg +__bundle/en/work/base/c/bundle/index.md/resources/en/work/base/c/bundle/logo-은행.png +__bundle/en/work/base/root/index.md/resources/en/work/base/root/1.md|en/work/base/root/c/logo.png +C: +/work/base/assets/pic1.png +/work/base/assets/pic2.png +/work/base/images/hugo-logo.png +` + + got := fileStore.sortedStr() + + if expected != got { + diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got)) + t.Log(got) + t.Fatalf("Failed:\n%s", diff) + } +} + +func TestPageBundlerCaptureMultilingual(t *testing.T) { + t.Parallel() + + assert := require.New(t) + fs, cfg := newTestBundleSourcesMultilingual(t) + assert.NoError(loadDefaultSettingsFor(cfg)) + assert.NoError(loadLanguageSettings(cfg, nil)) + + ps, err := helpers.NewPathSpec(fs, cfg) + assert.NoError(err) + + sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) + fileStore := &storeFilenames{} + c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil) + + assert.NoError(c.capture()) + + expected := ` +F: +/work/base/1s/mypage.md +/work/base/1s/mypage.nn.md +/work/base/bb/_1.md +/work/base/bb/_1.nn.md +/work/base/bb/en.md +/work/base/bc/page.md +/work/base/bc/page.nn.md +/work/base/be/_index.md +/work/base/be/page.md +/work/base/be/page.nn.md +D: +__bundle/en/work/base/bb/_index.md/resources/en/work/base/bb/a.png|en/work/base/bb/b.png|nn/work/base/bb/c.nn.png +__bundle/en/work/base/bc/_index.md/resources/en/work/base/bc/logo-bc.png +__bundle/en/work/base/bd/index.md/resources/en/work/base/bd/page.md +__bundle/en/work/base/bf/my-bf-bundle/index.md/resources/en/work/base/bf/my-bf-bundle/page.md +__bundle/en/work/base/lb/index.md/resources/en/work/base/lb/1.md|en/work/base/lb/2.md|en/work/base/lb/c/d/deep.png|en/work/base/lb/c/logo.png|en/work/base/lb/c/one.png|en/work/base/lb/c/page.md +__bundle/nn/work/base/bb/_index.nn.md/resources/en/work/base/bb/a.png|nn/work/base/bb/b.nn.png|nn/work/base/bb/c.nn.png +__bundle/nn/work/base/bd/index.md/resources/nn/work/base/bd/page.nn.md +__bundle/nn/work/base/bf/my-bf-bundle/index.nn.md/resources +__bundle/nn/work/base/lb/index.nn.md/resources/en/work/base/lb/c/d/deep.png|en/work/base/lb/c/one.png|nn/work/base/lb/2.nn.md|nn/work/base/lb/c/logo.nn.png +C: +/work/base/1s/mylogo.png +/work/base/bb/b/d.nn.png +` + + got := fileStore.sortedStr() + + if expected != got { + diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got)) + t.Log(got) + t.Fatalf("Failed:\n%s", strings.Join(diff, "\n")) + } + +} + +type noOpFileStore int + +func (noOpFileStore) handleSingles(fis ...*fileInfo) {} +func (noOpFileStore) handleBundles(b *bundleDirs) {} +func (noOpFileStore) handleCopyFiles(files ...pathLangFile) {} + +func BenchmarkPageBundlerCapture(b *testing.B) { + capturers := make([]*capturer, b.N) + + for i := 0; i < b.N; i++ { + cfg, fs := newTestCfg() + ps, _ := helpers.NewPathSpec(fs, cfg) + sourceSpec := source.NewSourceSpec(ps, fs.Source) + + base := fmt.Sprintf("base%d", i) + for j := 1; j <= 5; j++ { + js := fmt.Sprintf("j%d", j) + writeSource(b, fs, filepath.Join(base, js, "index.md"), "content") + writeSource(b, fs, filepath.Join(base, js, "logo1.png"), "content") + writeSource(b, fs, filepath.Join(base, js, "sub", "logo2.png"), "content") + writeSource(b, fs, filepath.Join(base, js, "section", "_index.md"), "content") + writeSource(b, fs, filepath.Join(base, js, "section", "logo.png"), "content") + writeSource(b, fs, filepath.Join(base, js, "section", "sub", "logo.png"), "content") + + for k := 1; k <= 5; k++ { + ks := fmt.Sprintf("k%d", k) + writeSource(b, fs, filepath.Join(base, js, ks, "logo1.png"), "content") + writeSource(b, fs, filepath.Join(base, js, "section", ks, "logo.png"), "content") + } + } + + for i := 1; i <= 5; i++ { + writeSource(b, fs, filepath.Join(base, "assetsonly", fmt.Sprintf("image%d.png", i)), "image") + } + + for i := 1; i <= 5; i++ { + writeSource(b, fs, filepath.Join(base, "contentonly", fmt.Sprintf("c%d.md", i)), "content") + } + + capturers[i] = newCapturer(loggers.NewErrorLogger(), sourceSpec, new(noOpFileStore), nil, base) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := capturers[i].capture() + if err != nil { + b.Fatal(err) + } + } +} diff --git a/hugolib/pagebundler_handlers.go b/hugolib/pagebundler_handlers.go new file mode 100644 index 00000000..2ab0ebaf --- /dev/null +++ b/hugolib/pagebundler_handlers.go @@ -0,0 +1,345 @@ +// Copyright 2017-present The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "errors" + "fmt" + "path/filepath" + "sort" + + "strings" + + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/resource" +) + +var ( + // This should be the only list of valid extensions for content files. + contentFileExtensions = []string{ + "html", "htm", + "mdown", "markdown", "md", + "asciidoc", "adoc", "ad", + "rest", "rst", + "mmark", + "org", + "pandoc", "pdc"} + + contentFileExtensionsSet map[string]bool +) + +func init() { + contentFileExtensionsSet = make(map[string]bool) + for _, ext := range contentFileExtensions { + contentFileExtensionsSet[ext] = true + } +} + +func newHandlerChain(s *Site) contentHandler { + c := &contentHandlers{s: s} + + contentFlow := c.parsePage(c.processFirstMatch( + // Handles all files with a content file extension. See above. + c.handlePageContent(), + + // Every HTML file without front matter will be passed on to this handler. + c.handleHTMLContent(), + )) + + c.rootHandler = c.processFirstMatch( + contentFlow, + + // Creates a file resource (image, CSS etc.) if there is a parent + // page set on the current context. + c.createResource(), + + // Everything that isn't handled above, will just be copied + // to destination. + c.copyFile(), + ) + + return c.rootHandler + +} + +type contentHandlers struct { + s *Site + rootHandler contentHandler +} + +func (c *contentHandlers) processFirstMatch(handlers ...contentHandler) func(ctx *handlerContext) handlerResult { + return func(ctx *handlerContext) handlerResult { + for _, h := range handlers { + res := h(ctx) + if res.handled || res.err != nil { + return res + } + } + return handlerResult{err: errors.New("no matching handler found")} + } +} + +type handlerContext struct { + // These are the pages stored in Site. + pages chan<- *Page + + doNotAddToSiteCollections bool + + currentPage *Page + parentPage *Page + + bundle *bundleDir + + source *fileInfo + + // Relative path to the target. + target string +} + +func (c *handlerContext) ext() string { + if c.currentPage != nil { + if c.currentPage.Markup != "" { + return c.currentPage.Markup + } + return c.currentPage.Ext() + } + + if c.bundle != nil { + return c.bundle.fi.Ext() + } else { + return c.source.Ext() + } +} + +func (c *handlerContext) targetPath() string { + if c.target != "" { + return c.target + } + + return c.source.Filename() +} + +func (c *handlerContext) file() *fileInfo { + if c.bundle != nil { + return c.bundle.fi + } + + return c.source +} + +// Create a copy with the current context as its parent. +func (c handlerContext) childCtx(fi *fileInfo) *handlerContext { + if c.currentPage == nil { + panic("Need a Page to create a child context") + } + + c.target = strings.TrimPrefix(fi.Path(), c.bundle.fi.Dir()) + c.source = fi + + c.doNotAddToSiteCollections = c.bundle != nil && c.bundle.tp != bundleBranch + + c.bundle = nil + + c.parentPage = c.currentPage + c.currentPage = nil + + return &c +} + +func (c *handlerContext) supports(exts ...string) bool { + ext := c.ext() + for _, s := range exts { + if s == ext { + return true + } + } + + return false +} + +func (c *handlerContext) isContentFile() bool { + return contentFileExtensionsSet[c.ext()] +} + +type ( + handlerResult struct { + err error + handled bool + resource resource.Resource + } + + contentHandler func(ctx *handlerContext) handlerResult +) + +var ( + notHandled handlerResult +) + +func (c *contentHandlers) parsePage(h contentHandler) contentHandler { + return func(ctx *handlerContext) handlerResult { + if !ctx.isContentFile() { + return notHandled + } + + result := handlerResult{handled: true} + fi := ctx.file() + + f, err := fi.Open() + if err != nil { + return handlerResult{err: fmt.Errorf("(%s) failed to open content file: %s", fi.Filename(), err)} + } + defer f.Close() + + p := c.s.newPageFromFile(fi) + + _, err = p.ReadFrom(f) + if err != nil { + return handlerResult{err: err} + } + + if !p.shouldBuild() { + if !ctx.doNotAddToSiteCollections { + ctx.pages <- p + } + return result + } + + ctx.currentPage = p + + if ctx.bundle != nil { + // Add the bundled files + for _, fi := range ctx.bundle.resources { + childCtx := ctx.childCtx(fi) + res := c.rootHandler(childCtx) + if res.err != nil { + return res + } + if res.resource != nil { + if pageResource, ok := res.resource.(*Page); ok { + pageResource.resourcePath = filepath.ToSlash(childCtx.target) + pageResource.parent = p + } + p.Resources = append(p.Resources, res.resource) + } + } + + sort.SliceStable(p.Resources, func(i, j int) bool { + if p.Resources[i].ResourceType() < p.Resources[j].ResourceType() { + return true + } + + p1, ok1 := p.Resources[i].(*Page) + p2, ok2 := p.Resources[j].(*Page) + + if ok1 != ok2 { + return ok2 + } + + if ok1 { + return defaultPageSort(p1, p2) + } + + return p.Resources[i].RelPermalink() < p.Resources[j].RelPermalink() + }) + + // Assign metadata from front matter if set + if len(p.resourcesMetadata) > 0 { + resource.AssignMetadata(p.resourcesMetadata, p.Resources...) + } + + } + + return h(ctx) + } +} + +func (c *contentHandlers) handlePageContent() contentHandler { + return func(ctx *handlerContext) handlerResult { + if ctx.supports("html", "htm") { + return notHandled + } + + p := ctx.currentPage + + if c.s.Cfg.GetBool("enableEmoji") { + p.workContent = helpers.Emojify(p.workContent) + } + + p.workContent = p.renderContent(p.workContent) + + tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.workContent) + p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents) + p.workContent = tmpContent + + if !ctx.doNotAddToSiteCollections { + ctx.pages <- p + } + + return handlerResult{handled: true, resource: p} + } +} + +func (c *contentHandlers) handleHTMLContent() contentHandler { + return func(ctx *handlerContext) handlerResult { + if !ctx.supports("html", "htm") { + return notHandled + } + + p := ctx.currentPage + + if !ctx.doNotAddToSiteCollections { + ctx.pages <- p + } + + return handlerResult{handled: true, resource: p} + } +} + +func (c *contentHandlers) createResource() contentHandler { + return func(ctx *handlerContext) handlerResult { + if ctx.parentPage == nil { + return notHandled + } + + resource, err := c.s.ResourceSpec.New( + resource.ResourceSourceDescriptor{ + TargetPathBuilder: ctx.parentPage.subResourceTargetPathFactory, + SourceFile: ctx.source, + RelTargetFilename: ctx.target, + URLBase: c.s.GetURLLanguageBasePath(), + TargetBasePaths: []string{c.s.GetTargetLanguageBasePath()}, + }) + + return handlerResult{err: err, handled: true, resource: resource} + } +} + +func (c *contentHandlers) copyFile() contentHandler { + return func(ctx *handlerContext) handlerResult { + f, err := c.s.BaseFs.Content.Fs.Open(ctx.source.Filename()) + if err != nil { + err := fmt.Errorf("failed to open file in copyFile: %s", err) + return handlerResult{err: err} + } + + target := ctx.targetPath() + + defer f.Close() + if err := c.s.publish(&c.s.PathSpec.ProcessingStats.Files, target, f); err != nil { + return handlerResult{err: err} + } + + return handlerResult{handled: true} + } +} diff --git a/hugolib/pagebundler_test.go b/hugolib/pagebundler_test.go new file mode 100644 index 00000000..1eb5aacd --- /dev/null +++ b/hugolib/pagebundler_test.go @@ -0,0 +1,751 @@ +// Copyright 2017-present The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "github.com/gohugoio/hugo/common/loggers" + + "os" + "runtime" + "testing" + + "github.com/gohugoio/hugo/helpers" + + "io" + + "github.com/spf13/afero" + + "github.com/gohugoio/hugo/media" + + "path/filepath" + + "fmt" + + "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/hugofs" + "github.com/spf13/viper" + + "github.com/stretchr/testify/require" +) + +func TestPageBundlerSiteRegular(t *testing.T) { + t.Parallel() + + for _, ugly := range []bool{false, true} { + t.Run(fmt.Sprintf("ugly=%t", ugly), + func(t *testing.T) { + + assert := require.New(t) + fs, cfg := newTestBundleSources(t) + assert.NoError(loadDefaultSettingsFor(cfg)) + assert.NoError(loadLanguageSettings(cfg, nil)) + + cfg.Set("permalinks", map[string]string{ + "a": ":sections/:filename", + "b": ":year/:slug/", + "c": ":sections/:slug", + "": ":filename/", + }) + + cfg.Set("outputFormats", map[string]interface{}{ + "CUSTOMO": map[string]interface{}{ + "mediaType": media.HTMLType, + "baseName": "cindex", + "path": "cpath", + }, + }) + + cfg.Set("outputs", map[string]interface{}{ + "home": []string{"HTML", "CUSTOMO"}, + "page": []string{"HTML", "CUSTOMO"}, + "section": []string{"HTML", "CUSTOMO"}, + }) + + cfg.Set("uglyURLs", ugly) + + s := buildSingleSite(t, deps.DepsCfg{Logger: loggers.NewWarningLogger(), Fs: fs, Cfg: cfg}, BuildCfg{}) + + th := testHelper{s.Cfg, s.Fs, t} + + assert.Len(s.RegularPages, 8) + + singlePage := s.getPage(KindPage, "a/1.md") + assert.Equal("", singlePage.BundleType()) + + assert.NotNil(singlePage) + assert.Equal(singlePage, s.getPage("page", "a/1")) + assert.Equal(singlePage, s.getPage("page", "1")) + + assert.Contains(singlePage.content(), "TheContent") + + if ugly { + assert.Equal("/a/1.html", singlePage.RelPermalink()) + th.assertFileContent(filepath.FromSlash("/work/public/a/1.html"), "TheContent") + + } else { + assert.Equal("/a/1/", singlePage.RelPermalink()) + th.assertFileContent(filepath.FromSlash("/work/public/a/1/index.html"), "TheContent") + } + + th.assertFileContent(filepath.FromSlash("/work/public/images/hugo-logo.png"), "content") + + // This should be just copied to destination. + th.assertFileContent(filepath.FromSlash("/work/public/assets/pic1.png"), "content") + + leafBundle1 := s.getPage(KindPage, "b/my-bundle/index.md") + assert.NotNil(leafBundle1) + assert.Equal("leaf", leafBundle1.BundleType()) + assert.Equal("b", leafBundle1.Section()) + sectionB := s.getPage(KindSection, "b") + assert.NotNil(sectionB) + home, _ := s.Info.Home() + assert.Equal("branch", home.BundleType()) + + // This is a root bundle and should live in the "home section" + // See https://github.com/gohugoio/hugo/issues/4332 + rootBundle := s.getPage(KindPage, "root") + assert.NotNil(rootBundle) + assert.True(rootBundle.Parent().IsHome()) + if ugly { + assert.Equal("/root.html", rootBundle.RelPermalink()) + } else { + assert.Equal("/root/", rootBundle.RelPermalink()) + } + + leafBundle2 := s.getPage(KindPage, "a/b/index.md") + assert.NotNil(leafBundle2) + unicodeBundle := s.getPage(KindPage, "c/bundle/index.md") + assert.NotNil(unicodeBundle) + + pageResources := leafBundle1.Resources.ByType(pageResourceType) + assert.Len(pageResources, 2) + firstPage := pageResources[0].(*Page) + secondPage := pageResources[1].(*Page) + assert.Equal(filepath.FromSlash("/work/base/b/my-bundle/1.md"), firstPage.pathOrTitle(), secondPage.pathOrTitle()) + assert.Contains(firstPage.content(), "TheContent") + assert.Equal(6, len(leafBundle1.Resources)) + + // Verify shortcode in bundled page + assert.Contains(secondPage.content(), filepath.FromSlash("MyShort in b/my-bundle/2.md")) + + // https://github.com/gohugoio/hugo/issues/4582 + assert.Equal(leafBundle1, firstPage.Parent()) + assert.Equal(leafBundle1, secondPage.Parent()) + + assert.Equal(firstPage, pageResources.GetMatch("1*")) + assert.Equal(secondPage, pageResources.GetMatch("2*")) + assert.Nil(pageResources.GetMatch("doesnotexist*")) + + imageResources := leafBundle1.Resources.ByType("image") + assert.Equal(3, len(imageResources)) + image := imageResources[0] + + altFormat := leafBundle1.OutputFormats().Get("CUSTOMO") + assert.NotNil(altFormat) + + assert.Equal("https://example.com/2017/pageslug/c/logo.png", image.Permalink()) + + th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content") + th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/c/logo.png"), "content") + + // Custom media type defined in site config. + assert.Len(leafBundle1.Resources.ByType("bepsays"), 1) + + if ugly { + assert.Equal("/2017/pageslug.html", leafBundle1.RelPermalink()) + th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug.html"), + "TheContent", + "Sunset RelPermalink: /2017/pageslug/sunset1.jpg", + "Thumb Width: 123", + "Thumb Name: my-sunset-1", + "Short Sunset RelPermalink: /2017/pageslug/sunset2.jpg", + "Short Thumb Width: 56", + "1: Image Title: Sunset Galore 1", + "1: Image Params: map[myparam:My Sunny Param]", + "2: Image Title: Sunset Galore 2", + "2: Image Params: map[myparam:My Sunny Param]", + "1: Image myParam: Lower: My Sunny Param Caps: My Sunny Param", + ) + th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug.html"), "TheContent") + + assert.Equal("/a/b.html", leafBundle2.RelPermalink()) + + // 은행 + assert.Equal("/c/%EC%9D%80%ED%96%89.html", unicodeBundle.RelPermalink()) + th.assertFileContent(filepath.FromSlash("/work/public/c/은행.html"), "Content for 은행") + th.assertFileContent(filepath.FromSlash("/work/public/c/은행/logo-은행.png"), "은행 PNG") + + } else { + assert.Equal("/2017/pageslug/", leafBundle1.RelPermalink()) + th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "TheContent") + th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/cindex.html"), "TheContent") + th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "Single Title") + th.assertFileContent(filepath.FromSlash("/work/public/root/index.html"), "Single Title") + + assert.Equal("/a/b/", leafBundle2.RelPermalink()) + + } + + }) + } + +} + +func TestPageBundlerSiteMultilingual(t *testing.T) { + t.Parallel() + + for _, ugly := range []bool{false, true} { + t.Run(fmt.Sprintf("ugly=%t", ugly), + func(t *testing.T) { + + assert := require.New(t) + fs, cfg := newTestBundleSourcesMultilingual(t) + cfg.Set("uglyURLs", ugly) + + assert.NoError(loadDefaultSettingsFor(cfg)) + assert.NoError(loadLanguageSettings(cfg, nil)) + sites, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg}) + assert.NoError(err) + assert.Equal(2, len(sites.Sites)) + + assert.NoError(sites.Build(BuildCfg{})) + + s := sites.Sites[0] + + assert.Equal(8, len(s.RegularPages)) + assert.Equal(16, len(s.Pages)) + assert.Equal(31, len(s.AllPages)) + + bundleWithSubPath := s.getPage(KindPage, "lb/index") + assert.NotNil(bundleWithSubPath) + + // See https://github.com/gohugoio/hugo/issues/4312 + // Before that issue: + // A bundle in a/b/index.en.md + // a/b/index.en.md => OK + // a/b/index => OK + // index.en.md => ambigous, but OK. + // With bundles, the file name has little meaning, the folder it lives in does. So this should also work: + // a/b + // and probably also just b (aka "my-bundle") + // These may also be translated, so we also need to test that. + // "bf", "my-bf-bundle", "index.md + nn + bfBundle := s.getPage(KindPage, "bf/my-bf-bundle/index") + assert.NotNil(bfBundle) + assert.Equal("en", bfBundle.Lang()) + assert.Equal(bfBundle, s.getPage(KindPage, "bf/my-bf-bundle/index.md")) + assert.Equal(bfBundle, s.getPage(KindPage, "bf/my-bf-bundle")) + assert.Equal(bfBundle, s.getPage(KindPage, "my-bf-bundle")) + + nnSite := sites.Sites[1] + assert.Equal(7, len(nnSite.RegularPages)) + + bfBundleNN := nnSite.getPage(KindPage, "bf/my-bf-bundle/index") + assert.NotNil(bfBundleNN) + assert.Equal("nn", bfBundleNN.Lang()) + assert.Equal(bfBundleNN, nnSite.getPage(KindPage, "bf/my-bf-bundle/index.nn.md")) + assert.Equal(bfBundleNN, nnSite.getPage(KindPage, "bf/my-bf-bundle")) + assert.Equal(bfBundleNN, nnSite.getPage(KindPage, "my-bf-bundle")) + + // See https://github.com/gohugoio/hugo/issues/4295 + // Every resource should have its Name prefixed with its base folder. + cBundleResources := bundleWithSubPath.Resources.Match("c/**") + assert.Equal(4, len(cBundleResources)) + bundlePage := bundleWithSubPath.Resources.GetMatch("c/page*") + assert.NotNil(bundlePage) + assert.IsType(&Page{}, bundlePage) + + }) + } +} + +func TestMultilingualDisableDefaultLanguage(t *testing.T) { + t.Parallel() + + assert := require.New(t) + _, cfg := newTestBundleSourcesMultilingual(t) + + cfg.Set("disableLanguages", []string{"en"}) + + err := loadDefaultSettingsFor(cfg) + assert.NoError(err) + err = loadLanguageSettings(cfg, nil) + assert.Error(err) + assert.Contains(err.Error(), "cannot disable default language") +} + +func TestMultilingualDisableLanguage(t *testing.T) { + t.Parallel() + + assert := require.New(t) + fs, cfg := newTestBundleSourcesMultilingual(t) + cfg.Set("disableLanguages", []string{"nn"}) + + assert.NoError(loadDefaultSettingsFor(cfg)) + assert.NoError(loadLanguageSettings(cfg, nil)) + + sites, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg}) + assert.NoError(err) + assert.Equal(1, len(sites.Sites)) + + assert.NoError(sites.Build(BuildCfg{})) + + s := sites.Sites[0] + + assert.Equal(8, len(s.RegularPages)) + assert.Equal(16, len(s.Pages)) + // No nn pages + assert.Equal(16, len(s.AllPages)) + for _, p := range s.rawAllPages { + assert.True(p.Lang() != "nn") + } + for _, p := range s.AllPages { + assert.True(p.Lang() != "nn") + } + +} + +func TestPageBundlerSiteWitSymbolicLinksInContent(t *testing.T) { + if runtime.GOOS == "windows" && os.Getenv("CI") == "" { + t.Skip("Skip TestPageBundlerSiteWitSymbolicLinksInContent as os.Symlink needs administrator rights on Windows") + } + + assert := require.New(t) + ps, clean, workDir := newTestBundleSymbolicSources(t) + defer clean() + + cfg := ps.Cfg + fs := ps.Fs + + s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: loggers.NewErrorLogger()}, BuildCfg{}) + + th := testHelper{s.Cfg, s.Fs, t} + + assert.Equal(7, len(s.RegularPages)) + a1Bundle := s.getPage(KindPage, "symbolic2/a1/index.md") + assert.NotNil(a1Bundle) + assert.Equal(2, len(a1Bundle.Resources)) + assert.Equal(1, len(a1Bundle.Resources.ByType(pageResourceType))) + + th.assertFileContent(filepath.FromSlash(workDir+"/public/a/page/index.html"), "TheContent") + th.assertFileContent(filepath.FromSlash(workDir+"/public/symbolic1/s1/index.html"), "TheContent") + th.assertFileContent(filepath.FromSlash(workDir+"/public/symbolic2/a1/index.html"), "TheContent") + +} + +func TestPageBundlerHeadless(t *testing.T) { + t.Parallel() + + cfg, fs := newTestCfg() + assert := require.New(t) + + workDir := "/work" + cfg.Set("workingDir", workDir) + cfg.Set("contentDir", "base") + cfg.Set("baseURL", "https://example.com") + + pageContent := `--- +title: "Bundle Galore" +slug: s1 +date: 2017-01-23 +--- + +TheContent. + +{{< myShort >}} +` + + writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "single.html"), "single {{ .Content }}") + writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "list.html"), "list") + writeSource(t, fs, filepath.Join(workDir, "layouts", "shortcodes", "myShort.html"), "SHORTCODE") + + writeSource(t, fs, filepath.Join(workDir, "base", "a", "index.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "a", "l1.png"), "PNG image") + writeSource(t, fs, filepath.Join(workDir, "base", "a", "l2.png"), "PNG image") + + writeSource(t, fs, filepath.Join(workDir, "base", "b", "index.md"), `--- +title: "Headless Bundle in Topless Bar" +slug: s2 +headless: true +date: 2017-01-23 +--- + +TheContent. +HEADLESS {{< myShort >}} +`) + writeSource(t, fs, filepath.Join(workDir, "base", "b", "l1.png"), "PNG image") + writeSource(t, fs, filepath.Join(workDir, "base", "b", "l2.png"), "PNG image") + writeSource(t, fs, filepath.Join(workDir, "base", "b", "p1.md"), pageContent) + + s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) + + assert.Equal(1, len(s.RegularPages)) + assert.Equal(1, len(s.headlessPages)) + + regular := s.getPage(KindPage, "a/index") + assert.Equal("/a/s1/", regular.RelPermalink()) + + headless := s.getPage(KindPage, "b/index") + assert.NotNil(headless) + assert.True(headless.headless) + assert.Equal("Headless Bundle in Topless Bar", headless.Title()) + assert.Equal("", headless.RelPermalink()) + assert.Equal("", headless.Permalink()) + assert.Contains(headless.content(), "HEADLESS SHORTCODE") + + headlessResources := headless.Resources + assert.Equal(3, len(headlessResources)) + assert.Equal(2, len(headlessResources.Match("l*"))) + pageResource := headlessResources.GetMatch("p*") + assert.NotNil(pageResource) + assert.IsType(&Page{}, pageResource) + p := pageResource.(*Page) + assert.Contains(p.content(), "SHORTCODE") + assert.Equal("p1.md", p.Name()) + + th := testHelper{s.Cfg, s.Fs, t} + + th.assertFileContent(filepath.FromSlash(workDir+"/public/a/s1/index.html"), "TheContent") + th.assertFileContent(filepath.FromSlash(workDir+"/public/a/s1/l1.png"), "PNG") + + th.assertFileNotExist(workDir + "/public/b/s2/index.html") + // But the bundled resources needs to be published + th.assertFileContent(filepath.FromSlash(workDir+"/public/b/s2/l1.png"), "PNG") + +} + +func newTestBundleSources(t *testing.T) (*hugofs.Fs, *viper.Viper) { + cfg, fs := newTestCfg() + assert := require.New(t) + + workDir := "/work" + cfg.Set("workingDir", workDir) + cfg.Set("contentDir", "base") + cfg.Set("baseURL", "https://example.com") + cfg.Set("mediaTypes", map[string]interface{}{ + "text/bepsays": map[string]interface{}{ + "suffixes": []string{"bep"}, + }, + }) + + pageContent := `--- +title: "Bundle Galore" +slug: pageslug +date: 2017-10-09 +--- + +TheContent. +` + + pageContentShortcode := `--- +title: "Bundle Galore" +slug: pageslug +date: 2017-10-09 +--- + +TheContent. + +{{< myShort >}} +` + + pageWithImageShortcodeAndResourceMetadataContent := `--- +title: "Bundle Galore" +slug: pageslug +date: 2017-10-09 +resources: +- src: "*.jpg" + name: "my-sunset-:counter" + title: "Sunset Galore :counter" + params: + myParam: "My Sunny Param" +--- + +TheContent. + +{{< myShort >}} +` + + pageContentNoSlug := `--- +title: "Bundle Galore #2" +date: 2017-10-09 +--- + +TheContent. +` + + singleLayout := ` +Single Title: {{ .Title }} +Content: {{ .Content }} +{{ $sunset := .Resources.GetMatch "my-sunset-1*" }} +{{ with $sunset }} +Sunset RelPermalink: {{ .RelPermalink }} +{{ $thumb := .Fill "123x123" }} +Thumb Width: {{ $thumb.Width }} +Thumb Name: {{ $thumb.Name }} +Thumb Title: {{ $thumb.Title }} +Thumb RelPermalink: {{ $thumb.RelPermalink }} +{{ end }} +{{ range $i, $e := .Resources.ByType "image" }} +{{ $i }}: Image Title: {{ .Title }} +{{ $i }}: Image Name: {{ .Name }} +{{ $i }}: Image Params: {{ printf "%v" .Params }} +{{ $i }}: Image myParam: Lower: {{ .Params.myparam }} Caps: {{ .Params.MYPARAM }} +{{ end }} +` + + myShort := ` +MyShort in {{ .Page.Path }}: +{{ $sunset := .Page.Resources.GetMatch "my-sunset-2*" }} +{{ with $sunset }} +Short Sunset RelPermalink: {{ .RelPermalink }} +{{ $thumb := .Fill "56x56" }} +Short Thumb Width: {{ $thumb.Width }} +{{ end }} +` + + listLayout := `{{ .Title }}|{{ .Content }}` + + writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "single.html"), singleLayout) + writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "list.html"), listLayout) + writeSource(t, fs, filepath.Join(workDir, "layouts", "shortcodes", "myShort.html"), myShort) + + writeSource(t, fs, filepath.Join(workDir, "base", "_index.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "_1.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "_1.png"), pageContent) + + writeSource(t, fs, filepath.Join(workDir, "base", "images", "hugo-logo.png"), "content") + writeSource(t, fs, filepath.Join(workDir, "base", "a", "2.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "a", "1.md"), pageContent) + + writeSource(t, fs, filepath.Join(workDir, "base", "a", "b", "index.md"), pageContentNoSlug) + writeSource(t, fs, filepath.Join(workDir, "base", "a", "b", "ab1.md"), pageContentNoSlug) + + // Mostly plain static assets in a folder with a page in a sub folder thrown in. + writeSource(t, fs, filepath.Join(workDir, "base", "assets", "pic1.png"), "content") + writeSource(t, fs, filepath.Join(workDir, "base", "assets", "pic2.png"), "content") + writeSource(t, fs, filepath.Join(workDir, "base", "assets", "pages", "mypage.md"), pageContent) + + // Bundle + writeSource(t, fs, filepath.Join(workDir, "base", "b", "my-bundle", "index.md"), pageWithImageShortcodeAndResourceMetadataContent) + writeSource(t, fs, filepath.Join(workDir, "base", "b", "my-bundle", "1.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "b", "my-bundle", "2.md"), pageContentShortcode) + writeSource(t, fs, filepath.Join(workDir, "base", "b", "my-bundle", "custom-mime.bep"), "bepsays") + writeSource(t, fs, filepath.Join(workDir, "base", "b", "my-bundle", "c", "logo.png"), "content") + + // Bundle with 은행 slug + // See https://github.com/gohugoio/hugo/issues/4241 + writeSource(t, fs, filepath.Join(workDir, "base", "c", "bundle", "index.md"), `--- +title: "은행 은행" +slug: 은행 +date: 2017-10-09 +--- + +Content for 은행. +`) + + // Bundle in root + writeSource(t, fs, filepath.Join(workDir, "base", "root", "index.md"), pageWithImageShortcodeAndResourceMetadataContent) + writeSource(t, fs, filepath.Join(workDir, "base", "root", "1.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "root", "c", "logo.png"), "content") + + writeSource(t, fs, filepath.Join(workDir, "base", "c", "bundle", "logo-은행.png"), "은행 PNG") + + // Write a real image into one of the bundle above. + src, err := os.Open("testdata/sunset.jpg") + assert.NoError(err) + + // We need 2 to test https://github.com/gohugoio/hugo/issues/4202 + out, err := fs.Source.Create(filepath.Join(workDir, "base", "b", "my-bundle", "sunset1.jpg")) + assert.NoError(err) + out2, err := fs.Source.Create(filepath.Join(workDir, "base", "b", "my-bundle", "sunset2.jpg")) + assert.NoError(err) + + _, err = io.Copy(out, src) + out.Close() + src.Seek(0, 0) + _, err = io.Copy(out2, src) + out2.Close() + src.Close() + assert.NoError(err) + + return fs, cfg + +} + +func newTestBundleSourcesMultilingual(t *testing.T) (*hugofs.Fs, *viper.Viper) { + cfg, fs := newTestCfg() + + workDir := "/work" + cfg.Set("workingDir", workDir) + cfg.Set("contentDir", "base") + cfg.Set("baseURL", "https://example.com") + cfg.Set("defaultContentLanguage", "en") + + langConfig := map[string]interface{}{ + "en": map[string]interface{}{ + "weight": 1, + "languageName": "English", + }, + "nn": map[string]interface{}{ + "weight": 2, + "languageName": "Nynorsk", + }, + } + + cfg.Set("languages", langConfig) + + pageContent := `--- +slug: pageslug +date: 2017-10-09 +--- + +TheContent. +` + + layout := `{{ .Title }}|{{ .Content }}|Lang: {{ .Site.Language.Lang }}` + + writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "single.html"), layout) + writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "list.html"), layout) + + writeSource(t, fs, filepath.Join(workDir, "base", "1s", "mypage.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "1s", "mypage.nn.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "1s", "mylogo.png"), "content") + + writeSource(t, fs, filepath.Join(workDir, "base", "bb", "_index.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "bb", "_index.nn.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "bb", "en.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "bb", "_1.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "bb", "_1.nn.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "bb", "a.png"), "content") + writeSource(t, fs, filepath.Join(workDir, "base", "bb", "b.png"), "content") + writeSource(t, fs, filepath.Join(workDir, "base", "bb", "b.nn.png"), "content") + writeSource(t, fs, filepath.Join(workDir, "base", "bb", "c.nn.png"), "content") + writeSource(t, fs, filepath.Join(workDir, "base", "bb", "b", "d.nn.png"), "content") + + writeSource(t, fs, filepath.Join(workDir, "base", "bc", "_index.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "bc", "page.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "bc", "logo-bc.png"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "bc", "page.nn.md"), pageContent) + + writeSource(t, fs, filepath.Join(workDir, "base", "bd", "index.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "bd", "page.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "bd", "page.nn.md"), pageContent) + + writeSource(t, fs, filepath.Join(workDir, "base", "be", "_index.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "be", "page.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "be", "page.nn.md"), pageContent) + + // Bundle leaf, multilingual + writeSource(t, fs, filepath.Join(workDir, "base", "lb", "index.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "lb", "index.nn.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "lb", "1.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "lb", "2.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "lb", "2.nn.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "lb", "c", "page.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "lb", "c", "logo.png"), "content") + writeSource(t, fs, filepath.Join(workDir, "base", "lb", "c", "logo.nn.png"), "content") + writeSource(t, fs, filepath.Join(workDir, "base", "lb", "c", "one.png"), "content") + writeSource(t, fs, filepath.Join(workDir, "base", "lb", "c", "d", "deep.png"), "content") + + //Translated bundle in some sensible sub path. + writeSource(t, fs, filepath.Join(workDir, "base", "bf", "my-bf-bundle", "index.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "bf", "my-bf-bundle", "index.nn.md"), pageContent) + writeSource(t, fs, filepath.Join(workDir, "base", "bf", "my-bf-bundle", "page.md"), pageContent) + + return fs, cfg +} + +func newTestBundleSymbolicSources(t *testing.T) (*helpers.PathSpec, func(), string) { + assert := require.New(t) + // We need to use the OS fs for this. + cfg := viper.New() + fs := hugofs.NewFrom(hugofs.Os, cfg) + fs.Destination = &afero.MemMapFs{} + loadDefaultSettingsFor(cfg) + + workDir, clean, err := createTempDir("hugosym") + assert.NoError(err) + + contentDir := "base" + cfg.Set("workingDir", workDir) + cfg.Set("contentDir", contentDir) + cfg.Set("baseURL", "https://example.com") + + if err := loadLanguageSettings(cfg, nil); err != nil { + t.Fatal(err) + } + + layout := `{{ .Title }}|{{ .Content }}` + pageContent := `--- +slug: %s +date: 2017-10-09 +--- + +TheContent. +` + + fs.Source.MkdirAll(filepath.Join(workDir, "layouts", "_default"), 0777) + fs.Source.MkdirAll(filepath.Join(workDir, contentDir), 0777) + fs.Source.MkdirAll(filepath.Join(workDir, contentDir, "a"), 0777) + for i := 1; i <= 3; i++ { + fs.Source.MkdirAll(filepath.Join(workDir, fmt.Sprintf("symcontent%d", i)), 0777) + + } + fs.Source.MkdirAll(filepath.Join(workDir, "symcontent2", "a1"), 0777) + + writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "single.html"), layout) + writeSource(t, fs, filepath.Join(workDir, "layouts", "_default", "list.html"), layout) + + writeSource(t, fs, filepath.Join(workDir, contentDir, "a", "regular.md"), fmt.Sprintf(pageContent, "a1")) + + // Regular files inside symlinked folder. + writeSource(t, fs, filepath.Join(workDir, "symcontent1", "s1.md"), fmt.Sprintf(pageContent, "s1")) + writeSource(t, fs, filepath.Join(workDir, "symcontent1", "s2.md"), fmt.Sprintf(pageContent, "s2")) + + // A bundle + writeSource(t, fs, filepath.Join(workDir, "symcontent2", "a1", "index.md"), fmt.Sprintf(pageContent, "")) + writeSource(t, fs, filepath.Join(workDir, "symcontent2", "a1", "page.md"), fmt.Sprintf(pageContent, "page")) + writeSource(t, fs, filepath.Join(workDir, "symcontent2", "a1", "logo.png"), "image") + + // Assets + writeSource(t, fs, filepath.Join(workDir, "symcontent3", "s1.png"), "image") + writeSource(t, fs, filepath.Join(workDir, "symcontent3", "s2.png"), "image") + + wd, _ := os.Getwd() + defer func() { + os.Chdir(wd) + }() + // Symlinked sections inside content. + os.Chdir(filepath.Join(workDir, contentDir)) + for i := 1; i <= 3; i++ { + assert.NoError(os.Symlink(filepath.FromSlash(fmt.Sprintf(("../symcontent%d"), i)), fmt.Sprintf("symbolic%d", i))) + } + + os.Chdir(filepath.Join(workDir, contentDir, "a")) + + // Create a symlink to one single content file + assert.NoError(os.Symlink(filepath.FromSlash("../../symcontent2/a1/page.md"), "page_s.md")) + + os.Chdir(filepath.FromSlash("../../symcontent3")) + + // Create a circular symlink. Will print some warnings. + assert.NoError(os.Symlink(filepath.Join("..", contentDir), filepath.FromSlash("circus"))) + + os.Chdir(workDir) + assert.NoError(err) + + ps, _ := helpers.NewPathSpec(fs, cfg) + + return ps, clean, workDir +} diff --git a/hugolib/pagecollections.go b/hugolib/pagecollections.go new file mode 100644 index 00000000..78325344 --- /dev/null +++ b/hugolib/pagecollections.go @@ -0,0 +1,341 @@ +// Copyright 2016 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "path" + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/cache" + "github.com/gohugoio/hugo/helpers" +) + +// PageCollections contains the page collections for a site. +type PageCollections struct { + // Includes only pages of all types, and only pages in the current language. + Pages Pages + + // Includes all pages in all languages, including the current one. + // Includes pages of all types. + AllPages Pages + + // A convenience cache for the traditional index types, taxonomies, home page etc. + // This is for the current language only. + indexPages Pages + + // A convenience cache for the regular pages. + // This is for the current language only. + RegularPages Pages + + // A convenience cache for the all the regular pages. + AllRegularPages Pages + + // Includes absolute all pages (of all types), including drafts etc. + rawAllPages Pages + + // Includes headless bundles, i.e. bundles that produce no output for its content page. + headlessPages Pages + + pageIndex *cache.Lazy +} + +// Get initializes the index if not already done so, then +// looks up the given page ref, returns nil if no value found. +func (c *PageCollections) getFromCache(ref string) (*Page, error) { + v, found, err := c.pageIndex.Get(ref) + if err != nil { + return nil, err + } + if !found { + return nil, nil + } + + p := v.(*Page) + + if p != ambiguityFlag { + return p, nil + } + return nil, fmt.Errorf("page reference %q is ambiguous", ref) +} + +var ambiguityFlag = &Page{Kind: kindUnknown, title: "ambiguity flag"} + +func (c *PageCollections) refreshPageCaches() { + c.indexPages = c.findPagesByKindNotIn(KindPage, c.Pages) + c.RegularPages = c.findPagesByKindIn(KindPage, c.Pages) + c.AllRegularPages = c.findPagesByKindIn(KindPage, c.AllPages) + + indexLoader := func() (map[string]interface{}, error) { + index := make(map[string]interface{}) + + add := func(ref string, p *Page) { + existing := index[ref] + if existing == nil { + index[ref] = p + } else if existing != ambiguityFlag && existing != p { + index[ref] = ambiguityFlag + } + } + + for _, pageCollection := range []Pages{c.RegularPages, c.headlessPages} { + for _, p := range pageCollection { + sourceRef := p.absoluteSourceRef() + + if sourceRef != "" { + // index the canonical ref + // e.g. /section/article.md + add(sourceRef, p) + } + + // Ref/Relref supports this potentially ambiguous lookup. + add(p.LogicalName(), p) + + translationBaseName := p.TranslationBaseName() + + dir, _ := path.Split(sourceRef) + dir = strings.TrimSuffix(dir, "/") + + if translationBaseName == "index" { + add(dir, p) + add(path.Base(dir), p) + } else { + add(translationBaseName, p) + } + + // We need a way to get to the current language version. + pathWithNoExtensions := path.Join(dir, translationBaseName) + add(pathWithNoExtensions, p) + } + } + + for _, p := range c.indexPages { + // index the canonical, unambiguous ref for any backing file + // e.g. /section/_index.md + sourceRef := p.absoluteSourceRef() + if sourceRef != "" { + add(sourceRef, p) + } + + ref := path.Join(p.sections...) + + // index the canonical, unambiguous virtual ref + // e.g. /section + // (this may already have been indexed above) + add("/"+ref, p) + } + + return index, nil + } + + c.pageIndex = cache.NewLazy(indexLoader) +} + +func newPageCollections() *PageCollections { + return &PageCollections{} +} + +func newPageCollectionsFromPages(pages Pages) *PageCollections { + return &PageCollections{rawAllPages: pages} +} + +// This is an adapter func for the old API with Kind as first argument. +// This is invoked when you do .Site.GetPage. We drop the Kind and fails +// if there are more than 2 arguments, which would be ambigous. +func (c *PageCollections) getPageOldVersion(ref ...string) (*Page, error) { + var refs []string + for _, r := range ref { + // A common construct in the wild is + // .Site.GetPage "home" "" or + // .Site.GetPage "home" "/" + if r != "" && r != "/" { + refs = append(refs, r) + } + } + + var key string + + if len(refs) > 2 { + // This was allowed in Hugo <= 0.44, but we cannot support this with the + // new API. This should be the most unusual case. + return nil, fmt.Errorf(`too many arguments to .Site.GetPage: %v. Use lookups on the form {{ .Site.GetPage "/posts/mypage-md" }}`, ref) + } + + if len(refs) == 0 || refs[0] == KindHome { + key = "/" + } else if len(refs) == 1 { + if len(ref) == 2 && refs[0] == KindSection { + // This is an old style reference to the "Home Page section". + // Typically fetched via {{ .Site.GetPage "section" .Section }} + // See https://github.com/gohugoio/hugo/issues/4989 + key = "/" + } else { + key = refs[0] + } + } else { + key = refs[1] + } + + key = filepath.ToSlash(key) + if !strings.HasPrefix(key, "/") { + key = "/" + key + } + + return c.getPageNew(nil, key) +} + +// Only used in tests. +func (c *PageCollections) getPage(typ string, sections ...string) *Page { + refs := append([]string{typ}, path.Join(sections...)) + p, _ := c.getPageOldVersion(refs...) + return p +} + +// Ref is either unix-style paths (i.e. callers responsible for +// calling filepath.ToSlash as necessary) or shorthand refs. +func (c *PageCollections) getPageNew(context *Page, ref string) (*Page, error) { + var anError error + + // Absolute (content root relative) reference. + if strings.HasPrefix(ref, "/") { + p, err := c.getFromCache(ref) + if err == nil && p != nil { + return p, nil + } + if err != nil { + anError = err + } + + } else if context != nil { + // Try the page-relative path. + ppath := path.Join("/", strings.Join(context.sections, "/"), ref) + p, err := c.getFromCache(ppath) + if err == nil && p != nil { + return p, nil + } + if err != nil { + anError = err + } + } + + if !strings.HasPrefix(ref, "/") { + // Many people will have "post/foo.md" in their content files. + p, err := c.getFromCache("/" + ref) + if err == nil && p != nil { + if context != nil { + // TODO(bep) remove this case and the message below when the storm has passed + helpers.DistinctFeedbackLog.Printf(`WARNING: make non-relative ref/relref page reference(s) in page %q absolute, e.g. {{< ref "/blog/my-post.md" >}}`, context.absoluteSourceRef()) + } + return p, nil + } + if err != nil { + anError = err + } + } + + // Last try. + ref = strings.TrimPrefix(ref, "/") + p, err := c.getFromCache(ref) + if err != nil { + anError = err + } + + if p == nil && anError != nil { + if context != nil { + return nil, fmt.Errorf("failed to resolve path from page %q: %s", context.absoluteSourceRef(), anError) + } + return nil, fmt.Errorf("failed to resolve page: %s", anError) + } + + return p, nil +} + +func (*PageCollections) findPagesByKindIn(kind string, inPages Pages) Pages { + var pages Pages + for _, p := range inPages { + if p.Kind == kind { + pages = append(pages, p) + } + } + return pages +} + +func (*PageCollections) findFirstPageByKindIn(kind string, inPages Pages) *Page { + for _, p := range inPages { + if p.Kind == kind { + return p + } + } + return nil +} + +func (*PageCollections) findPagesByKindNotIn(kind string, inPages Pages) Pages { + var pages Pages + for _, p := range inPages { + if p.Kind != kind { + pages = append(pages, p) + } + } + return pages +} + +func (c *PageCollections) findPagesByKind(kind string) Pages { + return c.findPagesByKindIn(kind, c.Pages) +} + +func (c *PageCollections) addPage(page *Page) { + c.rawAllPages = append(c.rawAllPages, page) +} + +func (c *PageCollections) removePageFilename(filename string) { + if i := c.rawAllPages.findPagePosByFilename(filename); i >= 0 { + c.clearResourceCacheForPage(c.rawAllPages[i]) + c.rawAllPages = append(c.rawAllPages[:i], c.rawAllPages[i+1:]...) + } + +} + +func (c *PageCollections) removePage(page *Page) { + if i := c.rawAllPages.findPagePos(page); i >= 0 { + c.clearResourceCacheForPage(c.rawAllPages[i]) + c.rawAllPages = append(c.rawAllPages[:i], c.rawAllPages[i+1:]...) + } + +} + +func (c *PageCollections) findPagesByShortcode(shortcode string) Pages { + var pages Pages + + for _, p := range c.rawAllPages { + if p.shortcodeState != nil { + if _, ok := p.shortcodeState.nameSet[shortcode]; ok { + pages = append(pages, p) + } + } + } + return pages +} + +func (c *PageCollections) replacePage(page *Page) { + // will find existing page that matches filepath and remove it + c.removePage(page) + c.addPage(page) +} + +func (c *PageCollections) clearResourceCacheForPage(page *Page) { + if len(page.Resources) > 0 { + page.s.ResourceSpec.DeleteCacheByPrefix(page.relTargetPathBase) + } +} diff --git a/hugolib/pagecollections_test.go b/hugolib/pagecollections_test.go new file mode 100644 index 00000000..2f8b3149 --- /dev/null +++ b/hugolib/pagecollections_test.go @@ -0,0 +1,242 @@ +// Copyright 2017 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "math/rand" + "path" + "path/filepath" + "testing" + "time" + + "github.com/gohugoio/hugo/deps" + "github.com/stretchr/testify/require" +) + +const pageCollectionsPageTemplate = `--- +title: "%s" +categories: +- Hugo +--- +# Doc +` + +func BenchmarkGetPage(b *testing.B) { + var ( + cfg, fs = newTestCfg() + r = rand.New(rand.NewSource(time.Now().UnixNano())) + ) + + for i := 0; i < 10; i++ { + for j := 0; j < 100; j++ { + writeSource(b, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", j)), "CONTENT") + } + } + + s := buildSingleSite(b, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) + + pagePaths := make([]string, b.N) + + for i := 0; i < b.N; i++ { + pagePaths[i] = fmt.Sprintf("sect%d", r.Intn(10)) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + home, _ := s.getPageNew(nil, "/") + if home == nil { + b.Fatal("Home is nil") + } + + p, _ := s.getPageNew(nil, pagePaths[i]) + if p == nil { + b.Fatal("Section is nil") + } + + } +} + +func BenchmarkGetPageRegular(b *testing.B) { + var ( + cfg, fs = newTestCfg() + r = rand.New(rand.NewSource(time.Now().UnixNano())) + ) + + for i := 0; i < 10; i++ { + for j := 0; j < 100; j++ { + content := fmt.Sprintf(pageCollectionsPageTemplate, fmt.Sprintf("Title%d_%d", i, j)) + writeSource(b, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", j)), content) + } + } + + s := buildSingleSite(b, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) + + pagePaths := make([]string, b.N) + + for i := 0; i < b.N; i++ { + pagePaths[i] = path.Join(fmt.Sprintf("sect%d", r.Intn(10)), fmt.Sprintf("page%d.md", r.Intn(100))) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + page, _ := s.getPageNew(nil, pagePaths[i]) + require.NotNil(b, page) + } +} + +type testCase struct { + kind string + context *Page + path []string + expectedTitle string +} + +func (t *testCase) check(p *Page, err error, errorMsg string, assert *require.Assertions) { + switch t.kind { + case "Ambiguous": + assert.Error(err) + assert.Nil(p, errorMsg) + case "NoPage": + assert.NoError(err) + assert.Nil(p, errorMsg) + default: + assert.NoError(err, errorMsg) + assert.NotNil(p, errorMsg) + assert.Equal(t.kind, p.Kind, errorMsg) + assert.Equal(t.expectedTitle, p.title, errorMsg) + } +} + +func TestGetPage(t *testing.T) { + + var ( + assert = require.New(t) + cfg, fs = newTestCfg() + ) + + for i := 0; i < 10; i++ { + for j := 0; j < 10; j++ { + content := fmt.Sprintf(pageCollectionsPageTemplate, fmt.Sprintf("Title%d_%d", i, j)) + writeSource(t, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", j)), content) + } + } + + content := fmt.Sprintf(pageCollectionsPageTemplate, "home page") + writeSource(t, fs, filepath.Join("content", "_index.md"), content) + + content = fmt.Sprintf(pageCollectionsPageTemplate, "about page") + writeSource(t, fs, filepath.Join("content", "about.md"), content) + + content = fmt.Sprintf(pageCollectionsPageTemplate, "section 3") + writeSource(t, fs, filepath.Join("content", "sect3", "_index.md"), content) + + content = fmt.Sprintf(pageCollectionsPageTemplate, "UniqueBase") + writeSource(t, fs, filepath.Join("content", "sect3", "unique.md"), content) + + content = fmt.Sprintf(pageCollectionsPageTemplate, "another sect7") + writeSource(t, fs, filepath.Join("content", "sect3", "sect7", "_index.md"), content) + + content = fmt.Sprintf(pageCollectionsPageTemplate, "deep page") + writeSource(t, fs, filepath.Join("content", "sect3", "subsect", "deep.md"), content) + + s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) + + sec3, err := s.getPageNew(nil, "/sect3") + assert.NoError(err, "error getting Page for /sec3") + assert.NotNil(sec3, "failed to get Page for /sec3") + + tests := []testCase{ + // legacy content root relative paths + {KindHome, nil, []string{}, "home page"}, + {KindPage, nil, []string{"about.md"}, "about page"}, + {KindSection, nil, []string{"sect3"}, "section 3"}, + {KindPage, nil, []string{"sect3/page1.md"}, "Title3_1"}, + {KindPage, nil, []string{"sect4/page2.md"}, "Title4_2"}, + {KindSection, nil, []string{"sect3/sect7"}, "another sect7"}, + {KindPage, nil, []string{"sect3/subsect/deep.md"}, "deep page"}, + {KindPage, nil, []string{filepath.FromSlash("sect5/page3.md")}, "Title5_3"}, //test OS-specific path + + // shorthand refs (potentially ambiguous) + {KindPage, nil, []string{"unique.md"}, "UniqueBase"}, + {"Ambiguous", nil, []string{"page1.md"}, ""}, + + // ISSUE: This is an ambiguous ref, but because we have to support the legacy + // content root relative paths without a leading slash, the lookup + // returns /sect7. This undermines ambiguity detection, but we have no choice. + //{"Ambiguous", nil, []string{"sect7"}, ""}, + {KindSection, nil, []string{"sect7"}, "Sect7s"}, + + // absolute paths + {KindHome, nil, []string{"/"}, "home page"}, + {KindPage, nil, []string{"/about.md"}, "about page"}, + {KindSection, nil, []string{"/sect3"}, "section 3"}, + {KindPage, nil, []string{"/sect3/page1.md"}, "Title3_1"}, + {KindPage, nil, []string{"/sect4/page2.md"}, "Title4_2"}, + {KindSection, nil, []string{"/sect3/sect7"}, "another sect7"}, + {KindPage, nil, []string{"/sect3/subsect/deep.md"}, "deep page"}, + {KindPage, nil, []string{filepath.FromSlash("/sect5/page3.md")}, "Title5_3"}, //test OS-specific path + {KindPage, nil, []string{"/sect3/unique.md"}, "UniqueBase"}, //next test depends on this page existing + // {"NoPage", nil, []string{"/unique.md"}, ""}, // ISSUE #4969: this is resolving to /sect3/unique.md + {"NoPage", nil, []string{"/missing-page.md"}, ""}, + {"NoPage", nil, []string{"/missing-section"}, ""}, + + // relative paths + {KindHome, sec3, []string{".."}, "home page"}, + {KindHome, sec3, []string{"../"}, "home page"}, + {KindPage, sec3, []string{"../about.md"}, "about page"}, + {KindSection, sec3, []string{"."}, "section 3"}, + {KindSection, sec3, []string{"./"}, "section 3"}, + {KindPage, sec3, []string{"page1.md"}, "Title3_1"}, + {KindPage, sec3, []string{"./page1.md"}, "Title3_1"}, + {KindPage, sec3, []string{"../sect4/page2.md"}, "Title4_2"}, + {KindSection, sec3, []string{"sect7"}, "another sect7"}, + {KindSection, sec3, []string{"./sect7"}, "another sect7"}, + {KindPage, sec3, []string{"./subsect/deep.md"}, "deep page"}, + {KindPage, sec3, []string{"./subsect/../../sect7/page9.md"}, "Title7_9"}, + {KindPage, sec3, []string{filepath.FromSlash("../sect5/page3.md")}, "Title5_3"}, //test OS-specific path + {KindPage, sec3, []string{"./unique.md"}, "UniqueBase"}, + {"NoPage", sec3, []string{"./sect2"}, ""}, + //{"NoPage", sec3, []string{"sect2"}, ""}, // ISSUE: /sect3 page relative query is resolving to /sect2 + + // absolute paths ignore context + {KindHome, sec3, []string{"/"}, "home page"}, + {KindPage, sec3, []string{"/about.md"}, "about page"}, + {KindPage, sec3, []string{"/sect4/page2.md"}, "Title4_2"}, + {KindPage, sec3, []string{"/sect3/subsect/deep.md"}, "deep page"}, //next test depends on this page existing + {"NoPage", sec3, []string{"/subsect/deep.md"}, ""}, + } + + for _, test := range tests { + errorMsg := fmt.Sprintf("Test case %s %v -> %s", test.context, test.path, test.expectedTitle) + + // test legacy public Site.GetPage (which does not support page context relative queries) + if test.context == nil { + args := append([]string{test.kind}, test.path...) + page, err := s.Info.GetPage(args...) + test.check(page, err, errorMsg, assert) + } + + // test new internal Site.getPageNew + var ref string + if len(test.path) == 1 { + ref = filepath.ToSlash(test.path[0]) + } else { + ref = path.Join(test.path...) + } + page2, err := s.getPageNew(test.context, ref) + test.check(page2, err, errorMsg, assert) + } + +}