Create robots.txt in the domain root directory

Before a robots.txt is created in every Site. So in public/robots.txt if there are no languages (was correct). But if there are multiple languages in every language directory, too (was wrong). If defaultContentLanguageInSubdir is true, no language is created into the root directory, so no robots.txt is in the root directory (was wrong). If multihosts are configured for each language, that is the only case where one robots.txt must be created in each language directory (was correct).

I've changed the behaviour, that only in the multihost case the robots.txt is created in the language directories. In any other case it is created in public/robots.txt. I've also added tests that files are not created in the wrong directories.

Fixes #5160
See also #4193

diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go
index e71e48d4124ff8a4ab87d83be8fdd0df504da3cb..ee0d5c563688e0fdee2b365317bfa30fe28a9409 100644 (file)
--- a/hugolib/hugo_sites.go
+++ b/hugolib/hugo_sites.go
@@ -49,6 +49,7 @@ import (
 
        "github.com/gohugoio/hugo/langs/i18n"
        "github.com/gohugoio/hugo/resources/page"
+       "github.com/gohugoio/hugo/resources/page/pagemeta"
        "github.com/gohugoio/hugo/tpl"
        "github.com/gohugoio/hugo/tpl/tplimpl"
 )
@@ -690,7 +691,7 @@ func (cfg *BuildCfg) shouldRender(p *pageState) bool {
        return false
 }
 
-func (h *HugoSites) renderCrossSitesArtifacts() error {
+func (h *HugoSites) renderCrossSitesSitemap() error {
 
        if !h.multilingual.enabled() || h.IsMultihost() {
                return nil
@@ -716,6 +717,38 @@ func (h *HugoSites) renderCrossSitesArtifacts() error {
                s.siteCfg.sitemap.Filename, h.toSiteInfos(), templ)
 }
 
+func (h *HugoSites) renderCrossSitesRobotsTXT() error {
+       if h.multihost {
+               return nil
+       }
+       if !h.Cfg.GetBool("enableRobotsTXT") {
+               return nil
+       }
+
+       s := h.Sites[0]
+
+       p, err := newPageStandalone(&pageMeta{
+               s:    s,
+               kind: kindRobotsTXT,
+               urlPaths: pagemeta.URLPath{
+                       URL: "robots.txt",
+               },
+       },
+               output.RobotsTxtFormat)
+
+       if err != nil {
+               return err
+       }
+
+       if !p.render {
+               return nil
+       }
+
+       templ := s.lookupLayouts("robots.txt", "_default/robots.txt", "_internal/_default/robots.txt")
+
+       return s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, "Robots Txt", "robots.txt", p, templ)
+}
+
 func (h *HugoSites) removePageByFilename(filename string) {
        h.getContentMaps().withMaps(func(m *pageMap) error {
                m.deleteBundleMatching(func(b *contentNode) bool {

diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go
index f39e7a7e53f540766ca37f4250c15cebec3c6bc0..67ee10e09787b0b5f5f49caeccbf19dd85ddaeab 100644 (file)
--- a/hugolib/hugo_sites_build.go
+++ b/hugolib/hugo_sites_build.go
@@ -334,7 +334,10 @@ func (h *HugoSites) render(config *BuildCfg) error {
        }
 
        if !config.SkipRender {
-               if err := h.renderCrossSitesArtifacts(); err != nil {
+               if err := h.renderCrossSitesSitemap(); err != nil {
+                       return err
+               }
+               if err := h.renderCrossSitesRobotsTXT(); err != nil {
                        return err
                }
        }

diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go
index 84655c1f20d455bb8ba17a3af8caf0beef50ffe9..8d0872bd5e04a27cf75d6f06aa6910be45351adf 100644 (file)
--- a/hugolib/hugo_sites_build_test.go
+++ b/hugolib/hugo_sites_build_test.go
@@ -255,8 +255,10 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) {
        b.AssertFileContent("public/fr/404.html", "404|fr|404 Page not found")
 
        // Check robots.txt
-       b.AssertFileContent("public/en/robots.txt", "robots|en|")
-       b.AssertFileContent("public/nn/robots.txt", "robots|nn|")
+       // the domain root is the public directory, so the robots.txt has to be created there and not in the language directories
+       b.AssertFileContent("public/robots.txt", "robots")
+       b.AssertFileDoesNotExist("public/en/robots.txt")
+       b.AssertFileDoesNotExist("public/nn/robots.txt")
 
        b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Permalink: http://example.com/blog/en/sect/doc1-slug/")
        b.AssertFileContent("public/en/sect/doc2/index.html", "Permalink: http://example.com/blog/en/sect/doc2/")

diff --git a/hugolib/hugo_sites_multihost_test.go b/hugolib/hugo_sites_multihost_test.go
index a15e8cd43cb3b54b2479aeb0a535d68f1baef506..4fe4960422caf44b4b324d42214618c67cb97ff7 100644 (file)
--- a/hugolib/hugo_sites_multihost_test.go
+++ b/hugolib/hugo_sites_multihost_test.go
@@ -19,6 +19,7 @@ disablePathToLower = true
 defaultContentLanguage = "fr"
 defaultContentLanguageInSubdir = false
 staticDir = ["s1", "s2"]
+enableRobotsTXT = true
 
 [permalinks]
 other = "/somewhere/else/:filename"
@@ -73,6 +74,12 @@ languageName = "Nynorsk"
        c.Assert(pageWithURLInFrontMatter.RelPermalink(), qt.Equals, "/docs/superbob/")
        b.AssertFileContent("public/en/superbob/index.html", "doc3|Hello|en")
 
+       // the domain root is the language directory for each language, so the robots.txt is created in the language directories
+       b.AssertFileContent("public/en/robots.txt", "robots|en")
+       b.AssertFileContent("public/fr/robots.txt", "robots|fr")
+       b.AssertFileContent("public/nn/robots.txt", "robots|nn")
+       b.AssertFileDoesNotExist("public/robots.txt")
+
        // check alias:
        b.AssertFileContent("public/en/al/alias1/index.html", `content="0; url=https://example.com/docs/superbob/"`)
        b.AssertFileContent("public/en/al/alias2/index.html", `content="0; url=https://example.com/docs/superbob/"`)

diff --git a/hugolib/site.go b/hugolib/site.go
index 5507d7a787eb9baf4c52435ffb1fbc9d19c956fd..ac65931d0c8dcf9e6d21e9621509142bd944009d 100644 (file)
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -1209,8 +1209,10 @@ func (s *Site) render(ctx *siteRenderContext) (err error) {
                        return
                }
 
-               if err = s.renderRobotsTXT(); err != nil {
-                       return
+               if ctx.multihost {
+                       if err = s.renderRobotsTXT(); err != nil {
+                               return
+                       }
                }
 
                if err = s.render404(); err != nil {

diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go
index c1ee27557eade35216685d6eb10bb94acc3ec59b..2af4691d173995fa5ab2bf66c8e2e04360508165 100644 (file)
--- a/hugolib/testhelpers_test.go
+++ b/hugolib/testhelpers_test.go
@@ -726,6 +726,12 @@ func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) {
        }
 }
 
+func (s *sitesBuilder) AssertFileDoesNotExist(filename string) {
+       if s.CheckExists(filename) {
+               s.Fatalf("File %q exists but must not exist.", filename)
+       }
+}
+
 func (s *sitesBuilder) AssertImage(width, height int, filename string) {
        filename = filepath.Join(s.workingDir, filename)
        f, err := s.Fs.Destination.Open(filename)