Add embeded template for robots.txt

diff --git a/commands/hugo.go b/commands/hugo.go
index 4d3f9bab6a59bc220333260317f0dcda008c6fae..3b2e172c6fe510eacb26cef273469a42ca0d76a6 100644 (file)
--- a/commands/hugo.go
+++ b/commands/hugo.go
@@ -112,7 +112,7 @@ Complete documentation is available at http://gohugo.io/.`,
 var hugoCmdV *cobra.Command
 
 // Flags that are to be added to commands.
-var BuildWatch, IgnoreCache, Draft, Future, UglyURLs, CanonifyURLs, Verbose, Logging, VerboseLog, DisableRSS, DisableSitemap, PluralizeListTitles, PreserveTaxonomyNames, NoTimes, ForceSync bool
+var BuildWatch, IgnoreCache, Draft, Future, UglyURLs, CanonifyURLs, Verbose, Logging, VerboseLog, DisableRSS, DisableSitemap, DisableRobotsTXT, PluralizeListTitles, PreserveTaxonomyNames, NoTimes, ForceSync bool
 var Source, CacheDir, Destination, Theme, BaseURL, CfgFile, LogFile, Editor string
 
 // Execute adds all child commands to the root command HugoCmd and sets flags appropriately.
@@ -159,6 +159,7 @@ func initCoreCommonFlags(cmd *cobra.Command) {
        cmd.Flags().BoolVarP(&Future, "buildFuture", "F", false, "include content with publishdate in the future")
        cmd.Flags().BoolVar(&DisableRSS, "disableRSS", false, "Do not build RSS files")
        cmd.Flags().BoolVar(&DisableSitemap, "disableSitemap", false, "Do not build Sitemap file")
+       cmd.Flags().BoolVar(&DisableRobotsTXT, "disableRobotsTXT", false, "Do not build Robots TXT file")
        cmd.Flags().StringVarP(&Source, "source", "s", "", "filesystem path to read files relative from")
        cmd.Flags().StringVarP(&CacheDir, "cacheDir", "", "", "filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/")
        cmd.Flags().BoolVarP(&IgnoreCache, "ignoreCache", "", false, "Ignores the cache directory for reading but still writes to it")
@@ -204,6 +205,7 @@ func LoadDefaultSettings() {
        viper.SetDefault("MetaDataFormat", "toml")
        viper.SetDefault("DisableRSS", false)
        viper.SetDefault("DisableSitemap", false)
+       viper.SetDefault("DisableRobotsTXT", false)
        viper.SetDefault("ContentDir", "content")
        viper.SetDefault("LayoutDir", "layouts")
        viper.SetDefault("StaticDir", "static")
@@ -294,6 +296,9 @@ func InitializeConfig(subCmdVs ...*cobra.Command) error {
                if cmdV.Flags().Lookup("disableSitemap").Changed {
                        viper.Set("DisableSitemap", DisableSitemap)
                }
+               if cmdV.Flags().Lookup("disableRobotsTXT").Changed {
+                       viper.Set("DisableRobotsTXT", DisableRobotsTXT)
+               }
                if cmdV.Flags().Lookup("pluralizeListTitles").Changed {
                        viper.Set("PluralizeListTitles", PluralizeListTitles)
                }

diff --git a/docs/content/commands/hugo.md b/docs/content/commands/hugo.md
index d36cb0bab1ba6842eb9e14d0eb7b147d196a218d..190f67a14a9ddb24e83d719b467edbea645b339a 100644 (file)
--- a/docs/content/commands/hugo.md
+++ b/docs/content/commands/hugo.md
@@ -34,6 +34,7 @@ hugo
   -d, --destination="": filesystem path to write files to
       --disableRSS[=false]: Do not build RSS files
       --disableSitemap[=false]: Do not build Sitemap file
+      --disableRobotsTXT[=false]: Do not build robots TXT file
       --editor="": edit new content with this editor, if provided
       --ignoreCache[=false]: Ignores the cache directory for reading but still writes to it
       --log[=false]: Enable Logging

diff --git a/docs/content/extras/robots-txt.md b/docs/content/extras/robots-txt.md
new file mode 100644 (file)
index 0000000..336999d
--- /dev/null
+++ b/docs/content/extras/robots-txt.md
@@ -0,0 +1,34 @@
+---
+date: 2013-07-09
+menu:
+  main:
+    parent: extras
+next: /community/mailing-list
+prev: /extras/urls
+title: Table of Contents
+weight: 120
+---
+
+Hugo can generated customized [robots.txt](http://www.robotstxt.org/) in the
+[same way than any other template]({{< ref "templates/go-templates.md" >}}).
+
+By default it generates a robots.txt which allows everything, it looks exactly
+
+  User-agent: *
+
+To disable it just set `disableRobotsTXT` option to false in the [command line]({{< ref "commands/hugo.md" >}}) or [configuration file]({{< ref "overview/configuration.md" >}}).
+
+Hugo will use the template `robots.txt` following the list starting with the one with more priority
+
+* /layouts/robots.txt
+* /themes/`THEME`/layout/robots.txt
+
+An example of a robots.txt layout is:
+
+    User-agent: *
+
+    {{range .Data.Pages}}
+    Disallow: {{.RelPermalink}}{{end}}
+
+
+This template disallows and all the pages of the site creating one `Disallow` entry for each one.

diff --git a/docs/content/extras/urls.md b/docs/content/extras/urls.md
index 497caa0fe9d6c59bcf8fac375faee96e8bb22612..fb53b8f13817493a85f809707dc96ef8f6f3b555 100644 (file)
--- a/docs/content/extras/urls.md
+++ b/docs/content/extras/urls.md
@@ -5,7 +5,7 @@ date: 2014-01-03
 menu:
   main:
     parent: extras
-next: /community/mailing-list
+next: /extras/robots-txt
 notoc: true
 prev: /extras/toc
 title: URLs

diff --git a/docs/content/overview/configuration.md b/docs/content/overview/configuration.md
index 89a2642c5d15db15b115b34417ac4fae9a52200e..d5db7520ec5da1068d778d816586611a74b0a435 100644 (file)
--- a/docs/content/overview/configuration.md
+++ b/docs/content/overview/configuration.md
@@ -96,6 +96,8 @@ Following is a list of Hugo-defined variables that you can configure and their c
     disableRSS:                 false
     # Do not build Sitemap file
     disableSitemap:             false
+    # Do not build robots.txt file
+    disableRobotsTXT:           false
     # edit new content with this editor, if provided
     editor:                     ""
     footnoteAnchorPrefix:       ""

diff --git a/hugolib/robotstxt_test.go b/hugolib/robotstxt_test.go
new file mode 100644 (file)
index 0000000..c964c02
--- /dev/null
+++ b/hugolib/robotstxt_test.go
@@ -0,0 +1,67 @@
+package hugolib
+
+import (
+       "bytes"
+       "testing"
+
+       "github.com/spf13/afero"
+       "github.com/spf13/hugo/helpers"
+       "github.com/spf13/hugo/hugofs"
+       "github.com/spf13/hugo/source"
+       "github.com/spf13/viper"
+)
+
+const ROBOTSTXT_TEMPLATE = `User-agent: Googlebot
+  {{ range .Data.Pages }}
+       Disallow: {{.RelPermalink}}
+       {{ end }}
+`
+
+func TestRobotsTXTOutput(t *testing.T) {
+       viper.Reset()
+       defer viper.Reset()
+
+       hugofs.DestinationFS = new(afero.MemMapFs)
+
+       viper.Set("baseurl", "http://auth/bub/")
+
+       s := &Site{
+               Source: &source.InMemorySource{ByteSource: WEIGHTED_SOURCES},
+       }
+
+       s.initializeSiteInfo()
+
+       s.prepTemplates()
+       s.addTemplate("robots.txt", ROBOTSTXT_TEMPLATE)
+
+       if err := s.CreatePages(); err != nil {
+               t.Fatalf("Unable to create pages: %s", err)
+       }
+
+       if err := s.BuildSiteMeta(); err != nil {
+               t.Fatalf("Unable to build site metadata: %s", err)
+       }
+
+       if err := s.RenderHomePage(); err != nil {
+               t.Fatalf("Unable to RenderHomePage: %s", err)
+       }
+
+       if err := s.RenderSitemap(); err != nil {
+               t.Fatalf("Unable to RenderSitemap: %s", err)
+       }
+
+       if err := s.RenderRobotsTXT(); err != nil {
+               t.Fatalf("Unable to RenderRobotsTXT :%s", err)
+       }
+
+       robotsFile, err := hugofs.DestinationFS.Open("robots.txt")
+
+       if err != nil {
+               t.Fatalf("Unable to locate: robots.txt")
+       }
+
+       robots := helpers.ReaderToBytes(robotsFile)
+       if !bytes.HasPrefix(robots, []byte("User-agent: Googlebot")) {
+               t.Errorf("Robots file should start with 'User-agentL Googlebot'. %s", robots)
+       }
+}

diff --git a/hugolib/site.go b/hugolib/site.go
index 648ba6454b178d44a4fe6cda0fb128937dd4fd28..957e8ed4c65ec74dcf64905fdac5059a0717f15b 100644 (file)
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -583,6 +583,12 @@ func (s *Site) Render() (err error) {
                return
        }
        s.timerStep("render and write Sitemap")
+
+       if err = s.RenderRobotsTXT(); err != nil {
+               return
+       }
+       s.timerStep("render and write robots.txt")
+
        return
 }
 
@@ -1561,6 +1567,26 @@ func (s *Site) RenderSitemap() error {
        return nil
 }
 
+func (s *Site) RenderRobotsTXT() error {
+       if viper.GetBool("DisableRobotsTXT") {
+               return nil
+       }
+
+       n := s.NewNode()
+       n.Data["Pages"] = s.Pages
+
+       rLayouts := []string{"robots.txt", "_default/robots.txt", "_internal/_default/robots.txt"}
+       outBuffer := bp.GetBuffer()
+       defer bp.PutBuffer(outBuffer)
+       err := s.render("robots", n, outBuffer, s.appendThemeTemplates(rLayouts)...)
+
+       if err == nil {
+               err = s.WriteDestFile("robots.txt", outBuffer)
+       }
+
+       return err
+}
+
 func (s *Site) Stats() {
        jww.FEEDBACK.Println(s.draftStats())
        jww.FEEDBACK.Println(s.futureStats())

diff --git a/hugolib/sitemap_test.go b/hugolib/sitemap_test.go
index 7bc16b482319418ea6410459cc8759ccd58de7fd..13bc92a188c671b5c2b743e650838a7d666cf2a1 100644 (file)
--- a/hugolib/sitemap_test.go
+++ b/hugolib/sitemap_test.go
@@ -68,6 +68,10 @@ func TestSitemapOutput(t *testing.T) {
                t.Fatalf("Unable to RenderSitemap: %s", err)
        }
 
+       if err := s.RenderRobotsTXT(); err != nil {
+               t.Fatalf("Unable to RenderRobotsTXT :%s", err)
+       }
+
        sitemapFile, err := hugofs.DestinationFS.Open("sitemap.xml")
 
        if err != nil {

diff --git a/tpl/template_embedded.go b/tpl/template_embedded.go
index a2fde4b10705eefc5498027b0daaf9651e83a7f8..14364f6db04b7523c9fd42544c8f6701f1aa327b 100644 (file)
--- a/tpl/template_embedded.go
+++ b/tpl/template_embedded.go
@@ -242,4 +242,6 @@ ga('send', 'pageview');
 </script>
 <script async src='//www.google-analytics.com/analytics.js'></script>
 {{ end }}`)
+
+       t.AddInternalTemplate("_default", "robots.txt", "User-agent: *")
 }