Do not create robots.txt by default

Meny people, including me, have a custom robots.txt in static.

Also remove that option from the command line; it doesn't feel
important enough.

Fixes ##2049

diff --git a/commands/hugo.go b/commands/hugo.go
index f7eb4cbc04d66f1dc5b16cc9b177531386369014..7961bef972d996393cb059c6187a8f7d41ee828e 100644 (file)
--- a/commands/hugo.go
+++ b/commands/hugo.go
@@ -126,7 +126,7 @@ var (
        buildWatch            bool
        canonifyURLs          bool
        cleanDestination      bool
-       disableRobotsTXT      bool
+       enableRobotsTXT       bool
        disableRSS            bool
        disableSitemap        bool
        draft                 bool
@@ -216,7 +216,6 @@ func initHugoBuildCommonFlags(cmd *cobra.Command) {
        cmd.Flags().BoolVarP(&future, "buildFuture", "F", false, "include content with publishdate in the future")
        cmd.Flags().BoolVar(&disableRSS, "disableRSS", false, "Do not build RSS files")
        cmd.Flags().BoolVar(&disableSitemap, "disableSitemap", false, "Do not build Sitemap file")
-       cmd.Flags().BoolVar(&disableRobotsTXT, "disableRobotsTXT", false, "Do not build Robots TXT file")
        cmd.Flags().StringVarP(&source, "source", "s", "", "filesystem path to read files relative from")
        cmd.Flags().StringVarP(&contentDir, "contentDir", "c", "", "filesystem path to content directory")
        cmd.Flags().StringVarP(&layoutDir, "layoutDir", "l", "", "filesystem path to layout directory")
@@ -364,8 +363,8 @@ func InitializeConfig(subCmdVs ...*cobra.Command) error {
                if flagChanged(cmdV.Flags(), "disableSitemap") {
                        viper.Set("DisableSitemap", disableSitemap)
                }
-               if flagChanged(cmdV.Flags(), "disableRobotsTXT") {
-                       viper.Set("DisableRobotsTXT", disableRobotsTXT)
+               if flagChanged(cmdV.Flags(), "enableRobotsTXT") {
+                       viper.Set("EnableRobotsTXT", enableRobotsTXT)
                }
                if flagChanged(cmdV.Flags(), "pluralizeListTitles") {
                        viper.Set("PluralizeListTitles", pluralizeListTitles)

diff --git a/docs/content/extras/robots-txt.md b/docs/content/extras/robots-txt.md
index 31a902be92f927994c3c7b090caac2c097ab5184..9122ae9d832bad2a196506c18c4235c2ba59022f 100644 (file)
--- a/docs/content/extras/robots-txt.md
+++ b/docs/content/extras/robots-txt.md
@@ -13,13 +13,12 @@ weight: 120
 Hugo can generated a customized [robots.txt](http://www.robotstxt.org/) in the
 [same way as any other templates]({{< ref "templates/go-templates.md" >}}).
 
-By default, it generates a robots.txt, which allows everything, with the following content:
+To enable it, just set `enableRobotsTXT` option to `true` in the [configuration file]({{< ref "overview/configuration.md" >}}). By default, it generates a robots.txt, which allows everything, with the following content:
 
 ```http
 User-agent: *
 ```
 
-To disable it, just set `disableRobotsTXT` option to `false` in the [command line]({{< ref "commands/hugo.md" >}}) or [configuration file]({{< ref "overview/configuration.md" >}}).
 
 Hugo will use the template `robots.txt` according to the following list in descending precedence:
 

diff --git a/docs/content/overview/configuration.md b/docs/content/overview/configuration.md
index 33813beb44cd10ac4f7a399060f36e6820ffaaac..e351c651a736277e9573b973bb35c00e87bde058 100644 (file)
--- a/docs/content/overview/configuration.md
+++ b/docs/content/overview/configuration.md
@@ -95,8 +95,8 @@ Following is a list of Hugo-defined variables that you can configure and their c
     disableRSS:                 false
     # Do not build Sitemap file
     disableSitemap:             false
-    # Do not build robots.txt file
-    disableRobotsTXT:           false
+    # Build robots.txt file
+    enableRobotsTXT:           false
     # edit new content with this editor, if provided
     editor:                     ""
     # Enable Emoji emoticons support for page content.

diff --git a/hugolib/robotstxt_test.go b/hugolib/robotstxt_test.go
index b0a843a7ec156ccdba33f37385ecbf6cfd2ff118..b88d2243b0ed8b0cc87160dd366220e11ce7eca3 100644 (file)
--- a/hugolib/robotstxt_test.go
+++ b/hugolib/robotstxt_test.go
@@ -36,6 +36,7 @@ func TestRobotsTXTOutput(t *testing.T) {
        hugofs.InitMemFs()
 
        viper.Set("baseurl", "http://auth/bub/")
+       viper.Set("enableRobotsTXT", true)
 
        s := &Site{
                Source: &source.InMemorySource{ByteSource: WEIGHTED_SOURCES},

diff --git a/hugolib/site.go b/hugolib/site.go
index 53e37bfbf212b170480d4c0b77760d1fbd1f2cc8..ea9297b3dad02914c1347559f5780cf1296be40c 100644 (file)
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -1864,7 +1864,7 @@ func (s *Site) RenderSitemap() error {
 }
 
 func (s *Site) RenderRobotsTXT() error {
-       if viper.GetBool("DisableRobotsTXT") {
+       if !viper.GetBool("EnableRobotsTXT") {
                return nil
        }