Implement configuration in a directory for modules
authorBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Wed, 16 Jun 2021 17:11:01 +0000 (19:11 +0200)
committerBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Fri, 18 Jun 2021 10:54:30 +0000 (12:54 +0200)
Fixes #8654

config/configLoader.go
hugolib/config.go
hugolib/config_test.go
modules/client.go
modules/collect.go
modules/module.go

index 0998b1befab133665994e2194c914b43faddc432..8dcfcbdcc0cccefeb09f2d8f7ff694e82b9e9190 100644 (file)
 package config
 
 import (
+       "os"
        "path/filepath"
        "strings"
 
+       "github.com/pkg/errors"
+
+       "github.com/gohugoio/hugo/common/paths"
+
        "github.com/gohugoio/hugo/common/maps"
        "github.com/gohugoio/hugo/parser/metadecoders"
        "github.com/spf13/afero"
@@ -84,6 +89,102 @@ func loadConfigFromFile(fs afero.Fs, filename string) (map[string]interface{}, e
        return m, nil
 }
 
+func LoadConfigFromDir(sourceFs afero.Fs, configDir, environment string) (Provider, []string, error) {
+       defaultConfigDir := filepath.Join(configDir, "_default")
+       environmentConfigDir := filepath.Join(configDir, environment)
+       cfg := New()
+
+       var configDirs []string
+       // Merge from least to most specific.
+       for _, dir := range []string{defaultConfigDir, environmentConfigDir} {
+               if _, err := sourceFs.Stat(dir); err == nil {
+                       configDirs = append(configDirs, dir)
+               }
+       }
+
+       if len(configDirs) == 0 {
+               return nil, nil, nil
+       }
+
+       // Keep track of these so we can watch them for changes.
+       var dirnames []string
+
+       for _, configDir := range configDirs {
+               err := afero.Walk(sourceFs, configDir, func(path string, fi os.FileInfo, err error) error {
+                       if fi == nil || err != nil {
+                               return nil
+                       }
+
+                       if fi.IsDir() {
+                               dirnames = append(dirnames, path)
+                               return nil
+                       }
+
+                       if !IsValidConfigFilename(path) {
+                               return nil
+                       }
+
+                       name := paths.Filename(filepath.Base(path))
+
+                       item, err := metadecoders.Default.UnmarshalFileToMap(sourceFs, path)
+                       if err != nil {
+                               // This will be used in error reporting, use the most specific value.
+                               dirnames = []string{path}
+                               return errors.Wrapf(err, "failed to unmarshl config for path %q", path)
+                       }
+
+                       var keyPath []string
+
+                       if name != "config" {
+                               // Can be params.jp, menus.en etc.
+                               name, lang := paths.FileAndExtNoDelimiter(name)
+
+                               keyPath = []string{name}
+
+                               if lang != "" {
+                                       keyPath = []string{"languages", lang}
+                                       switch name {
+                                       case "menu", "menus":
+                                               keyPath = append(keyPath, "menus")
+                                       case "params":
+                                               keyPath = append(keyPath, "params")
+                                       }
+                               }
+                       }
+
+                       root := item
+                       if len(keyPath) > 0 {
+                               root = make(map[string]interface{})
+                               m := root
+                               for i, key := range keyPath {
+                                       if i >= len(keyPath)-1 {
+                                               m[key] = item
+                                       } else {
+                                               nm := make(map[string]interface{})
+                                               m[key] = nm
+                                               m = nm
+                                       }
+                               }
+                       }
+
+                       // Migrate menu => menus etc.
+                       RenameKeys(root)
+
+                       // Set will overwrite keys with the same name, recursively.
+                       cfg.Set("", root)
+
+                       return nil
+               })
+               if err != nil {
+                       return nil, dirnames, err
+               }
+
+       }
+
+       return cfg, dirnames, nil
+
+}
+
 var keyAliases maps.KeyRenamer
 
 func init() {
index 091827660445072a32e2bf25aea9d8ad5d34244c..cad8451990db41690faa7935ca444e5faee34b45 100644 (file)
@@ -79,10 +79,16 @@ func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provid
        }
 
        if d.AbsConfigDir != "" {
-               dirnames, err := l.loadConfigFromConfigDir()
+               dcfg, dirnames, err := config.LoadConfigFromDir(l.Fs, d.AbsConfigDir, l.Environment)
                if err == nil {
-                       configFiles = append(configFiles, dirnames...)
+                       if len(dirnames) > 0 {
+                               l.cfg.Set("", dcfg.Get(""))
+                               configFiles = append(configFiles, dirnames...)
+                       }
                } else if err != ErrNoConfigFile {
+                       if len(dirnames) > 0 {
+                               return nil, nil, l.wrapFileError(err, dirnames[0])
+                       }
                        return nil, nil, err
                }
        }
@@ -381,9 +387,9 @@ func (l configLoader) collectModules(modConfig modules.Config, v1 config.Provide
 
        hook := func(m *modules.ModulesConfig) error {
                for _, tc := range m.ActiveModules {
-                       if tc.ConfigFilename() != "" {
+                       if len(tc.ConfigFilenames()) > 0 {
                                if tc.Watch() {
-                                       configFilenames = append(configFilenames, tc.ConfigFilename())
+                                       configFilenames = append(configFilenames, tc.ConfigFilenames()...)
                                }
 
                                // Merge from theme config into v1 based on configured
@@ -406,6 +412,7 @@ func (l configLoader) collectModules(modConfig modules.Config, v1 config.Provide
                HookBeforeFinalize: hook,
                WorkingDir:         workingDir,
                ThemesDir:          themesDir,
+               Environment:        l.Environment,
                CacheDir:           filecacheConfigs.CacheDirModules(),
                ModuleConfig:       modConfig,
                IgnoreVendor:       ignoreVendor,
@@ -468,106 +475,6 @@ func (l configLoader) loadConfig(configName string) (string, error) {
        return filename, nil
 }
 
-func (l configLoader) loadConfigFromConfigDir() ([]string, error) {
-       sourceFs := l.Fs
-       configDir := l.AbsConfigDir
-
-       if _, err := sourceFs.Stat(configDir); err != nil {
-               // Config dir does not exist.
-               return nil, nil
-       }
-
-       defaultConfigDir := filepath.Join(configDir, "_default")
-       environmentConfigDir := filepath.Join(configDir, l.Environment)
-
-       var configDirs []string
-       // Merge from least to most specific.
-       for _, dir := range []string{defaultConfigDir, environmentConfigDir} {
-               if _, err := sourceFs.Stat(dir); err == nil {
-                       configDirs = append(configDirs, dir)
-               }
-       }
-
-       if len(configDirs) == 0 {
-               return nil, nil
-       }
-
-       // Keep track of these so we can watch them for changes.
-       var dirnames []string
-
-       for _, configDir := range configDirs {
-               err := afero.Walk(sourceFs, configDir, func(path string, fi os.FileInfo, err error) error {
-                       if fi == nil || err != nil {
-                               return nil
-                       }
-
-                       if fi.IsDir() {
-                               dirnames = append(dirnames, path)
-                               return nil
-                       }
-
-                       if !config.IsValidConfigFilename(path) {
-                               return nil
-                       }
-
-                       name := cpaths.Filename(filepath.Base(path))
-
-                       item, err := metadecoders.Default.UnmarshalFileToMap(sourceFs, path)
-                       if err != nil {
-                               return l.wrapFileError(err, path)
-                       }
-
-                       var keyPath []string
-
-                       if name != "config" {
-                               // Can be params.jp, menus.en etc.
-                               name, lang := cpaths.FileAndExtNoDelimiter(name)
-
-                               keyPath = []string{name}
-
-                               if lang != "" {
-                                       keyPath = []string{"languages", lang}
-                                       switch name {
-                                       case "menu", "menus":
-                                               keyPath = append(keyPath, "menus")
-                                       case "params":
-                                               keyPath = append(keyPath, "params")
-                                       }
-                               }
-                       }
-
-                       root := item
-                       if len(keyPath) > 0 {
-                               root = make(map[string]interface{})
-                               m := root
-                               for i, key := range keyPath {
-                                       if i >= len(keyPath)-1 {
-                                               m[key] = item
-                                       } else {
-                                               nm := make(map[string]interface{})
-                                               m[key] = nm
-                                               m = nm
-                                       }
-                               }
-                       }
-
-                       // Migrate menu => menus etc.
-                       config.RenameKeys(root)
-
-                       // Set will overwrite keys with the same name, recursively.
-                       l.cfg.Set("", root)
-
-                       return nil
-               })
-               if err != nil {
-                       return nil, err
-               }
-
-       }
-
-       return dirnames, nil
-}
-
 func (l configLoader) loadLanguageSettings(oldLangs langs.Languages) error {
        _, err := langs.LoadLanguageSettings(l.cfg, oldLangs)
        return err
index 77ac9b92f162efeecb3a1e0a55b09d20f151340c..65cb246b9c934339bf4d49ea57d1f17bdf9f492b 100644 (file)
@@ -318,6 +318,59 @@ name = "menu-theme"
 
 }
 
+func TestLoadConfigFromThemeDir(t *testing.T) {
+       t.Parallel()
+
+       mainConfig := `
+theme = "test-theme"
+
+[params]
+m1 = "mv1"     
+`
+
+       themeConfig := `
+[params]
+t1 = "tv1"     
+t2 = "tv2"
+`
+
+       themeConfigDir := filepath.Join("themes", "test-theme", "config")
+       themeConfigDirDefault := filepath.Join(themeConfigDir, "_default")
+       themeConfigDirProduction := filepath.Join(themeConfigDir, "production")
+
+       projectConfigDir := "config"
+
+       b := newTestSitesBuilder(t)
+       b.WithConfigFile("toml", mainConfig).WithThemeConfigFile("toml", themeConfig)
+       b.Assert(b.Fs.Source.MkdirAll(themeConfigDirDefault, 0777), qt.IsNil)
+       b.Assert(b.Fs.Source.MkdirAll(themeConfigDirProduction, 0777), qt.IsNil)
+       b.Assert(b.Fs.Source.MkdirAll(projectConfigDir, 0777), qt.IsNil)
+
+       b.WithSourceFile(filepath.Join(projectConfigDir, "config.toml"), `[params]
+m2 = "mv2"
+`)
+       b.WithSourceFile(filepath.Join(themeConfigDirDefault, "config.toml"), `[params]
+t2 = "tv2d"
+t3 = "tv3d"
+`)
+
+       b.WithSourceFile(filepath.Join(themeConfigDirProduction, "config.toml"), `[params]
+t3 = "tv3p"
+`)
+
+       b.Build(BuildCfg{})
+
+       got := b.Cfg.Get("params").(maps.Params)
+
+       b.Assert(got, qt.DeepEquals, maps.Params{
+               "t3": "tv3p",
+               "m1": "mv1",
+               "t1": "tv1",
+               "t2": "tv2d",
+       })
+
+}
+
 func TestPrivacyConfig(t *testing.T) {
        t.Parallel()
 
index 571ece15ebd2cf08d1f04cded8cfe2198716fa1d..73c3242a8058a67371ca9cd9cc132a712766820d 100644 (file)
@@ -653,6 +653,9 @@ type ClientConfig struct {
        // Absolute path to the project's themes dir.
        ThemesDir string
 
+       // Eg. "production"
+       Environment string
+
        CacheDir     string // Module cache
        ModuleConfig Config
 }
index 163eda74a57ef139bd1690d779b00ff7d9ccd34c..52d75af59a55ff3de1f1be8b422cc67c55e55c27 100644 (file)
@@ -396,17 +396,16 @@ func (c *collector) applyMounts(moduleImport Import, mod *moduleAdapter) error {
 func (c *collector) applyThemeConfig(tc *moduleAdapter) error {
        var (
                configFilename string
-               cfg            config.Provider
                themeCfg       map[string]interface{}
-               hasConfig      bool
+               hasConfigFile  bool
                err            error
        )
 
        // Viper supports more, but this is the sub-set supported by Hugo.
        for _, configFormats := range config.ValidConfigFileExtensions {
                configFilename = filepath.Join(tc.Dir(), "config."+configFormats)
-               hasConfig, _ = afero.Exists(c.fs, configFilename)
-               if hasConfig {
+               hasConfigFile, _ = afero.Exists(c.fs, configFilename)
+               if hasConfigFile {
                        break
                }
        }
@@ -428,20 +427,38 @@ func (c *collector) applyThemeConfig(tc *moduleAdapter) error {
                }
        }
 
-       if hasConfig {
+       if hasConfigFile {
                if configFilename != "" {
                        var err error
-                       cfg, err = config.FromFile(c.fs, configFilename)
+                       tc.cfg, err = config.FromFile(c.fs, configFilename)
                        if err != nil {
                                return errors.Wrapf(err, "failed to read module config for %q in %q", tc.Path(), configFilename)
                        }
                }
 
-               tc.configFilename = configFilename
-               tc.cfg = cfg
+               tc.configFilenames = append(tc.configFilenames, configFilename)
+
+       }
+
+       // Also check for a config dir, which we overlay on top of the file configuration.
+       configDir := filepath.Join(tc.Dir(), "config")
+       dcfg, dirnames, err := config.LoadConfigFromDir(c.fs, configDir, c.ccfg.Environment)
+       if err != nil {
+               return err
+       }
+
+       if len(dirnames) > 0 {
+               tc.configFilenames = append(tc.configFilenames, dirnames...)
+
+               if hasConfigFile {
+                       // Set will overwrite existing keys.
+                       tc.cfg.Set("", dcfg.Get(""))
+               } else {
+                       tc.cfg = dcfg
+               }
        }
 
-       config, err := decodeConfig(cfg, c.moduleConfig.replacementsMap)
+       config, err := decodeConfig(tc.cfg, c.moduleConfig.replacementsMap)
        if err != nil {
                return err
        }
index a5f70763587872705b5c3600136e41900d974641..c3343c820d83408d690b251d91e91ee4532d7049 100644 (file)
@@ -30,10 +30,10 @@ type Module interface {
        // The decoded module config and mounts.
        Config() Config
 
-       // Optional configuration filename (e.g. "/themes/mytheme/config.json").
+       // Optional configuration filenames (e.g. "/themes/mytheme/config.json").
        // This will be added to the special configuration watch list when in
        // server mode.
-       ConfigFilename() string
+       ConfigFilenames() []string
 
        // Directory holding files for this module.
        Dir() string
@@ -82,9 +82,9 @@ type moduleAdapter struct {
 
        mounts []Mount
 
-       configFilename string
-       cfg            config.Provider
-       config         Config
+       configFilenames []string
+       cfg             config.Provider
+       config          Config
 
        // Set if a Go module.
        gomod *goModule
@@ -98,8 +98,8 @@ func (m *moduleAdapter) Config() Config {
        return m.config
 }
 
-func (m *moduleAdapter) ConfigFilename() string {
-       return m.configFilename
+func (m *moduleAdapter) ConfigFilenames() []string {
+       return m.configFilenames
 }
 
 func (m *moduleAdapter) Dir() string {