name: Test
run: |
mage -v test
- go clean -i -testcache
env:
HUGO_BUILD_TAGS: extended,withdeploy
- if: matrix.os == 'ubuntu-latest'
imports.*
dist/
public/
-.DS_Store
\ No newline at end of file
+.DS_Store
+cache/filecache/_gen/
\ No newline at end of file
import (
"bytes"
"errors"
+ "fmt"
"io"
"os"
"path/filepath"
nlocker *lockTracker
+ isResourceDir bool
+ baseDir string
+
initOnce sync.Once
+ isInited bool
initErr error
}
}
func (c *Cache) init() error {
+ if c == nil {
+ panic("cache is nil")
+ }
c.initOnce.Do(func() {
+ c.isInited = true
// Create the base dir if it does not exist.
if err := c.Fs.MkdirAll("", 0o777); err != nil && !os.IsExist(err) {
+ err = fmt.Errorf("failled to create base cache directory: %s", err)
c.initErr = err
}
})
return info, b, nil
}
+// SetBytes sets the file content with the given id in the cache.
+func (c *Cache) SetBytes(id string, data []byte) error {
+ if err := c.init(); err != nil {
+ return err
+ }
+ id = cleanID(id)
+
+ c.nlocker.Lock(id)
+ defer c.nlocker.Unlock(id)
+
+ if c.maxAge == 0 {
+ // No caching.
+ return nil
+ }
+
+ return c.writeReader(id, bytes.NewReader(data))
+}
+
// GetBytes gets the file content with the given id from the cache, nil if none found.
-func (c *Cache) GetBytes(id string) (ItemInfo, []byte, error) {
+func (c *Cache) GetBytes(id string) ([]byte, error) {
+ _, b, err := c.GetItemBytes(id)
+ return b, err
+}
+
+// GetItemBytes gets the ItemInfo and file content with the given id from the cache, nil if none found.
+func (c *Cache) GetItemBytes(id string) (ItemInfo, []byte, error) {
if err := c.init(); err != nil {
return ItemInfo{}, nil, err
}
// Caches is a named set of caches.
type Caches map[string]*Cache
+func (f Caches) SetResourceFs(fs afero.Fs) {
+ for _, c := range f {
+ if c.isResourceDir {
+ if c.isInited {
+ panic("cannot set resource fs after init")
+ }
+ c.Fs = hugofs.NewBasePathFs(fs, c.baseDir)
+ }
+ }
+}
+
// Get gets a named cache, nil if none found.
func (f Caches) Get(name string) *Cache {
return f[strings.ToLower(name)]
// NewCaches creates a new set of file caches from the given
// configuration.
-func NewCaches(p *helpers.PathSpec) (Caches, error) {
- dcfg := p.Cfg.GetConfigSection("caches").(Configs)
- fs := p.Fs.Source
+func NewCaches(dcfg Configs, sourceFs afero.Fs) (Caches, error) {
+ // dcfg := p.Cfg.GetConfigSection("caches").(Configs)
+ fs := sourceFs
m := make(Caches)
for k, v := range dcfg {
var cfs afero.Fs
-
if v.IsResourceDir {
- cfs = p.BaseFs.ResourcesCache
+ cfs = nil // Set later. TODO(bep) this needs to be cleanded up.
} else {
cfs = fs
}
- if cfs == nil {
- panic("nil fs")
- }
-
baseDir := v.DirCompiled
bfs := hugofs.NewBasePathFs(cfs, baseDir)
pruneAllRootDir = "pkg"
}
- m[k] = NewCache(bfs, v.MaxAge, pruneAllRootDir)
+ c := NewCache(bfs, v.MaxAge, pruneAllRootDir)
+ c.isResourceDir = v.IsResourceDir
+ c.baseDir = baseDir
+
+ m[k] = c
}
return m, nil
}
const (
- CacheKeyGetJSON = "getjson"
- CacheKeyGetCSV = "getcsv"
- CacheKeyImages = "images"
- CacheKeyAssets = "assets"
- CacheKeyModules = "modules"
- CacheKeyGetResource = "getresource"
- CacheKeyMisc = "misc"
+ CacheKeyGetJSON = "getjson"
+ CacheKeyGetCSV = "getcsv"
+ CacheKeyImages = "images"
+ CacheKeyAssets = "assets"
+ CacheKeyModules = "modules"
+ CacheKeyModuleQueries = "modulequeries"
+ CacheKeyGetResource = "getresource"
+ CacheKeyMisc = "misc"
)
type Configs map[string]FileCacheConfig
MaxAge: -1,
Dir: ":cacheDir/modules",
},
+ CacheKeyModuleQueries: {
+ MaxAge: 24 * time.Hour,
+ Dir: ":cacheDir/modules",
+ },
CacheKeyGetJSON: defaultCacheConfig,
CacheKeyGetCSV: defaultCacheConfig,
CacheKeyImages: {
return f[CacheKeyModules]
}
+// ModuleQueriesCache gets the file cache for Hugo Module version queries.
+// Returns nil if not found.
+func (f Caches) ModuleQueriesCache() *Cache {
+ c, ok := f[CacheKeyModuleQueries]
+ if !ok {
+ panic("module queries cache not set")
+ }
+ return c
+}
+
// AssetsCache gets the file cache for assets (processed resources, SCSS etc.).
func (f Caches) AssetsCache() *Cache {
return f[CacheKeyAssets]
c.Assert(err, qt.IsNil)
fs := afero.NewMemMapFs()
decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches
- c.Assert(len(decoded), qt.Equals, 7)
+ c.Assert(len(decoded), qt.Equals, 8)
c2 := decoded["getcsv"]
c.Assert(c2.MaxAge.String(), qt.Equals, "11h0m0s")
c.Assert(err, qt.IsNil)
fs := afero.NewMemMapFs()
decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches
- c.Assert(len(decoded), qt.Equals, 7)
+ c.Assert(len(decoded), qt.Equals, 8)
for _, v := range decoded {
c.Assert(v.MaxAge, qt.Equals, time.Duration(0))
fs := afero.NewMemMapFs()
decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches
- c.Assert(len(decoded), qt.Equals, 7)
+ c.Assert(len(decoded), qt.Equals, 8)
imgConfig := decoded[filecache.CacheKeyImages]
jsonConfig := decoded[filecache.CacheKeyGetJSON]
for _, name := range []string{filecache.CacheKeyGetCSV, filecache.CacheKeyGetJSON, filecache.CacheKeyAssets, filecache.CacheKeyImages} {
msg := qt.Commentf("cache: %s", name)
- p := newPathsSpec(t, afero.NewMemMapFs(), configStr)
- caches, err := filecache.NewCaches(p)
+ fs := afero.NewMemMapFs()
+ p := newPathsSpec(t, fs, configStr)
+ fileCachConfig := p.Cfg.GetConfigSection("caches").(filecache.Configs)
+ caches, err := filecache.NewCaches(fileCachConfig, fs)
c.Assert(err, qt.IsNil)
+ caches.SetResourceFs(fs)
cache := caches[name]
for i := range 10 {
id := fmt.Sprintf("i%d", i)
}
}
- caches, err = filecache.NewCaches(p)
+ caches, err = filecache.NewCaches(fileCachConfig, fs)
c.Assert(err, qt.IsNil)
+ caches.SetResourceFs(fs)
cache = caches[name]
// Touch one and then prune.
cache.GetOrCreateBytes("i5", func() ([]byte, error) {
configStr = strings.Replace(configStr, "\\", winPathSep, -1)
p := newPathsSpec(t, osfs, configStr)
+ fileCachConfig := p.Cfg.GetConfigSection("caches").(filecache.Configs)
- caches, err := filecache.NewCaches(p)
+ caches, err := filecache.NewCaches(fileCachConfig, p.Fs.Source)
c.Assert(err, qt.IsNil)
+ caches.SetResourceFs(p.SourceFs)
cache := caches.Get("GetJSON")
c.Assert(cache, qt.Not(qt.IsNil))
r.Close()
c.Assert(string(b), qt.Equals, "Hugo is great!")
- info, b, err = caches.ImageCache().GetBytes("mykey")
+ info, b, err = caches.ImageCache().GetItemBytes("mykey")
c.Assert(err, qt.IsNil)
c.Assert(info.Name, qt.Equals, "mykey")
c.Assert(string(b), qt.Equals, "Hugo is great!")
`
p := newPathsSpec(t, afero.NewMemMapFs(), configStr)
-
- caches, err := filecache.NewCaches(p)
+ fileCachConfig := p.Cfg.GetConfigSection("caches").(filecache.Configs)
+ caches, err := filecache.NewCaches(fileCachConfig, p.Fs.Source)
c.Assert(err, qt.IsNil)
+ caches.SetResourceFs(p.Fs.Source)
const cacheName = "getjson"
return h.Sum64(), nil
}
-// XxHashFromStringHexEncoded calculates the xxHash for the given string
+// XxHashFromStringHexEncoded calculates the xxHash for the given strings
// and returns the hash as a hex encoded string.
-func XxHashFromStringHexEncoded(f string) string {
+func XxHashFromStringHexEncoded(s ...string) string {
h := xxhash.New()
- h.WriteString(f)
+ for _, f := range s {
+ h.WriteString(f)
+ }
hash := h.Sum(nil)
return hex.EncodeToString(hash)
}
Modules modules.Modules
ModulesClient *modules.Client
+ FileCaches filecache.Caches
// All below is set in Init.
Languages langs.Languages
return c == nil || len(c.Languages) == 0
}
-func (c *Configs) Init(logger loggers.Logger) error {
+func (c *Configs) Init(sourceFs afero.Fs, logger loggers.Logger) error {
var languages langs.Languages
for _, f := range c.Base.Languages.Config.Sorted {
l.CommonDirs.CacheDir = bcfg.CacheDir
}
+ caches, err := filecache.NewCaches(all.Caches, fs)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create file caches from configuration: %w", err)
+ }
+
cm := &Configs{
Base: all,
LanguageConfigMap: langConfigMap,
LoadingInfo: res,
+ FileCaches: caches,
IsMultihost: isMultihost,
}
)
type ConfigLanguage struct {
- config *Config
- baseConfig config.BaseConfig
-
+ config *Config
+ baseConfig config.BaseConfig
m *Configs
language *langs.Language
languageIndex int
return c.baseConfig
}
+func (c ConfigLanguage) FileCaches() any {
+ return c.m.FileCaches
+}
+
func (c ConfigLanguage) Dirs() config.CommonDirs {
return c.config.CommonDirs
}
configs.Modules = moduleConfig.AllModules
configs.ModulesClient = modulesClient
- if err := configs.Init(d.Logger); err != nil {
+ if err := configs.Init(d.Fs, d.Logger); err != nil {
return nil, fmt.Errorf("failed to init config: %w", err)
}
PublishDir: publishDir,
Environment: l.Environment,
CacheDir: conf.Caches.CacheDirModules(),
+ ModuleQueriesCache: configs.FileCaches.ModuleQueriesCache(),
ModuleConfig: conf.Module,
IgnoreVendor: ignoreVendor,
IgnoreModuleDoesNotExist: ignoreModuleDoesNotExist,
Dirs() CommonDirs
Quiet() bool
DirsBase() CommonDirs
+ FileCaches() any
ContentTypes() ContentTypesProvider
GetConfigSection(string) any
GetConfig() any
common = d.ResourceSpec.SpecCommon
}
- fileCaches, err := filecache.NewCaches(d.PathSpec)
- if err != nil {
- return fmt.Errorf("failed to create file caches from configuration: %w", err)
- }
+ d.Cfg.BaseConfig()
+
+ fileCaches := d.Cfg.FileCaches().(filecache.Caches)
+ fileCaches.SetResourceFs(d.BaseFs.ResourcesCache)
resourceSpec, err := resources.NewSpec(d.PathSpec, common, d.WasmDispatchers, fileCaches, d.MemCache, d.BuildState, d.Log, d, d.ExecHelper, d.BuildClosers, d.BuildState)
if err != nil {
// Run tests
func Test() error {
env := map[string]string{"GOFLAGS": testGoFlags()}
- return runCmd(env, goexe, "test", "-p", "2", "./...", "-tags", buildTags())
+ if isCI() {
+ // We have space issues on GitHub Actions (lots tests, incresing usage of Go generics).
+ // Test each package separately and clean up in between.
+ pkgs, err := hugoPackages()
+ if err != nil {
+ return err
+ }
+ for _, pkg := range pkgs {
+ if err := cmp.Or(CleanTest(), UninstallAll()); err != nil {
+ return err
+ }
+ if err := runCmd(env, goexe, "test", "-p", "2", pkg, "-tags", buildTags()); err != nil {
+ return err
+ }
+ }
+ return nil
+ } else {
+ return runCmd(env, goexe, "test", "-p", "2", "./...", "-tags", buildTags())
+ }
}
// Run tests with race detector
return err
}
for _, pkg := range pkgs {
- /*slashCount := strings.Count(pkg, "/")
- if slashCount > 1 {
- continue
- }
- if pkg != "." {
- pkg += "/..."
- }*/
if err := cmp.Or(CleanTest(), UninstallAll()); err != nil {
return err
}
}
}
return nil
-
} else {
return runCmd(env, goexe, "test", "-p", "2", "-race", "./...", "-tags", buildTags())
}
"time"
"github.com/gohugoio/hugo/common/collections"
+ "github.com/gohugoio/hugo/common/hashing"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/hexec"
+ "github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/config"
"golang.org/x/mod/module"
- "github.com/gohugoio/hugo/common/hugio"
-
"github.com/spf13/afero"
)
}
func (c *Client) downloadModuleVersion(path, version string) (*goModule, error) {
+ // If it's already cached, use it.
+ const keyPrefix = "downloadmoduleversion_"
+ cacheKey := keyPrefix + hashing.XxHashFromStringHexEncoded(c.ccfg.CacheDir, path, version) + ".json"
+ if b, _ := c.ccfg.ModuleQueriesCache.GetBytes(cacheKey); len(b) > 0 {
+ var cm goModule
+ if err := json.Unmarshal(b, &cm); err == nil && cm.Dir != "" {
+ if c.isDirNonEmpty(cm.Dir) {
+ return &cm, nil
+ }
+ }
+ }
+
+ // No valid cache, need to query.
args := []string{"mod", "download", "-json", fmt.Sprintf("%s@%s", path, version)}
- b := &bytes.Buffer{}
+ b := &bytes.Buffer{}
err := c.runGo(context.Background(), b, args...)
if err != nil {
return nil, fmt.Errorf("failed to download module %s@%s: %w", path, version, err)
}
+ jsonResult := b.Bytes()
+
m := &goModule{}
- if err := json.NewDecoder(b).Decode(m); err != nil {
+ if err := json.NewDecoder(bytes.NewReader(jsonResult)).Decode(m); err != nil {
return nil, fmt.Errorf("failed to decode module download result: %w", err)
}
return nil, errors.New(m.Error.Err)
}
+ // Cache the successful result if it has a Dir field.
+ if m.Dir != "" {
+ _ = c.ccfg.ModuleQueriesCache.SetBytes(cacheKey, jsonResult)
+ }
+
return m, nil
}
+// isDirNonEmpty returns true if dir exists and contains at least one file or subdirectory.
+func (c *Client) isDirNonEmpty(dir string) bool {
+ f, err := c.fs.Open(dir)
+ if err != nil {
+ return false
+ }
+ defer f.Close()
+ // Read just one entry to check if directory is non-empty.
+ _, err = f.Readdirnames(1)
+ return err == nil
+}
+
func (c *Client) listGoMods() (goModules, error) {
if c.GoModulesFilename == "" || !c.moduleConfig.hasModuleImport() {
return nil, nil
Exec *hexec.Exec
- CacheDir string // Module cache
- ModuleConfig Config
+ CacheDir string // Module cache
+ ModuleQueriesCache ModuleQueriesCache
+ ModuleConfig Config
+}
+
+// ModuleQueriesCache is a cache for module version queries.
+type ModuleQueriesCache interface {
+ GetBytes(id string) ([]byte, error)
+ SetBytes(id string, data []byte) error
}
func (c ClientConfig) shouldIgnoreVendor(path string) bool {
pk := pathVersionKey{path: tc.Path(), version: tc.Version()}
seenInCurrent := seen[pk]
if seenInCurrent {
- // Only one import of the same module per project.
- if owner.projectMod {
- // In Hugo v0.150.0 we introduced direct dependencies, and it may be tempting to import the same version
- // with different mount setups. We may allow that in the future, but we need to get some experience first.
- // For now, we just warn. The user needs to add multiple mount points in the same import.
- c.logger.Warnf("module with path %q is imported for the same version %q more than once", tc.Path(), tc.Version())
+ // Only one import of the same module per project, unless this is the main project.
+ if !owner.projectMod {
+ c.logger.Warnf("module with path %q is imported for the same version %q more than once; this is currently only allowed in the main project's config.", tc.Path(), tc.Version())
+ continue
}
- continue
}
seen[pk] = true
var meta transformedResourceMetadata
filenameMeta, filenameContent := c.getFilenames(key)
- _, jsonContent, _ := c.fileCache.GetBytes(filenameMeta)
+ jsonContent, _ := c.fileCache.GetBytes(filenameMeta)
if jsonContent == nil {
return filecache.ItemInfo{}, nil, meta, false
}
"scripts": {},
"devDependencies": {
- "@babel/cli": "7.8.4",
- "@babel/core": "7.9.0",
- "@babel/preset-env": "7.9.5"
+ "@babel/cli": "7.28.6",
+ "@babel/core": "7.28.6",
+ "@babel/preset-env": "7.28.6"
}
}
projects / brevno-suite / hugo / commitdiff