"mime"
"os"
"path/filepath"
+ "regexp"
"runtime"
+ "sort"
"strings"
"sync"
type Deployer struct {
localFs afero.Fs
- target *target // the target to deploy to
- matchers []*matcher // matchers to apply to uploaded files
- quiet bool // true reduces STDOUT
- confirm bool // true enables confirmation before making changes
- dryRun bool // true skips conformations and prints changes instead of applying them
- force bool // true forces upload of all files
- invalidateCDN bool // true enables invalidate CDN cache (if possible)
- maxDeletes int // caps the # of files to delete; -1 to disable
+ target *target // the target to deploy to
+ matchers []*matcher // matchers to apply to uploaded files
+ ordering []*regexp.Regexp // orders uploads
+ quiet bool // true reduces STDOUT
+ confirm bool // true enables confirmation before making changes
+ dryRun bool // true skips conformations and prints changes instead of applying them
+ force bool // true forces upload of all files
+ invalidateCDN bool // true enables invalidate CDN cache (if possible)
+ maxDeletes int // caps the # of files to delete; -1 to disable
}
// New constructs a new *Deployer.
localFs: localFs,
target: tgt,
matchers: dcfg.Matchers,
+ ordering: dcfg.ordering,
quiet: cfg.GetBool("quiet"),
confirm: cfg.GetBool("confirm"),
dryRun: cfg.GetBool("dryRun"),
}
}
+ // Order the uploads. They are organized in groups; all uploads in a group
+ // must be complete before moving on to the next group.
+ uploadGroups := applyOrdering(d.ordering, uploads)
+
// Apply the changes in parallel, using an inverted worker
// pool (https://www.youtube.com/watch?v=5zXAHh5tJqQ&t=26m58s).
// sem prevents more than nParallel concurrent goroutines.
const nParallel = 10
- sem := make(chan struct{}, nParallel)
var errs []error
var errMu sync.Mutex // protects errs
- for _, upload := range uploads {
- if d.dryRun {
- if !d.quiet {
- jww.FEEDBACK.Printf("[DRY RUN] Would upload: %v\n", upload)
- }
+ for _, uploads := range uploadGroups {
+ // Short-circuit for an empty group.
+ if len(uploads) == 0 {
continue
}
- // TODO: Add a progress indicator, as this can take a while
- // depending on the number of files, upload speed, and size of the
- // site.
-
- sem <- struct{}{}
- go func(upload *fileToUpload) {
- if err := doSingleUpload(ctx, bucket, upload); err != nil {
- errMu.Lock()
- defer errMu.Unlock()
- errs = append(errs, err)
+ // Within the group, apply uploads in parallel.
+ sem := make(chan struct{}, nParallel)
+ for _, upload := range uploads {
+ if d.dryRun {
+ if !d.quiet {
+ jww.FEEDBACK.Printf("[DRY RUN] Would upload: %v\n", upload)
+ }
+ continue
}
- <-sem
- }(upload)
+
+ sem <- struct{}{}
+ go func(upload *fileToUpload) {
+ if err := doSingleUpload(ctx, bucket, upload); err != nil {
+ errMu.Lock()
+ defer errMu.Unlock()
+ errs = append(errs, err)
+ }
+ <-sem
+ }(upload)
+ }
+ // Wait for all uploads in the group to finish.
+ for n := nParallel; n > 0; n-- {
+ sem <- struct{}{}
+ }
}
if d.maxDeletes != -1 && len(deletes) > d.maxDeletes {
jww.WARN.Printf("Skipping %d deletes because it is more than --maxDeletes (%d). If this is expected, set --maxDeletes to a larger number, or -1 to disable this check.\n", len(deletes), d.maxDeletes)
} else {
+ // Apply deletes in parallel.
+ sort.Slice(deletes, func(i, j int) bool { return deletes[i] < deletes[j] })
+ sem := make(chan struct{}, nParallel)
for _, del := range deletes {
if d.dryRun {
if !d.quiet {
<-sem
}(del)
}
- }
- // Wait for all uploads/deletes to finish.
- for n := nParallel; n > 0; n-- {
- sem <- struct{}{}
+ // Wait for all deletes to finish.
+ for n := nParallel; n > 0; n-- {
+ sem <- struct{}{}
+ }
}
if len(errs) > 0 {
if !d.quiet {
}
return uploads, deletes
}
+
+// applyOrdering returns an ordered slice of slices of uploads.
+//
+// The returned slice will have length len(ordering)+1.
+//
+// The subslice at index i, for i = 0 ... len(ordering)-1, will have all of the
+// uploads whose Local.Path matched the regex at ordering[i] (but not any
+// previous ordering regex).
+// The subslice at index len(ordering) will have the remaining uploads that
+// didn't match any ordering regex.
+//
+// The subslices are sorted by Local.Path.
+func applyOrdering(ordering []*regexp.Regexp, uploads []*fileToUpload) [][]*fileToUpload {
+
+ // Sort the whole slice by Local.Path first.
+ sort.Slice(uploads, func(i, j int) bool { return uploads[i].Local.Path < uploads[j].Local.Path })
+
+ retval := make([][]*fileToUpload, len(ordering)+1)
+ for _, u := range uploads {
+ matched := false
+ for i, re := range ordering {
+ if re.MatchString(u.Local.Path) {
+ retval[i] = append(retval[i], u)
+ matched = true
+ break
+ }
+ }
+ if !matched {
+ retval[len(ordering)] = append(retval[len(ordering)], u)
+ }
+ }
+ return retval
+}
type deployConfig struct {
Targets []*target
Matchers []*matcher
+ Order []string
+
+ ordering []*regexp.Regexp // compiled Order
}
type target struct {
return dcfg, fmt.Errorf("invalid deployment.matchers.pattern: %v", err)
}
}
+ for _, o := range dcfg.Order {
+ re, err := regexp.Compile(o)
+ if err != nil {
+ return dcfg, fmt.Errorf("invalid deployment.orderings.pattern: %v", err)
+ }
+ dcfg.ordering = append(dcfg.ordering, re)
+ }
return dcfg, nil
}
someOtherValue = "foo"
[deployment]
+
+order = ["o1", "o2"]
+
[[deployment.targets]]
Name = "name1"
URL = "url1"
dcfg, err := decodeConfig(cfg)
assert.NoError(err)
+ assert.Equal(2, len(dcfg.Order))
+ assert.Equal("o1", dcfg.Order[0])
+ assert.Equal("o2", dcfg.Order[1])
+ assert.Equal(2, len(dcfg.ordering))
+
assert.Equal(2, len(dcfg.Targets))
assert.Equal("name1", dcfg.Targets[0].Name)
assert.Equal("url1", dcfg.Targets[0].URL)
assert.Equal(2, len(dcfg.Matchers))
assert.Equal("^pattern1$", dcfg.Matchers[0].Pattern)
+ assert.NotNil(dcfg.Matchers[0].re)
assert.Equal("cachecontrol1", dcfg.Matchers[0].CacheControl)
assert.Equal("contentencoding1", dcfg.Matchers[0].ContentEncoding)
assert.Equal("contenttype1", dcfg.Matchers[0].ContentType)
assert.True(dcfg.Matchers[0].Gzip)
assert.True(dcfg.Matchers[0].Force)
+ assert.Equal("^pattern2$", dcfg.Matchers[1].Pattern)
+ assert.NotNil(dcfg.Matchers[1].re)
+ assert.Equal("cachecontrol2", dcfg.Matchers[1].CacheControl)
+ assert.Equal("contentencoding2", dcfg.Matchers[1].ContentEncoding)
+ assert.Equal("contenttype2", dcfg.Matchers[1].ContentType)
+ assert.False(dcfg.Matchers[1].Gzip)
+ assert.False(dcfg.Matchers[1].Force)
+}
+
+func TestInvalidOrderingPattern(t *testing.T) {
+ assert := require.New(t)
+
+ tomlConfig := `
+
+someOtherValue = "foo"
+
+[deployment]
+order = ["["] # invalid regular expression
+`
+ cfg, err := config.FromConfigString(tomlConfig, "toml")
+ assert.NoError(err)
+
+ _, err = decodeConfig(cfg)
+ assert.Error(err)
}
func TestInvalidMatcherPattern(t *testing.T) {
"crypto/md5"
"io/ioutil"
"os"
+ "regexp"
"sort"
"testing"
remote[r.Key] = r
}
gotUpdates, gotDeletes := findDiffs(local, remote, tc.Force)
- sort.Slice(gotUpdates, func(i, j int) bool { return gotUpdates[i].Local.Path < gotUpdates[j].Local.Path })
+ gotUpdates = applyOrdering(nil, gotUpdates)[0]
sort.Slice(gotDeletes, func(i, j int) bool { return gotDeletes[i] < gotDeletes[j] })
if diff := cmp.Diff(gotUpdates, tc.WantUpdates, cmpopts.IgnoreUnexported(localFile{})); diff != "" {
t.Errorf("updates differ:\n%s", diff)
-
}
if diff := cmp.Diff(gotDeletes, tc.WantDeletes); diff != "" {
t.Errorf("deletes differ:\n%s", diff)
})
}
}
+
+func TestOrdering(t *testing.T) {
+ tests := []struct {
+ Description string
+ Uploads []string
+ Ordering []*regexp.Regexp
+ Want [][]string
+ }{
+ {
+ Description: "empty",
+ Want: [][]string{nil},
+ },
+ {
+ Description: "no ordering",
+ Uploads: []string{"c", "b", "a", "d"},
+ Want: [][]string{{"a", "b", "c", "d"}},
+ },
+ {
+ Description: "one ordering",
+ Uploads: []string{"db", "c", "b", "a", "da"},
+ Ordering: []*regexp.Regexp{regexp.MustCompile("^d")},
+ Want: [][]string{{"da", "db"}, {"a", "b", "c"}},
+ },
+ {
+ Description: "two orderings",
+ Uploads: []string{"db", "c", "b", "a", "da"},
+ Ordering: []*regexp.Regexp{
+ regexp.MustCompile("^d"),
+ regexp.MustCompile("^b"),
+ },
+ Want: [][]string{{"da", "db"}, {"b"}, {"a", "c"}},
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.Description, func(t *testing.T) {
+ uploads := make([]*fileToUpload, len(tc.Uploads))
+ for i, u := range tc.Uploads {
+ uploads[i] = &fileToUpload{Local: &localFile{Path: u}}
+ }
+ gotUploads := applyOrdering(tc.Ordering, uploads)
+ var got [][]string
+ for _, subslice := range gotUploads {
+ var gotsubslice []string
+ for _, u := range subslice {
+ gotsubslice = append(gotsubslice, u.Local.Path)
+ }
+ got = append(got, gotsubslice)
+ }
+ if diff := cmp.Diff(got, tc.Want); diff != "" {
+ t.Error(diff)
+ }
+ })
+ }
+}
projects / brevno-suite / hugo / commitdiff