deploy: Add include and exclude support for remote

diff --git a/deploy/deploy.go b/deploy/deploy.go
index c0c6ed4f3977c9e2c451ec8c4e03e7f1a70fb3ad..c7a4510c2a90eb4a698dfd00aa59a9dbe115de52 100644 (file)
--- a/deploy/deploy.go
+++ b/deploy/deploy.go
@@ -138,7 +138,7 @@ func (d *Deployer) Deploy(ctx context.Context) error {
        d.summary.NumLocal = len(local)
 
        // Load remote files from the target.
-       remote, err := walkRemote(ctx, bucket)
+       remote, err := walkRemote(ctx, bucket, include, exclude)
        if err != nil {
                return err
        }
@@ -499,7 +499,7 @@ func walkLocal(fs afero.Fs, matchers []*matcher, include, exclude glob.Glob) (ma
 }
 
 // walkRemote walks the target bucket and returns a flat list.
-func walkRemote(ctx context.Context, bucket *blob.Bucket) (map[string]*blob.ListObject, error) {
+func walkRemote(ctx context.Context, bucket *blob.Bucket, include, exclude glob.Glob) (map[string]*blob.ListObject, error) {
        retval := map[string]*blob.ListObject{}
        iter := bucket.List(nil)
        for {
@@ -510,6 +510,15 @@ func walkRemote(ctx context.Context, bucket *blob.Bucket) (map[string]*blob.List
                if err != nil {
                        return nil, err
                }
+               // Check include/exclude matchers.
+               if include != nil && !include.Match(obj.Key) {
+                       jww.INFO.Printf("  remote dropping %q due to include\n", obj.Key)
+                       continue
+               }
+               if exclude != nil && exclude.Match(obj.Key) {
+                       jww.INFO.Printf("  remote dropping %q due to exclude\n", obj.Key)
+                       continue
+               }
                // If the remote didn't give us an MD5, compute one.
                // This can happen for some providers (e.g., fileblob, which uses the
                // local filesystem), but not for the most common Cloud providers

diff --git a/deploy/deploy_test.go b/deploy/deploy_test.go
index be1a628d2e5e669a7a826be4bbbf5d2ef45a3bfa..6ada0cf8e9bf49fdc2670628e918ec2a54cc87a0 100644 (file)
--- a/deploy/deploy_test.go
+++ b/deploy/deploy_test.go
@@ -720,6 +720,88 @@ func TestIncludeExclude(t *testing.T) {
        }
 }
 
+// TestIncludeExcludeRemoteDelete verifies deleted local files that don't match include/exclude patterns
+// are not deleted on the remote.
+func TestIncludeExcludeRemoteDelete(t *testing.T) {
+       ctx := context.Background()
+
+       tests := []struct {
+               Include string
+               Exclude string
+               Want    deploySummary
+       }{
+               {
+                       Want: deploySummary{NumLocal: 3, NumRemote: 5, NumUploads: 0, NumDeletes: 2},
+               },
+               {
+                       Include: "**aaa",
+                       Want:    deploySummary{NumLocal: 2, NumRemote: 3, NumUploads: 0, NumDeletes: 1},
+               },
+               {
+                       Include: "subdir/**",
+                       Want:    deploySummary{NumLocal: 1, NumRemote: 2, NumUploads: 0, NumDeletes: 1},
+               },
+               {
+                       Exclude: "**bbb",
+                       Want:    deploySummary{NumLocal: 2, NumRemote: 3, NumUploads: 0, NumDeletes: 1},
+               },
+               {
+                       Exclude: "bbb",
+                       Want:    deploySummary{NumLocal: 3, NumRemote: 4, NumUploads: 0, NumDeletes: 1},
+               },
+       }
+       for _, test := range tests {
+               t.Run(fmt.Sprintf("include %q exclude %q", test.Include, test.Exclude), func(t *testing.T) {
+                       fsTests, cleanup, err := initFsTests()
+                       if err != nil {
+                               t.Fatal(err)
+                       }
+                       defer cleanup()
+                       fsTest := fsTests[1] // just do file-based test
+
+                       local, err := initLocalFs(ctx, fsTest.fs)
+                       if err != nil {
+                               t.Fatal(err)
+                       }
+                       deployer := &Deployer{
+                               localFs:    fsTest.fs,
+                               maxDeletes: -1,
+                               bucket:     fsTest.bucket,
+                       }
+
+                       // Initial sync to get the files on the remote
+                       if err := deployer.Deploy(ctx); err != nil {
+                               t.Errorf("deploy: failed: %v", err)
+                       }
+
+                       // Delete two files, [1] and [2].
+                       if err := fsTest.fs.Remove(local[1].Name); err != nil {
+                               t.Fatal(err)
+                       }
+                       if err := fsTest.fs.Remove(local[2].Name); err != nil {
+                               t.Fatal(err)
+                       }
+
+                       // Second sync
+                       tgt := &target{
+                               Include: test.Include,
+                               Exclude: test.Exclude,
+                       }
+                       if err := tgt.parseIncludeExclude(); err != nil {
+                               t.Error(err)
+                       }
+                       deployer.target = tgt
+                       if err := deployer.Deploy(ctx); err != nil {
+                               t.Errorf("deploy: failed: %v", err)
+                       }
+
+                       if !cmp.Equal(deployer.summary, test.Want) {
+                               t.Errorf("deploy: got %v, want %v", deployer.summary, test.Want)
+                       }
+               })
+       }
+}
+
 // TestCompression verifies that gzip compression works correctly.
 // In particular, MD5 hashes must be of the compressed content.
 func TestCompression(t *testing.T) {

diff --git a/docs/content/en/hosting-and-deployment/hugo-deploy.md b/docs/content/en/hosting-and-deployment/hugo-deploy.md
index a571d366d6e3696dc841d1232f41fd1d7ce229d7..178c9b1767f9c2977a34e389a86ba7d1d571172d 100644 (file)
--- a/docs/content/en/hosting-and-deployment/hugo-deploy.md
+++ b/docs/content/en/hosting-and-deployment/hugo-deploy.md
@@ -85,8 +85,11 @@ cloudFrontDistributionID = <ID>
 # Optionally, you can include or exclude specific files.
 # See https://godoc.org/github.com/gobwas/glob#Glob for the glob pattern syntax.
 # If non-empty, the pattern is matched against the local path.
-# If exclude is non-empty, and a file's path matches it, that file is dropped.
-# If include is non-empty, and a file's path does not match it, that file is dropped.
+# All paths are matched against in their filepath.ToSlash form.
+# If exclude is non-empty, and a local or remote file's path matches it, that file is not synced.
+# If include is non-empty, and a local or remote file's path does not match it, that file is not synced.
+# As a result, local files that don't pass the include/exclude filters are not uploaded to remote,
+# and remote files that don't pass the include/exclude filters are not deleted.
 # include = "**.html" # would only include files with ".html" suffix
 # exclude = "**.{jpg, png}" # would exclude files with ".jpg" or ".png" suffix