If there's an error doing I/O to block device, and the client resends the
I/O to the MDS, the MDS must recall the layout from the client before
processing the I/O.  Let's preempt that exchange by returning the layout
before falling back to the MDS when there's an error.
Signed-off-by: Benjamin Coddington <bcodding@redhat.com>
Signed-off-by: Trond Myklebust <trond.myklebust@primarydata.com>
        .name                           = "LAYOUT_BLOCK_VOLUME",
        .owner                          = THIS_MODULE,
        .flags                          = PNFS_LAYOUTRET_ON_SETATTR |
+                                         PNFS_LAYOUTRET_ON_ERROR |
                                          PNFS_READ_WHOLE_PAGE,
        .read_pagelist                  = bl_read_pagelist,
        .write_pagelist                 = bl_write_pagelist,
        .name                           = "LAYOUT_SCSI",
        .owner                          = THIS_MODULE,
        .flags                          = PNFS_LAYOUTRET_ON_SETATTR |
+                                         PNFS_LAYOUTRET_ON_ERROR |
                                          PNFS_READ_WHOLE_PAGE,
        .read_pagelist                  = bl_read_pagelist,
        .write_pagelist                 = bl_write_pagelist,
 
        struct nfs_inode *nfsi = NFS_I(ino);
        struct nfs_server *nfss = NFS_SERVER(ino);
 
-       if (pnfs_enabled_sb(nfss) && nfsi->layout)
+       if (pnfs_enabled_sb(nfss) && nfsi->layout) {
+               set_bit(NFS_LAYOUT_RETURN_REQUESTED, &nfsi->layout->plh_flags);
                return _pnfs_return_layout(ino);
+       }
 
        return 0;
 }