return retval;
        }
 
+       /*
+        * We need to protect against old.inode directory getting converted
+        * from inline directory format into a normal one.
+        */
+       if (S_ISDIR(old.inode->i_mode))
+               inode_lock_nested(old.inode, I_MUTEX_NONDIR2);
+
        old.bh = ext4_find_entry(old.dir, &old.dentry->d_name, &old.de,
                                 &old.inlined);
-       if (IS_ERR(old.bh))
-               return PTR_ERR(old.bh);
+       if (IS_ERR(old.bh)) {
+               retval = PTR_ERR(old.bh);
+               goto unlock_moved_dir;
+       }
+
        /*
         *  Check for inode number is _not_ due to possible IO errors.
         *  We might rmdir the source, keep it as pwd of some process
                        if (new.dir != old.dir && EXT4_DIR_LINK_MAX(new.dir))
                                goto end_rename;
                }
-               /*
-                * We need to protect against old.inode directory getting
-                * converted from inline directory format into a normal one.
-                */
-               inode_lock_nested(old.inode, I_MUTEX_NONDIR2);
                retval = ext4_rename_dir_prepare(handle, &old);
                if (retval) {
                        inode_unlock(old.inode);
        } else {
                ext4_journal_stop(handle);
        }
-       if (old.dir_bh)
-               inode_unlock(old.inode);
 release_bh:
        brelse(old.dir_bh);
        brelse(old.bh);
        brelse(new.bh);
+
+unlock_moved_dir:
+       if (S_ISDIR(old.inode->i_mode))
+               inode_unlock(old.inode);
+
        return retval;
 }