{
        struct dp83822_private *dp83822 = phydev->priv;
        struct device *dev = &phydev->mdio.dev;
-       int rgmii_delay;
+       int rgmii_delay = 0;
        s32 rx_int_delay;
        s32 tx_int_delay;
        int err = 0;
                rx_int_delay = phy_get_internal_delay(phydev, dev, NULL, 0,
                                                      true);
 
-               if (rx_int_delay <= 0)
-                       rgmii_delay = 0;
-               else
-                       rgmii_delay = DP83822_RX_CLK_SHIFT;
+               /* Set DP83822_RX_CLK_SHIFT to enable rx clk internal delay */
+               if (rx_int_delay > 0)
+                       rgmii_delay |= DP83822_RX_CLK_SHIFT;
 
                tx_int_delay = phy_get_internal_delay(phydev, dev, NULL, 0,
                                                      false);
+
+               /* Set DP83822_TX_CLK_SHIFT to disable tx clk internal delay */
                if (tx_int_delay <= 0)
-                       rgmii_delay &= ~DP83822_TX_CLK_SHIFT;
-               else
                        rgmii_delay |= DP83822_TX_CLK_SHIFT;
 
-               if (rgmii_delay) {
-                       err = phy_set_bits_mmd(phydev, DP83822_DEVADDR,
-                                              MII_DP83822_RCSR, rgmii_delay);
-                       if (err)
-                               return err;
-               }
+               err = phy_modify_mmd(phydev, DP83822_DEVADDR, MII_DP83822_RCSR,
+                                    DP83822_RX_CLK_SHIFT | DP83822_TX_CLK_SHIFT, rgmii_delay);
+               if (err)
+                       return err;
+
+               err = phy_set_bits_mmd(phydev, DP83822_DEVADDR,
+                                      MII_DP83822_RCSR, DP83822_RGMII_MODE_EN);
 
-               phy_set_bits_mmd(phydev, DP83822_DEVADDR,
-                                       MII_DP83822_RCSR, DP83822_RGMII_MODE_EN);
+               if (err)
+                       return err;
        } else {
-               phy_clear_bits_mmd(phydev, DP83822_DEVADDR,
-                                       MII_DP83822_RCSR, DP83822_RGMII_MODE_EN);
+               err = phy_clear_bits_mmd(phydev, DP83822_DEVADDR,
+                                        MII_DP83822_RCSR, DP83822_RGMII_MODE_EN);
+
+               if (err)
+                       return err;
        }
 
        if (dp83822->fx_enabled) {