iommu/arm-smmu-v3: Move CD table to arm_smmu_master

With this change, each master will now own its own CD table instead of
sharing one with other masters attached to the same domain. Attaching a
stage 1 domain installs CD entries into the master's CD table. SVA
writes its CD entries into each master's CD table if the domain is
shared across masters.

Also add the device to the devices list before writing the CD to the
table so that SVA will know that the CD needs to be re-written to this
device's CD table as well if it decides to update the CD's ASID
concurrently with this function.

Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Michael Shavit <mshavit@google.com>
Link: https://lore.kernel.org/r/20230915211705.v8.6.Ice063dcf87d1b777a72e008d9e3406d2bcf6d876@changeid
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index eacfa82b921ada4df2dd55ca96ea2d6c3826aa78..96f68161695e9ae682cb972356fea18b5359e392 100644 (file)
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -1025,7 +1025,7 @@ static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, u32 ssid)
        unsigned int idx;
        struct arm_smmu_l1_ctx_desc *l1_desc;
        struct arm_smmu_device *smmu = master->smmu;
-       struct arm_smmu_ctx_desc_cfg *cdcfg = &master->domain->cd_table;
+       struct arm_smmu_ctx_desc_cfg *cdcfg = &master->cd_table;
 
        if (cdcfg->s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
                return cdcfg->cdtab + ssid * CTXDESC_CD_DWORDS;
@@ -1062,7 +1062,7 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid,
        u64 val;
        bool cd_live;
        __le64 *cdptr;
-       struct arm_smmu_ctx_desc_cfg *cd_table = &master->domain->cd_table;
+       struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
 
        if (WARN_ON(ssid >= (1 << cd_table->s1cdmax)))
                return -E2BIG;
@@ -1125,14 +1125,13 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid,
        return 0;
 }
 
-static int arm_smmu_alloc_cd_tables(struct arm_smmu_domain *smmu_domain,
-                                   struct arm_smmu_master *master)
+static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master)
 {
        int ret;
        size_t l1size;
        size_t max_contexts;
        struct arm_smmu_device *smmu = master->smmu;
-       struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->cd_table;
+       struct arm_smmu_ctx_desc_cfg *cdcfg = &master->cd_table;
 
        cdcfg->stall_enabled = master->stall_enabled;
        cdcfg->s1cdmax = master->ssid_bits;
@@ -1176,12 +1175,12 @@ err_free_l1:
        return ret;
 }
 
-static void arm_smmu_free_cd_tables(struct arm_smmu_domain *smmu_domain)
+static void arm_smmu_free_cd_tables(struct arm_smmu_master *master)
 {
        int i;
        size_t size, l1size;
-       struct arm_smmu_device *smmu = smmu_domain->smmu;
-       struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->cd_table;
+       struct arm_smmu_device *smmu = master->smmu;
+       struct arm_smmu_ctx_desc_cfg *cdcfg = &master->cd_table;
 
        if (cdcfg->l1_desc) {
                size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
@@ -1289,7 +1288,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
        if (smmu_domain) {
                switch (smmu_domain->stage) {
                case ARM_SMMU_DOMAIN_S1:
-                       cd_table = &smmu_domain->cd_table;
+                       cd_table = &master->cd_table;
                        break;
                case ARM_SMMU_DOMAIN_S2:
                case ARM_SMMU_DOMAIN_NESTED:
@@ -2062,14 +2061,10 @@ static void arm_smmu_domain_free(struct iommu_domain *domain)
 
        free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 
-       /* Free the CD and ASID, if we allocated them */
+       /* Free the ASID or VMID */
        if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
-               struct arm_smmu_ctx_desc_cfg *cd_table = &smmu_domain->cd_table;
-
                /* Prevent SVA from touching the CD while we're freeing it */
                mutex_lock(&arm_smmu_asid_lock);
-               if (cd_table->cdtab)
-                       arm_smmu_free_cd_tables(smmu_domain);
                arm_smmu_free_asid(&smmu_domain->cd);
                mutex_unlock(&arm_smmu_asid_lock);
        } else {
@@ -2100,10 +2095,6 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
        if (ret)
                goto out_unlock;
 
-       ret = arm_smmu_alloc_cd_tables(smmu_domain, master);
-       if (ret)
-               goto out_free_asid;
-
        cd->asid        = (u16)asid;
        cd->ttbr        = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
        cd->tcr         = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
@@ -2115,17 +2106,9 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
                          CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64;
        cd->mair        = pgtbl_cfg->arm_lpae_s1_cfg.mair;
 
-       ret = arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, cd);
-       if (ret)
-               goto out_free_cd_tables;
-
        mutex_unlock(&arm_smmu_asid_lock);
        return 0;
 
-out_free_cd_tables:
-       arm_smmu_free_cd_tables(smmu_domain);
-out_free_asid:
-       arm_smmu_free_asid(cd);
 out_unlock:
        mutex_unlock(&arm_smmu_asid_lock);
        return ret;
@@ -2389,6 +2372,14 @@ static void arm_smmu_detach_dev(struct arm_smmu_master *master)
        master->domain = NULL;
        master->ats_enabled = false;
        arm_smmu_install_ste_for_dev(master);
+       /*
+        * Clearing the CD entry isn't strictly required to detach the domain
+        * since the table is uninstalled anyway, but it helps avoid confusion
+        * in the call to arm_smmu_write_ctx_desc on the next attach (which
+        * expects the entry to be empty).
+        */
+       if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 && master->cd_table.cdtab)
+               arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, NULL);
 }
 
 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
@@ -2423,23 +2414,14 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
        if (!smmu_domain->smmu) {
                smmu_domain->smmu = smmu;
                ret = arm_smmu_domain_finalise(domain, master);
-               if (ret) {
+               if (ret)
                        smmu_domain->smmu = NULL;
-                       goto out_unlock;
-               }
-       } else if (smmu_domain->smmu != smmu) {
-               ret = -EINVAL;
-               goto out_unlock;
-       } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
-                  master->ssid_bits != smmu_domain->cd_table.s1cdmax) {
+       } else if (smmu_domain->smmu != smmu)
                ret = -EINVAL;
-               goto out_unlock;
-       } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
-                  smmu_domain->cd_table.stall_enabled !=
-                          master->stall_enabled) {
-               ret = -EINVAL;
-               goto out_unlock;
-       }
+
+       mutex_unlock(&smmu_domain->init_mutex);
+       if (ret)
+               return ret;
 
        master->domain = smmu_domain;
 
@@ -2453,16 +2435,42 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
        if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)
                master->ats_enabled = arm_smmu_ats_supported(master);
 
-       arm_smmu_install_ste_for_dev(master);
-
        spin_lock_irqsave(&smmu_domain->devices_lock, flags);
        list_add(&master->domain_head, &smmu_domain->devices);
        spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
 
+       if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
+               if (!master->cd_table.cdtab) {
+                       ret = arm_smmu_alloc_cd_tables(master);
+                       if (ret) {
+                               master->domain = NULL;
+                               goto out_list_del;
+                       }
+               }
+
+               /*
+                * Prevent SVA from concurrently modifying the CD or writing to
+                * the CD entry
+                */
+               mutex_lock(&arm_smmu_asid_lock);
+               ret = arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, &smmu_domain->cd);
+               mutex_unlock(&arm_smmu_asid_lock);
+               if (ret) {
+                       master->domain = NULL;
+                       goto out_list_del;
+               }
+       }
+
+       arm_smmu_install_ste_for_dev(master);
+
        arm_smmu_enable_ats(master);
+       return 0;
+
+out_list_del:
+       spin_lock_irqsave(&smmu_domain->devices_lock, flags);
+       list_del(&master->domain_head);
+       spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
 
-out_unlock:
-       mutex_unlock(&smmu_domain->init_mutex);
        return ret;
 }
 
@@ -2707,6 +2715,8 @@ static void arm_smmu_release_device(struct device *dev)
        arm_smmu_detach_dev(master);
        arm_smmu_disable_pasid(master);
        arm_smmu_remove_master(master);
+       if (master->cd_table.cdtab)
+               arm_smmu_free_cd_tables(master);
        kfree(master);
 }
 

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index d2fc0a9793e5424ecae8e8b0292ac17655588d91..961205ba86d25d0cbaa2c116ea9091b59ccd0ddf 100644 (file)
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -695,6 +695,8 @@ struct arm_smmu_master {
        struct arm_smmu_domain          *domain;
        struct list_head                domain_head;
        struct arm_smmu_stream          *streams;
+       /* Locked by the iommu core using the group mutex */
+       struct arm_smmu_ctx_desc_cfg    cd_table;
        unsigned int                    num_streams;
        bool                            ats_enabled;
        bool                            stall_enabled;
@@ -721,11 +723,8 @@ struct arm_smmu_domain {
 
        enum arm_smmu_domain_stage      stage;
        union {
-               struct {
                struct arm_smmu_ctx_desc        cd;
-               struct arm_smmu_ctx_desc_cfg    cd_table;
-               };
-               struct arm_smmu_s2_cfg  s2_cfg;
+               struct arm_smmu_s2_cfg          s2_cfg;
        };
 
        struct iommu_domain             domain;