iommu/arm-smmu-v3: Hook up ATC invalidation to mm ops

The invalidate_range() notifier is called for any change to the address
space. Perform the required ATC invalidations.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
Link: https://lore.kernel.org/r/20201106155048.997886-5-jean-philippe@linaro.org
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
index 4d03481289ff31c52b91f9e0ada0c76164079bde..e13b092e60049b683159bb7e2e530b5558c69d56 100644 (file)
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
@@ -177,6 +177,16 @@ static void arm_smmu_free_shared_cd(struct arm_smmu_ctx_desc *cd)
        }
 }
 
+static void arm_smmu_mm_invalidate_range(struct mmu_notifier *mn,
+                                        struct mm_struct *mm,
+                                        unsigned long start, unsigned long end)
+{
+       struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn);
+
+       arm_smmu_atc_inv_domain(smmu_mn->domain, mm->pasid, start,
+                               end - start + 1);
+}
+
 static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
 {
        struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn);
@@ -195,6 +205,7 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
        arm_smmu_write_ctx_desc(smmu_domain, mm->pasid, &quiet_cd);
 
        arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_mn->cd->asid);
+       arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, 0, 0);
 
        smmu_mn->cleared = true;
        mutex_unlock(&sva_lock);
@@ -206,6 +217,7 @@ static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn)
 }
 
 static struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = {
+       .invalidate_range       = arm_smmu_mm_invalidate_range,
        .release                = arm_smmu_mm_release,
        .free_notifier          = arm_smmu_mmu_notifier_free,
 };
@@ -278,8 +290,10 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
         * If we went through clear(), we've already invalidated, and no
         * new TLB entry can have been formed.
         */
-       if (!smmu_mn->cleared)
+       if (!smmu_mn->cleared) {
                arm_smmu_tlb_inv_asid(smmu_domain->smmu, cd->asid);
+               arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, 0, 0);
+       }
 
        /* Frees smmu_mn */
        mmu_notifier_put(&smmu_mn->mn);

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 034ed126e5c8a5b1a777a9b7faab5018c8a63777..bfa953a07cf7eb57856618d86b252da3a6894b4a 100644 (file)
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -1524,6 +1524,20 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
        size_t inval_grain_shift = 12;
        unsigned long page_start, page_end;
 
+       /*
+        * ATS and PASID:
+        *
+        * If substream_valid is clear, the PCIe TLP is sent without a PASID
+        * prefix. In that case all ATC entries within the address range are
+        * invalidated, including those that were requested with a PASID! There
+        * is no way to invalidate only entries without PASID.
+        *
+        * When using STRTAB_STE_1_S1DSS_SSID0 (reserving CD 0 for non-PASID
+        * traffic), translation requests without PASID create ATC entries
+        * without PASID, which must be invalidated with substream_valid clear.
+        * This has the unpleasant side-effect of invalidating all PASID-tagged
+        * ATC entries within the address range.
+        */
        *cmd = (struct arm_smmu_cmdq_ent) {
                .opcode                 = CMDQ_OP_ATC_INV,
                .substream_valid        = !!ssid,
@@ -1582,8 +1596,8 @@ static int arm_smmu_atc_inv_master(struct arm_smmu_master *master)
        return arm_smmu_cmdq_issue_sync(master->smmu);
 }
 
-static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
-                                  int ssid, unsigned long iova, size_t size)
+int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
+                           unsigned long iova, size_t size)
 {
        int i;
        unsigned long flags;

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index e03ca01e0908260d6b2a52c3f46316d3321303a6..96c2e9565e002827503d801e381c2e99380126bc 100644 (file)
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -695,6 +695,8 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
                            struct arm_smmu_ctx_desc *cd);
 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
 bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd);
+int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
+                           unsigned long iova, size_t size);
 
 #ifdef CONFIG_ARM_SMMU_V3_SVA
 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);