From: Joerg Roedel Date: Mon, 13 May 2024 12:06:54 +0000 (+0200) Subject: Merge branches 'arm/renesas', 'arm/smmu', 'x86/amd', 'core' and 'x86/vt-d' into next X-Git-Url: http://git.maquefel.me/?a=commitdiff_plain;h=2bd5059c6cc04b02073d4d9f57137ab74e1d8e7a;p=linux.git Merge branches 'arm/renesas', 'arm/smmu', 'x86/amd', 'core' and 'x86/vt-d' into next --- 2bd5059c6cc04b02073d4d9f57137ab74e1d8e7a diff --cc drivers/iommu/amd/amd_iommu.h index f482aab420f78,f482aab420f78,f482aab420f78,dca1204681987,f482aab420f78,410c360e7e245..2fde1302a5843 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@@@@@@ -38,11 -38,11 -38,11 -44,28 -38,11 -38,11 +44,29 @@@@@@@ extern int amd_iommu_guest_ir extern enum io_pgtable_fmt amd_iommu_pgtable; extern int amd_iommu_gpt_level; --- --bool amd_iommu_v2_supported(void); --- -- --- --/* Device capabilities */ --- --int amd_iommu_pdev_enable_cap_pri(struct pci_dev *pdev); --- --void amd_iommu_pdev_disable_cap_pri(struct pci_dev *pdev); +++ ++/* Protection domain ops */ +++ ++struct protection_domain *protection_domain_alloc(unsigned int type); +++ ++void protection_domain_free(struct protection_domain *domain); +++ ++struct iommu_domain *amd_iommu_domain_alloc_sva(struct device *dev, +++ ++ struct mm_struct *mm); +++ ++void amd_iommu_domain_free(struct iommu_domain *dom); +++ ++int iommu_sva_set_dev_pasid(struct iommu_domain *domain, +++ ++ struct device *dev, ioasid_t pasid); - void amd_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid); ++++++void amd_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid, ++++++ struct iommu_domain *domain); +++ ++ +++ ++/* SVA/PASID */ +++ ++bool amd_iommu_pasid_supported(void); +++ ++ +++ ++/* IOPF */ +++ ++int amd_iommu_iopf_init(struct amd_iommu *iommu); +++ ++void amd_iommu_iopf_uninit(struct amd_iommu *iommu); +++ ++void amd_iommu_page_response(struct device *dev, struct iopf_fault *evt, +++ ++ struct iommu_page_response *resp); +++ ++int amd_iommu_iopf_add_device(struct amd_iommu *iommu, +++ ++ struct iommu_dev_data *dev_data); +++ ++void amd_iommu_iopf_remove_device(struct amd_iommu *iommu, +++ ++ struct iommu_dev_data *dev_data); /* GCR3 setup */ int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, diff --cc drivers/iommu/amd/iommu.c index fb727f5b0b82d,d35c1b8c8e65c,d35c1b8c8e65c,1ff80cc79dbc3,085abf098fa9a,e692217fcb280..1fdf37e042156 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@@@@@@ -2789,9 -2782,9 -2782,9 -2868,10 -2775,8 -2785,9 +2861,9 @@@@@@@ const struct iommu_ops amd_iommu_ops = .capable = amd_iommu_capable, .domain_alloc = amd_iommu_domain_alloc, .domain_alloc_user = amd_iommu_domain_alloc_user, +++ ++ .domain_alloc_sva = amd_iommu_domain_alloc_sva, .probe_device = amd_iommu_probe_device, .release_device = amd_iommu_release_device, ---- - .probe_finalize = amd_iommu_probe_finalize, .device_group = amd_iommu_device_group, .get_resv_regions = amd_iommu_get_resv_regions, .is_attach_deferred = amd_iommu_is_attach_deferred, diff --cc drivers/iommu/amd/pasid.c index 0000000000000,0000000000000,0000000000000,04beb2217d938,0000000000000,0000000000000..a68215f2b3e1d mode 000000,000000,000000,100644,000000,000000..100644 --- a/drivers/iommu/amd/pasid.c +++ b/drivers/iommu/amd/pasid.c @@@@@@@ -1,0 -1,0 -1,0 -1,202 -1,0 -1,0 +1,198 @@@@@@@ +++ ++// SPDX-License-Identifier: GPL-2.0-only +++ ++/* +++ ++ * Copyright (C) 2024 Advanced Micro Devices, Inc. +++ ++ */ +++ ++ +++ ++#define pr_fmt(fmt) "AMD-Vi: " fmt +++ ++#define dev_fmt(fmt) pr_fmt(fmt) +++ ++ +++ ++#include +++ ++#include +++ ++ +++ ++#include "amd_iommu.h" +++ ++ +++ ++static inline bool is_pasid_enabled(struct iommu_dev_data *dev_data) +++ ++{ +++ ++ if (dev_data->pasid_enabled && dev_data->max_pasids && +++ ++ dev_data->gcr3_info.gcr3_tbl != NULL) +++ ++ return true; +++ ++ +++ ++ return false; +++ ++} +++ ++ +++ ++static inline bool is_pasid_valid(struct iommu_dev_data *dev_data, +++ ++ ioasid_t pasid) +++ ++{ +++ ++ if (pasid > 0 && pasid < dev_data->max_pasids) +++ ++ return true; +++ ++ +++ ++ return false; +++ ++} +++ ++ +++ ++static void remove_dev_pasid(struct pdom_dev_data *pdom_dev_data) +++ ++{ +++ ++ /* Update GCR3 table and flush IOTLB */ +++ ++ amd_iommu_clear_gcr3(pdom_dev_data->dev_data, pdom_dev_data->pasid); +++ ++ +++ ++ list_del(&pdom_dev_data->list); +++ ++ kfree(pdom_dev_data); +++ ++} +++ ++ +++ ++/* Clear PASID from device GCR3 table and remove pdom_dev_data from list */ +++ ++static void remove_pdom_dev_pasid(struct protection_domain *pdom, +++ ++ struct device *dev, ioasid_t pasid) +++ ++{ +++ ++ struct pdom_dev_data *pdom_dev_data; +++ ++ struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); +++ ++ +++ ++ lockdep_assert_held(&pdom->lock); +++ ++ +++ ++ for_each_pdom_dev_data(pdom_dev_data, pdom) { +++ ++ if (pdom_dev_data->dev_data == dev_data && +++ ++ pdom_dev_data->pasid == pasid) { +++ ++ remove_dev_pasid(pdom_dev_data); +++ ++ break; +++ ++ } +++ ++ } +++ ++} +++ ++ +++ ++static void sva_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, +++ ++ struct mm_struct *mm, +++ ++ unsigned long start, unsigned long end) +++ ++{ +++ ++ struct pdom_dev_data *pdom_dev_data; +++ ++ struct protection_domain *sva_pdom; +++ ++ unsigned long flags; +++ ++ +++ ++ sva_pdom = container_of(mn, struct protection_domain, mn); +++ ++ +++ ++ spin_lock_irqsave(&sva_pdom->lock, flags); +++ ++ +++ ++ for_each_pdom_dev_data(pdom_dev_data, sva_pdom) { +++ ++ amd_iommu_dev_flush_pasid_pages(pdom_dev_data->dev_data, +++ ++ pdom_dev_data->pasid, +++ ++ start, end - start); +++ ++ } +++ ++ +++ ++ spin_unlock_irqrestore(&sva_pdom->lock, flags); +++ ++} +++ ++ +++ ++static void sva_mn_release(struct mmu_notifier *mn, struct mm_struct *mm) +++ ++{ +++ ++ struct pdom_dev_data *pdom_dev_data, *next; +++ ++ struct protection_domain *sva_pdom; +++ ++ unsigned long flags; +++ ++ +++ ++ sva_pdom = container_of(mn, struct protection_domain, mn); +++ ++ +++ ++ spin_lock_irqsave(&sva_pdom->lock, flags); +++ ++ +++ ++ /* Assume dev_data_list contains same PASID with different devices */ +++ ++ for_each_pdom_dev_data_safe(pdom_dev_data, next, sva_pdom) +++ ++ remove_dev_pasid(pdom_dev_data); +++ ++ +++ ++ spin_unlock_irqrestore(&sva_pdom->lock, flags); +++ ++} +++ ++ +++ ++static const struct mmu_notifier_ops sva_mn = { +++ ++ .arch_invalidate_secondary_tlbs = sva_arch_invalidate_secondary_tlbs, +++ ++ .release = sva_mn_release, +++ ++}; +++ ++ +++ ++int iommu_sva_set_dev_pasid(struct iommu_domain *domain, +++ ++ struct device *dev, ioasid_t pasid) +++ ++{ +++ ++ struct pdom_dev_data *pdom_dev_data; +++ ++ struct protection_domain *sva_pdom = to_pdomain(domain); +++ ++ struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); +++ ++ unsigned long flags; +++ ++ int ret = -EINVAL; +++ ++ +++ ++ /* PASID zero is used for requests from the I/O device without PASID */ +++ ++ if (!is_pasid_valid(dev_data, pasid)) +++ ++ return ret; +++ ++ +++ ++ /* Make sure PASID is enabled */ +++ ++ if (!is_pasid_enabled(dev_data)) +++ ++ return ret; +++ ++ +++ ++ /* Add PASID to protection domain pasid list */ +++ ++ pdom_dev_data = kzalloc(sizeof(*pdom_dev_data), GFP_KERNEL); +++ ++ if (pdom_dev_data == NULL) +++ ++ return ret; +++ ++ +++ ++ pdom_dev_data->pasid = pasid; +++ ++ pdom_dev_data->dev_data = dev_data; +++ ++ +++ ++ spin_lock_irqsave(&sva_pdom->lock, flags); +++ ++ +++ ++ /* Setup GCR3 table */ +++ ++ ret = amd_iommu_set_gcr3(dev_data, pasid, +++ ++ iommu_virt_to_phys(domain->mm->pgd)); +++ ++ if (ret) { +++ ++ kfree(pdom_dev_data); +++ ++ goto out_unlock; +++ ++ } +++ ++ +++ ++ list_add(&pdom_dev_data->list, &sva_pdom->dev_data_list); +++ ++ +++ ++out_unlock: +++ ++ spin_unlock_irqrestore(&sva_pdom->lock, flags); +++ ++ return ret; +++ ++} +++ ++ - void amd_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid) ++++++void amd_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid, ++++++ struct iommu_domain *domain) +++ ++{ +++ ++ struct protection_domain *sva_pdom; - struct iommu_domain *domain; +++ ++ unsigned long flags; +++ ++ +++ ++ if (!is_pasid_valid(dev_iommu_priv_get(dev), pasid)) +++ ++ return; +++ ++ - /* Get protection domain */ - domain = iommu_get_domain_for_dev_pasid(dev, pasid, IOMMU_DOMAIN_SVA); - if (!domain) - return; +++ ++ sva_pdom = to_pdomain(domain); +++ ++ +++ ++ spin_lock_irqsave(&sva_pdom->lock, flags); +++ ++ +++ ++ /* Remove PASID from dev_data_list */ +++ ++ remove_pdom_dev_pasid(sva_pdom, dev, pasid); +++ ++ +++ ++ spin_unlock_irqrestore(&sva_pdom->lock, flags); +++ ++} +++ ++ +++ ++static void iommu_sva_domain_free(struct iommu_domain *domain) +++ ++{ +++ ++ struct protection_domain *sva_pdom = to_pdomain(domain); +++ ++ +++ ++ if (sva_pdom->mn.ops) +++ ++ mmu_notifier_unregister(&sva_pdom->mn, domain->mm); +++ ++ +++ ++ amd_iommu_domain_free(domain); +++ ++} +++ ++ +++ ++static const struct iommu_domain_ops amd_sva_domain_ops = { +++ ++ .set_dev_pasid = iommu_sva_set_dev_pasid, +++ ++ .free = iommu_sva_domain_free +++ ++}; +++ ++ +++ ++struct iommu_domain *amd_iommu_domain_alloc_sva(struct device *dev, +++ ++ struct mm_struct *mm) +++ ++{ +++ ++ struct protection_domain *pdom; +++ ++ int ret; +++ ++ +++ ++ pdom = protection_domain_alloc(IOMMU_DOMAIN_SVA); +++ ++ if (!pdom) +++ ++ return ERR_PTR(-ENOMEM); +++ ++ +++ ++ pdom->domain.ops = &amd_sva_domain_ops; +++ ++ pdom->mn.ops = &sva_mn; +++ ++ +++ ++ ret = mmu_notifier_register(&pdom->mn, mm); +++ ++ if (ret) { +++ ++ protection_domain_free(pdom); +++ ++ return ERR_PTR(ret); +++ ++ } +++ ++ +++ ++ return &pdom->domain; +++ ++} diff --cc drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 41f93c3ab160d,41f93c3ab160d,aa62f0ecd0533,41f93c3ab160d,2e1988c861f15,41f93c3ab160d..ab415e107054c --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@@@@@@ -1214,113 -1214,113 -1192,162 -1214,113 -1214,113 -1214,113 +1192,162 @@@@@@@ static void arm_smmu_write_cd_l1_desc(_ WRITE_ONCE(*dst, cpu_to_le64(val)); } -- ---static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, u32 ssid) ++ +++struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, ++ +++ u32 ssid) { -- --- __le64 *l1ptr; -- --- unsigned int idx; struct arm_smmu_l1_ctx_desc *l1_desc; -- --- struct arm_smmu_device *smmu = master->smmu; struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; ++ +++ if (!cd_table->cdtab) ++ +++ return NULL; ++ +++ if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_LINEAR) -- --- return cd_table->cdtab + ssid * CTXDESC_CD_DWORDS; ++ +++ return (struct arm_smmu_cd *)(cd_table->cdtab + ++ +++ ssid * CTXDESC_CD_DWORDS); -- --- idx = ssid >> CTXDESC_SPLIT; -- --- l1_desc = &cd_table->l1_desc[idx]; -- --- if (!l1_desc->l2ptr) { -- --- if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc)) ++ +++ l1_desc = &cd_table->l1_desc[ssid / CTXDESC_L2_ENTRIES]; ++ +++ if (!l1_desc->l2ptr) ++ +++ return NULL; ++ +++ return &l1_desc->l2ptr[ssid % CTXDESC_L2_ENTRIES]; ++ +++} ++ +++ ++ +++struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master, ++ +++ u32 ssid) ++ +++{ ++ +++ struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; ++ +++ struct arm_smmu_device *smmu = master->smmu; ++ +++ ++ +++ might_sleep(); ++ +++ iommu_group_mutex_assert(master->dev); ++ +++ ++ +++ if (!cd_table->cdtab) { ++ +++ if (arm_smmu_alloc_cd_tables(master)) return NULL; ++ +++ } ++ +++ ++ +++ if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_64K_L2) { ++ +++ unsigned int idx = ssid / CTXDESC_L2_ENTRIES; ++ +++ struct arm_smmu_l1_ctx_desc *l1_desc; ++ +++ ++ +++ l1_desc = &cd_table->l1_desc[idx]; ++ +++ if (!l1_desc->l2ptr) { ++ +++ __le64 *l1ptr; + -- - - l1ptr = cd_table->cdtab + idx * CTXDESC_L1_DESC_DWORDS; -- - - arm_smmu_write_cd_l1_desc(l1ptr, l1_desc); -- - - /* An invalid L1CD can be cached */ -- - - arm_smmu_sync_cd(master, ssid, false); ++ +++ if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc)) ++ +++ return NULL; ++ + + - l1ptr = cd_table->cdtab + idx * CTXDESC_L1_DESC_DWORDS; - arm_smmu_write_cd_l1_desc(l1ptr, l1_desc); - /* An invalid L1CD can be cached */ - arm_smmu_sync_cd(master, ssid, false); ++ +++ l1ptr = cd_table->cdtab + idx * CTXDESC_L1_DESC_DWORDS; ++ +++ arm_smmu_write_cd_l1_desc(l1ptr, l1_desc); ++ +++ /* An invalid L1CD can be cached */ ++ +++ arm_smmu_sync_cd(master, ssid, false); ++ +++ } } -- --- idx = ssid & (CTXDESC_L2_ENTRIES - 1); -- --- return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS; ++ +++ return arm_smmu_get_cd_ptr(master, ssid); } -- ---int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, -- --- struct arm_smmu_ctx_desc *cd) ++ +++struct arm_smmu_cd_writer { ++ +++ struct arm_smmu_entry_writer writer; ++ +++ unsigned int ssid; ++ +++}; ++ +++ ++ +++VISIBLE_IF_KUNIT ++ +++void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits) { ++ +++ used_bits[0] = cpu_to_le64(CTXDESC_CD_0_V); ++ +++ if (!(ent[0] & cpu_to_le64(CTXDESC_CD_0_V))) ++ +++ return; ++ +++ memset(used_bits, 0xFF, sizeof(struct arm_smmu_cd)); ++ +++ /* -- --- * This function handles the following cases: -- --- * -- --- * (1) Install primary CD, for normal DMA traffic (SSID = IOMMU_NO_PASID = 0). -- --- * (2) Install a secondary CD, for SID+SSID traffic. -- --- * (3) Update ASID of a CD. Atomically write the first 64 bits of the -- --- * CD, then invalidate the old entry and mappings. -- --- * (4) Quiesce the context without clearing the valid bit. Disable -- --- * translation, and ignore any translation fault. -- --- * (5) Remove a secondary CD. ++ +++ * If EPD0 is set by the make function it means ++ +++ * T0SZ/TG0/IR0/OR0/SH0/TTB0 are IGNORED */ -- --- u64 val; -- --- bool cd_live; -- --- __le64 *cdptr; -- --- struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; -- --- struct arm_smmu_device *smmu = master->smmu; ++ +++ if (ent[0] & cpu_to_le64(CTXDESC_CD_0_TCR_EPD0)) { ++ +++ used_bits[0] &= ~cpu_to_le64( ++ +++ CTXDESC_CD_0_TCR_T0SZ | CTXDESC_CD_0_TCR_TG0 | ++ +++ CTXDESC_CD_0_TCR_IRGN0 | CTXDESC_CD_0_TCR_ORGN0 | ++ +++ CTXDESC_CD_0_TCR_SH0); ++ +++ used_bits[1] &= ~cpu_to_le64(CTXDESC_CD_1_TTB0_MASK); ++ +++ } ++ +++} ++ +++EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_cd_used); -- --- if (WARN_ON(ssid >= (1 << cd_table->s1cdmax))) -- --- return -E2BIG; ++ +++static void arm_smmu_cd_writer_sync_entry(struct arm_smmu_entry_writer *writer) ++ +++{ ++ +++ struct arm_smmu_cd_writer *cd_writer = ++ +++ container_of(writer, struct arm_smmu_cd_writer, writer); -- --- cdptr = arm_smmu_get_cd_ptr(master, ssid); -- --- if (!cdptr) -- --- return -ENOMEM; ++ +++ arm_smmu_sync_cd(writer->master, cd_writer->ssid, true); ++ +++} -- --- val = le64_to_cpu(cdptr[0]); -- --- cd_live = !!(val & CTXDESC_CD_0_V); -- --- -- --- if (!cd) { /* (5) */ -- --- val = 0; -- --- } else if (cd == &quiet_cd) { /* (4) */ -- --- if (!(smmu->features & ARM_SMMU_FEAT_STALL_FORCE)) -- --- val &= ~(CTXDESC_CD_0_S | CTXDESC_CD_0_R); -- --- val |= CTXDESC_CD_0_TCR_EPD0; -- --- } else if (cd_live) { /* (3) */ -- --- val &= ~CTXDESC_CD_0_ASID; -- --- val |= FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid); -- --- /* -- --- * Until CD+TLB invalidation, both ASIDs may be used for tagging -- --- * this substream's traffic -- --- */ -- --- } else { /* (1) and (2) */ -- --- cdptr[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK); -- --- cdptr[2] = 0; -- --- cdptr[3] = cpu_to_le64(cd->mair); ++ +++static const struct arm_smmu_entry_writer_ops arm_smmu_cd_writer_ops = { ++ +++ .sync = arm_smmu_cd_writer_sync_entry, ++ +++ .get_used = arm_smmu_get_cd_used, ++ +++}; -- --- /* -- --- * STE may be live, and the SMMU might read dwords of this CD in any -- --- * order. Ensure that it observes valid values before reading -- --- * V=1. -- --- */ -- --- arm_smmu_sync_cd(master, ssid, true); ++ +++void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid, ++ +++ struct arm_smmu_cd *cdptr, ++ +++ const struct arm_smmu_cd *target) ++ +++{ ++ +++ struct arm_smmu_cd_writer cd_writer = { ++ +++ .writer = { ++ +++ .ops = &arm_smmu_cd_writer_ops, ++ +++ .master = master, ++ +++ }, ++ +++ .ssid = ssid, ++ +++ }; ++ + + - val = cd->tcr | ++ +++ arm_smmu_write_entry(&cd_writer.writer, cdptr->data, target->data); ++ +++} + -- - - val = cd->tcr | ++ +++void arm_smmu_make_s1_cd(struct arm_smmu_cd *target, ++ +++ struct arm_smmu_master *master, ++ +++ struct arm_smmu_domain *smmu_domain) ++ +++{ ++ +++ struct arm_smmu_ctx_desc *cd = &smmu_domain->cd; ++ +++ const struct io_pgtable_cfg *pgtbl_cfg = ++ +++ &io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops)->cfg; ++ +++ typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = ++ +++ &pgtbl_cfg->arm_lpae_s1_cfg.tcr; ++ +++ ++ +++ memset(target, 0, sizeof(*target)); ++ +++ ++ +++ target->data[0] = cpu_to_le64( ++ +++ FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) | ++ +++ FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) | ++ +++ FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) | ++ +++ FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) | ++ +++ FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) | #ifdef __BIG_ENDIAN -- --- CTXDESC_CD_0_ENDI | ++ +++ CTXDESC_CD_0_ENDI | #endif -- --- CTXDESC_CD_0_R | CTXDESC_CD_0_A | -- --- (cd->mm ? 0 : CTXDESC_CD_0_ASET) | -- --- CTXDESC_CD_0_AA64 | -- --- FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) | -- --- CTXDESC_CD_0_V; -- --- -- --- if (cd_table->stall_enabled) -- --- val |= CTXDESC_CD_0_S; -- --- } -- --- -- --- /* -- --- * The SMMU accesses 64-bit values atomically. See IHI0070Ca 3.21.3 -- --- * "Configuration structures and configuration invalidation completion" -- --- * -- --- * The size of single-copy atomic reads made by the SMMU is -- --- * IMPLEMENTATION DEFINED but must be at least 64 bits. Any single -- --- * field within an aligned 64-bit span of a structure can be altered -- --- * without first making the structure invalid. -- --- */ -- --- WRITE_ONCE(cdptr[0], cpu_to_le64(val)); -- --- arm_smmu_sync_cd(master, ssid, true); -- --- return 0; ++ +++ CTXDESC_CD_0_TCR_EPD1 | ++ +++ CTXDESC_CD_0_V | ++ +++ FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) | ++ +++ CTXDESC_CD_0_AA64 | ++ +++ (master->stall_enabled ? CTXDESC_CD_0_S : 0) | ++ +++ CTXDESC_CD_0_R | ++ +++ CTXDESC_CD_0_A | ++ +++ CTXDESC_CD_0_ASET | ++ +++ FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) ++ +++ ); ++ +++ target->data[1] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.ttbr & ++ +++ CTXDESC_CD_1_TTB0_MASK); ++ +++ target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.mair); ++ +++} ++ +++EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s1_cd); ++ +++ ++ +++void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid) ++ +++{ ++ +++ struct arm_smmu_cd target = {}; ++ +++ struct arm_smmu_cd *cdptr; ++ +++ ++ +++ if (!master->cd_table.cdtab) ++ +++ return; ++ +++ cdptr = arm_smmu_get_cd_ptr(master, ssid); ++ +++ if (WARN_ON(!cdptr)) ++ +++ return; ++ +++ arm_smmu_write_cd_entry(master, ssid, cdptr, &target); } static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master) diff --cc drivers/iommu/intel/iommu.c index a7ecd90303dc4,50eb9aed47cc5,50eb9aed47cc5,daaa7a22595e3,64adfdadeb496,e8e62ff1a65eb..2e9811bf2a4e5 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@@@@@@ -865,9 -865,9 -865,9 -850,9 -865,9 -859,9 +844,9 @@@@@@@ static struct dma_pte *pfn_to_dma_pte(s break; if (!dma_pte_present(pte)) { ----- uint64_t pteval; +++++ uint64_t pteval, tmp; --- -- tmp_page = alloc_pgtable_page(domain->nid, gfp); +++ ++ tmp_page = iommu_alloc_page_node(domain->nid, gfp); if (!tmp_page) return NULL; @@@@@@@ -877,9 -877,9 -877,9 -862,9 -877,9 -871,10 +856,10 @@@@@@@ if (domain->use_first_level) pteval |= DMA_FL_PTE_XD | DMA_FL_PTE_US | DMA_FL_PTE_ACCESS; ----- if (cmpxchg64(&pte->val, 0ULL, pteval)) +++++ tmp = 0ULL; +++++ if (!try_cmpxchg64(&pte->val, &tmp, pteval)) /* Someone else set it while we were thinking; use theirs. */ --- -- free_pgtable_page(tmp_page); +++ ++ iommu_free_page(tmp_page); else domain_flush_cache(domain, pte, sizeof(*pte)); } @@@@@@@ -3417,16 -3417,16 -3417,16 -3402,16 -3417,16 -3219,7 +3204,7 @@@@@@@ static int intel_iommu_memory_notifier( LIST_HEAD(freelist); domain_unmap(si_domain, start_vpfn, last_vpfn, &freelist); ----- ----- rcu_read_lock(); ----- for_each_active_iommu(iommu, drhd) ----- iommu_flush_iotlb_psi(iommu, si_domain, ----- start_vpfn, mhp->nr_pages, ----- list_empty(&freelist), 0); ----- rcu_read_unlock(); --- -- put_pages_list(&freelist); +++ ++ iommu_put_pages_list(&freelist); } break; } @@@@@@@ -4122,26 -4122,26 -4122,26 -4107,26 -4122,26 -3914,9 +3899,9 @@@@@@@ static size_t intel_iommu_unmap_pages(s static void intel_iommu_tlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { ----- struct dmar_domain *dmar_domain = to_dmar_domain(domain); ----- unsigned long iova_pfn = IOVA_PFN(gather->start); ----- size_t size = gather->end - gather->start; ----- struct iommu_domain_info *info; ----- unsigned long start_pfn; ----- unsigned long nrpages; ----- unsigned long i; ----- ----- nrpages = aligned_nrpages(gather->start, size); ----- start_pfn = mm_to_dma_pfn_start(iova_pfn); ----- ----- xa_for_each(&dmar_domain->iommu_array, i, info) ----- iommu_flush_iotlb_psi(info->iommu, dmar_domain, ----- start_pfn, nrpages, ----- list_empty(&gather->freelist), 0); ----- ----- if (dmar_domain->nested_parent) ----- parent_domain_flush(dmar_domain, start_pfn, nrpages, ----- list_empty(&gather->freelist)); --- - put_pages_list(&gather->freelist); +++++ cache_tag_flush_range(to_dmar_domain(domain), gather->start, +++++ gather->end, list_empty(&gather->freelist)); - put_pages_list(&gather->freelist); +++ ++ iommu_put_pages_list(&gather->freelist); } static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, @@@@@@@ -4588,32 -4585,32 -4585,32 -4573,32 -4579,27 -4359,20 +4338,15 @@@@@@@ static int intel_iommu_iotlb_sync_map(s return 0; } ---- -static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid) ++++ +static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid, ++++ + struct iommu_domain *domain) { struct device_domain_info *info = dev_iommu_priv_get(dev); ++++ + struct dmar_domain *dmar_domain = to_dmar_domain(domain); struct dev_pasid_info *curr, *dev_pasid = NULL; struct intel_iommu *iommu = info->iommu; ---- - struct dmar_domain *dmar_domain; ---- - struct iommu_domain *domain; unsigned long flags; ---- - domain = iommu_get_domain_for_dev_pasid(dev, pasid, 0); ---- - if (WARN_ON_ONCE(!domain)) ---- goto out_tear_down; ---- ----- /* ----- * The SVA implementation needs to handle its own stuffs like the mm ----- * notification. Before consolidating that code into iommu core, let ----- * the intel sva code handle it. ----- */ ----- if (domain->type == IOMMU_DOMAIN_SVA) { ----- intel_svm_remove_dev_pasid(dev, pasid); ------ goto out_tear_down; ----- } ----- ---- - dmar_domain = to_dmar_domain(domain); - spin_lock_irqsave(&dmar_domain->lock, flags); list_for_each_entry(curr, &dmar_domain->dev_pasids, link_domain) { if (curr->dev == dev && curr->pasid == pasid) { @@@@@@@ -4628,7 -4625,7 -4625,7 -4613,7 -4614,7 -4388,7 +4362,6 @@@@@@@ domain_detach_iommu(dmar_domain, iommu); intel_iommu_debugfs_remove_dev_pasid(dev_pasid); kfree(dev_pasid); ------out_tear_down: intel_pasid_tear_down_entry(iommu, dev, pasid, false); intel_drain_pasid_prq(dev, pasid); } @@@@@@@ -4841,8 -4838,8 -4838,8 -4826,8 -4827,7 -4607,9 +4580,8 @@@@@@@ const struct iommu_ops intel_iommu_ops .hw_info = intel_iommu_hw_info, .domain_alloc = intel_iommu_domain_alloc, .domain_alloc_user = intel_iommu_domain_alloc_user, +++++ .domain_alloc_sva = intel_svm_domain_alloc, .probe_device = intel_iommu_probe_device, ---- - .probe_finalize = intel_iommu_probe_finalize, .release_device = intel_iommu_release_device, .get_resv_regions = intel_iommu_get_resv_regions, .device_group = intel_iommu_device_group, diff --cc drivers/iommu/intel/svm.c index ee3b469e2da15,c1bed89b10261,c1bed89b10261,71c1b2a0ca169,c1bed89b10261,268a0082d37f7..0e3a9b38bef21 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@@@@@@ -29,45 -29,45 -29,45 -30,44 -29,45 -29,11 +30,10 @@@@@@@ static irqreturn_t prq_event_thread(in int intel_svm_enable_prq(struct intel_iommu *iommu) { struct iopf_queue *iopfq; --- -- struct page *pages; int irq, ret; - - pages = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, PRQ_ORDER); -- - pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, PRQ_ORDER); --- -- if (!pages) { +++ ++ iommu->prq = iommu_alloc_pages_node(iommu->node, GFP_KERNEL, PRQ_ORDER); +++ ++ if (!iommu->prq) { pr_warn("IOMMU: %s: Failed to allocate page request queue\n", iommu->name); return -ENOMEM;