From: Joerg Roedel Date: Tue, 2 Jun 2020 08:32:04 +0000 (+0200) Subject: Merge branches 'arm/msm', 'arm/allwinner', 'arm/smmu', 'x86/vt-d', 'hyper-v', 'core... X-Git-Url: http://git.maquefel.me/?a=commitdiff_plain;h=cc69fc4861705c27c0506d39189015d7404129ed;p=linux.git Merge branches 'arm/msm', 'arm/allwinner', 'arm/smmu', 'x86/vt-d', 'hyper-v', 'core' and 'x86/amd' into next --- cc69fc4861705c27c0506d39189015d7404129ed diff --cc drivers/iommu/amd_iommu.c index 2883ac389abbe,20cce366e9518,1880811cec333,c303674136832,1880811cec333,20cce366e9518,254f028cd18e5,7c1884cab91f3..311ef7105c6d0 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@@@@@@@@ -99,10 -99,8 -99,8 -99,8 -99,8 -99,8 -99,8 -99,9 +99,9 @@@@@@@@@ struct iommu_cmd struct kmem_cache *amd_iommu_irq_cache; static void update_domain(struct protection_domain *domain); ------- static int protection_domain_init(struct protection_domain *domain); static void detach_device(struct device *dev); ++++++ static void update_and_flush_device_table(struct protection_domain *domain, ++++++ struct domain_pgtable *pgtable); /**************************************************************************** * @@@@@@@@@ -369,21 -346,21 -346,9 -346,9 -346,9 -346,21 -345,9 -362,9 +361,9 @@@@@@@@@ static bool check_device(struct device static int iommu_init_device(struct device *dev) { struct iommu_dev_data *dev_data; -- - struct amd_iommu *iommu; int devid; ------- if (dev->archdata.iommu) +++++++ if (dev_iommu_priv_get(dev)) return 0; devid = get_device_id(dev); @@@@@@@@@ -412,9 -389,9 -375,7 -375,7 -375,7 -389,9 -374,7 -391,7 +390,7 @@@@@@@@@ dev_data->iommu_v2 = iommu->is_iommu_v2; } ------- dev->archdata.iommu = dev_data; -- - -- - iommu_device_link(&iommu->iommu, dev); +++++++ dev_iommu_priv_set(dev, dev_data); return 0; } @@@@@@@@@ -433,32 -410,32 -394,22 -394,25 -394,22 -410,32 -393,22 -410,19 +409,19 @@@@@@@@@ static void iommu_ignore_device(struct setup_aliases(dev); } -- - static void iommu_uninit_device(struct device *dev) ++ + static void amd_iommu_uninit_device(struct device *dev) { struct iommu_dev_data *dev_data; -- - - struct amd_iommu *iommu; ------- int devid; ------ ------ devid = get_device_id(dev); ------ if (devid < 0) ------ return; - - - - - - iommu = amd_iommu_rlookup_table[devid]; - devid = get_device_id(dev); - if (devid < 0) - return; - - iommu = amd_iommu_rlookup_table[devid]; - ------- dev_data = search_dev_data(devid); +++++++ dev_data = dev_iommu_priv_get(dev); if (!dev_data) return; if (dev_data->domain) detach_device(dev); -- - iommu_device_unlink(&iommu->iommu, dev); -- - -- - iommu_group_remove_device(dev); -- - -- - /* Remove dma-ops */ -- - dev->dma_ops = NULL; +++++++ dev_iommu_priv_set(dev, NULL); +++ + /* * We keep dev_data around for unplugged devices and reuse it when the * device is re-plugged - not doing so would introduce a ton of races. @@@@@@@@@ -1418,20 -1395,15 -1369,15 -1372,15 -1369,15 -1395,15 -1368,15 -1382,19 +1381,19 @@@@@@@@@ static struct page *free_sub_pt(unsigne return freelist; } ------- static void free_pagetable(struct protection_domain *domain) +++++++ static void free_pagetable(struct domain_pgtable *pgtable) { - struct domain_pgtable pgtable; ------ unsigned long root = (unsigned long)domain->pt_root; struct page *freelist = NULL; ++++++ unsigned long root; + + - amd_iommu_domain_get_pgtable(domain, &pgtable); - atomic64_set(&domain->pt_root, 0); --- - BUG_ON(domain->mode < PAGE_MODE_NONE || --- - domain->mode > PAGE_MODE_6_LEVEL); +++++++ if (pgtable->mode == PAGE_MODE_NONE) +++++++ return; + - BUG_ON(pgtable.mode < PAGE_MODE_NONE || - pgtable.mode > PAGE_MODE_6_LEVEL); - - BUG_ON(domain->mode < PAGE_MODE_NONE || - - domain->mode > PAGE_MODE_6_LEVEL); - - - freelist = free_sub_pt(root, domain->mode, freelist); +++++++ BUG_ON(pgtable->mode < PAGE_MODE_NONE || +++++++ pgtable->mode > PAGE_MODE_6_LEVEL); + + + - root = (unsigned long)pgtable.root; - freelist = free_sub_pt(root, pgtable.mode, freelist); - - - freelist = free_sub_pt(root, domain->mode, freelist); +++++++ root = (unsigned long)pgtable->root; +++++++ freelist = free_sub_pt(root, pgtable->mode, freelist); free_page_list(freelist); } @@@@@@@@@ -1500,23 -1456,11 -1430,11 -1433,11 -1430,11 -1456,11 -1429,11 -1463,23 +1462,23 @@@@@@@@@ static u64 *alloc_pte(struct protection BUG_ON(!is_power_of_2(page_size)); ------ while (address > PM_LEVEL_SIZE(domain->mode)) ------ *updated = increase_address_space(domain, address, gfp) || *updated; ++++++ amd_iommu_domain_get_pgtable(domain, &pgtable); + + + - - - level = domain->mode - 1; - - - pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)]; ++++++ while (address > PM_LEVEL_SIZE(pgtable.mode)) { ++++++ /* ++++++ * Return an error if there is no memory to update the ++++++ * page-table. ++++++ */ ++++++ if (!increase_address_space(domain, address, gfp)) ++++++ return NULL; + + + - - - level = domain->mode - 1; - - - pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)]; ++++++ /* Read new values to check if update was successful */ ++++++ amd_iommu_domain_get_pgtable(domain, &pgtable); ++++++ } ++++++ ++++++ ++++++ level = pgtable.mode - 1; ++++++ pte = &pgtable.root[PM_LEVEL_INDEX(level, address)]; address = PAGE_SIZE_ALIGN(address, page_size); end_lvl = PAGE_SIZE_LEVEL(page_size); @@@@@@@@@ -1844,72 -1779,68 -1753,68 -1756,68 -1753,68 -1779,68 -1752,68 -1807,8 +1806,8 @@@@@@@@@ static void free_gcr3_table(struct prot free_page((unsigned long)domain->gcr3_tbl); } ------- /* ------- * Free a domain, only used if something went wrong in the ------- * allocation path and we need to free an already allocated page table ------- */ ------- static void dma_ops_domain_free(struct protection_domain *domain) ------- { ------- if (!domain) ------- return; ------- ------- iommu_put_dma_cookie(&domain->domain); ------- ------- free_pagetable(domain); ------- ------- if (domain->id) ------- domain_id_free(domain->id); ------- ------- kfree(domain); ------- } ------- ------- /* ------- * Allocates a new protection domain usable for the dma_ops functions. ------- * It also initializes the page table and the address allocator data ------- * structures required for the dma_ops interface ------- */ ------- static struct protection_domain *dma_ops_domain_alloc(void) ------- { ------- struct protection_domain *domain; - u64 *pt_root, root; ------- ------- domain = kzalloc(sizeof(struct protection_domain), GFP_KERNEL); ------- if (!domain) ------- return NULL; ------- ------- if (protection_domain_init(domain)) - goto free_domain; - - pt_root = (void *)get_zeroed_page(GFP_KERNEL); - if (!pt_root) ------- goto free_domain; ------- - root = amd_iommu_domain_encode_pgtable(pt_root, PAGE_MODE_3_LEVEL); - atomic64_set(&domain->pt_root, root); ------ domain->mode = PAGE_MODE_3_LEVEL; ------ domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL); ------- domain->flags = PD_DMA_OPS_MASK; ------ if (!domain->pt_root) ------ goto free_domain; ------- ------- if (iommu_get_dma_cookie(&domain->domain) == -ENOMEM) ------- goto free_domain; ------- ------- return domain; ------- ------- free_domain: ------- dma_ops_domain_free(domain); ------- ------- return NULL; ------- } ------- ------- /* ------- * little helper function to check whether a given protection domain is a ------- * dma_ops domain ------- */ ------- static bool dma_ops_domain(struct protection_domain *domain) ------- { ------- return domain->flags & PD_DMA_OPS_MASK; ------- } ------- static void set_dte_entry(u16 devid, struct protection_domain *domain, ++++++ struct domain_pgtable *pgtable, bool ats, bool ppr) { u64 pte_root = 0; @@@@@@@@@ -2238,6 -2166,6 -2141,9 -2144,9 -2141,9 -2166,6 -2140,9 -2138,9 +2137,9 @@@@@@@@@ static struct iommu_device *amd_iommu_p iommu = amd_iommu_rlookup_table[devid]; --- - if (get_dev_data(dev)) +++++++ if (dev_iommu_priv_get(dev)) ++ + return &iommu->iommu; ++ + ret = iommu_init_device(dev); if (ret) { if (ret != -ENOTSUPP) @@@@@@@@@ -2258,32 -2186,32 -2165,25 -2168,25 -2165,25 -2186,32 -2164,25 -2162,21 +2161,21 @@@@@@@@@ static void amd_iommu_probe_finalize(st /* Domains are initialized for this device - have a look what we ended up with */ domain = iommu_get_domain_for_dev(dev); -- - if (domain->type == IOMMU_DOMAIN_IDENTITY) -- - dev_data->passthrough = true; -- - else if (domain->type == IOMMU_DOMAIN_DMA) ++ + if (domain->type == IOMMU_DOMAIN_DMA) iommu_setup_dma_ops(dev, IOVA_START_PFN << PAGE_SHIFT, 0); -- - -- - out: -- - iommu_completion_wait(iommu); -- - -- - return 0; } -- - static void amd_iommu_remove_device(struct device *dev) ++ + static void amd_iommu_release_device(struct device *dev) { +++++++ int devid = get_device_id(dev); struct amd_iommu *iommu; ------- int devid; if (!check_device(dev)) return; ------- devid = get_device_id(dev); ------- if (devid < 0) ------- return; ------- iommu = amd_iommu_rlookup_table[devid]; -- - iommu_uninit_device(dev); ++ + amd_iommu_uninit_device(dev); iommu_completion_wait(iommu); } @@@@@@@@@ -2459,48 -2374,45 -2346,45 -2349,45 -2346,45 -2374,45 -2345,45 -2371,30 +2370,30 @@@@@@@@@ out_err static struct iommu_domain *amd_iommu_domain_alloc(unsigned type) { ------- struct protection_domain *pdomain; - u64 *pt_root, root; +++++++ struct protection_domain *domain; +++++++ int mode = DEFAULT_PGTABLE_LEVEL; ------- switch (type) { ------- case IOMMU_DOMAIN_UNMANAGED: ------- pdomain = protection_domain_alloc(); ------- if (!pdomain) ------- return NULL; +++++++ if (type == IOMMU_DOMAIN_IDENTITY) +++++++ mode = PAGE_MODE_NONE; - pt_root = (void *)get_zeroed_page(GFP_KERNEL); - if (!pt_root) { ------ pdomain->mode = PAGE_MODE_3_LEVEL; ------ pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL); ------ if (!pdomain->pt_root) { ------- protection_domain_free(pdomain); ------- return NULL; ------- } +++++++ domain = protection_domain_alloc(mode); +++++++ if (!domain) +++++++ return NULL; - root = amd_iommu_domain_encode_pgtable(pt_root, PAGE_MODE_3_LEVEL); - atomic64_set(&pdomain->pt_root, root); ------ pdomain->domain.geometry.aperture_start = 0; ------ pdomain->domain.geometry.aperture_end = ~0ULL; ------ pdomain->domain.geometry.force_aperture = true; +++++++ domain->domain.geometry.aperture_start = 0; +++++++ domain->domain.geometry.aperture_end = ~0ULL; +++++++ domain->domain.geometry.force_aperture = true; - pdomain->domain.geometry.aperture_start = 0; - pdomain->domain.geometry.aperture_end = ~0ULL; - pdomain->domain.geometry.force_aperture = true; ------ break; ------ case IOMMU_DOMAIN_DMA: ------ pdomain = dma_ops_domain_alloc(); ------ if (!pdomain) { ------ pr_err("Failed to allocate\n"); ------ return NULL; ------ } ------ break; ------ case IOMMU_DOMAIN_IDENTITY: ------ pdomain = protection_domain_alloc(); ------ if (!pdomain) ------ return NULL; +++++++ if (type == IOMMU_DOMAIN_DMA && +++++++ iommu_get_dma_cookie(&domain->domain) == -ENOMEM) +++++++ goto free_domain; - break; - case IOMMU_DOMAIN_DMA: - pdomain = dma_ops_domain_alloc(); - if (!pdomain) { - pr_err("Failed to allocate\n"); - return NULL; - } ------ pdomain->mode = PAGE_MODE_NONE; ------- break; - case IOMMU_DOMAIN_IDENTITY: - pdomain = protection_domain_alloc(); - if (!pdomain) - return NULL; ------ default: ------ return NULL; ------ } +++++++ return &domain->domain; - atomic64_set(&pdomain->pt_root, PAGE_MODE_NONE); - break; - default: - return NULL; - } ------ return &pdomain->domain; +++++++ free_domain: +++++++ protection_domain_free(domain); ++++++ - return &pdomain->domain; +++++++ return NULL; } static void amd_iommu_domain_free(struct iommu_domain *dom) @@@@@@@@@ -2518,23 -2429,21 -2401,21 -2404,21 -2401,21 -2429,21 -2400,21 -2411,13 +2410,13 @@@@@@@@@ if (!dom) return; ------- switch (dom->type) { ------- case IOMMU_DOMAIN_DMA: ------- /* Now release the domain */ ------- dma_ops_domain_free(domain); ------- break; ------- default: - amd_iommu_domain_get_pgtable(domain, &pgtable); - - if (pgtable.mode != PAGE_MODE_NONE) ------ if (domain->mode != PAGE_MODE_NONE) ------- free_pagetable(domain); +++++++ if (dom->type == IOMMU_DOMAIN_DMA) +++++++ iommu_put_dma_cookie(&domain->domain); ------- if (domain->flags & PD_IOMMUV2_MASK) ------- free_gcr3_table(domain); +++++++ if (domain->flags & PD_IOMMUV2_MASK) +++++++ free_gcr3_table(domain); ------- protection_domain_free(domain); ------- break; ------- } +++++++ protection_domain_free(domain); } static void amd_iommu_detach_device(struct iommu_domain *dom, @@@@@@@@@ -2758,6 -2661,6 -2633,20 -2636,20 -2633,20 -2661,6 -2632,20 -2643,20 +2642,20 @@@@@@@@@ static void amd_iommu_iotlb_sync(struc amd_iommu_flush_iotlb_all(domain); } ++ + static int amd_iommu_def_domain_type(struct device *dev) ++ + { ++ + struct iommu_dev_data *dev_data; ++ + --- - dev_data = get_dev_data(dev); +++++++ dev_data = dev_iommu_priv_get(dev); ++ + if (!dev_data) ++ + return 0; ++ + ++ + if (dev_data->iommu_v2) ++ + return IOMMU_DOMAIN_IDENTITY; ++ + ++ + return 0; ++ + } ++ + const struct iommu_ops amd_iommu_ops = { .capable = amd_iommu_capable, .domain_alloc = amd_iommu_domain_alloc, @@@@@@@@@ -2805,28 -2708,18 -2696,18 -2699,18 -2696,18 -2708,18 -2695,18 -2706,22 +2705,22 @@@@@@@@@ EXPORT_SYMBOL(amd_iommu_unregister_ppr_ void amd_iommu_domain_direct_map(struct iommu_domain *dom) { struct protection_domain *domain = to_pdomain(dom); ++++++ struct domain_pgtable pgtable; unsigned long flags; - u64 pt_root; spin_lock_irqsave(&domain->lock, flags); ++++++ /* First save pgtable configuration*/ ++++++ amd_iommu_domain_get_pgtable(domain, &pgtable); ++++++ /* Update data structure */ - pt_root = amd_iommu_domain_encode_pgtable(NULL, PAGE_MODE_NONE); - atomic64_set(&domain->pt_root, pt_root); ------ domain->mode = PAGE_MODE_NONE; +++++++ atomic64_set(&domain->pt_root, 0); /* Make changes visible to IOMMUs */ update_domain(domain); - /* Restore old pgtable in domain->ptroot to free page-table */ - pt_root = amd_iommu_domain_encode_pgtable(pgtable.root, pgtable.mode); - atomic64_set(&domain->pt_root, pt_root); - /* Page-table is not visible to IOMMU anymore, so free it */ ------- free_pagetable(domain); +++++++ free_pagetable(&pgtable); spin_unlock_irqrestore(&domain->lock, flags); } diff --cc drivers/iommu/arm-smmu-v3.c index 82508730feb7a,82508730feb7a,42e1ee7e5197b,8a908c50c306f,42e1ee7e5197b,82508730feb7a,cb086924419fc,42e1ee7e5197b..f578677a5c414 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@@@@@@@@ -3008,10 -3008,10 -2992,8 -3002,7 -2992,8 -3008,10 -2986,8 -2992,8 +2996,7 @@@@@@@@@ static void arm_smmu_release_device(str return; master = dev_iommu_priv_get(dev); --- ---- smmu = master->smmu; arm_smmu_detach_dev(master); -- - iommu_group_remove_device(dev); -- - iommu_device_unlink(&smmu->iommu, dev); arm_smmu_disable_pasid(master); kfree(master); iommu_fwspec_free(dev); diff --cc drivers/iommu/intel-iommu.c index 0182cff2c7ac7,ef0a5246700e5,34e08fa2ce3a4,b906727f5b85f,5767882aa80f3,ef0a5246700e5,5dea3042820be,34e08fa2ce3a4..648a785e078a2 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@@@@@@@@ -5799,60 -5799,60 -5793,15 -5793,15 -5662,10 -5799,60 -5790,15 -5793,15 +5659,10 @@@@@@@@@ static struct iommu_device *intel_iommu if (translation_pre_enabled(iommu)) dev->archdata.iommu = DEFER_DEVICE_DOMAIN_INFO; -- - group = iommu_group_get_for_dev(dev); -- - -- - if (IS_ERR(group)) { -- - ret = PTR_ERR(group); -- - goto unlink; -- - } -- - -- - iommu_group_put(group); -- - -- - domain = iommu_get_domain_for_dev(dev); -- - dmar_domain = to_dmar_domain(domain); -- - if (domain->type == IOMMU_DOMAIN_DMA) { -- - if (device_def_domain_type(dev) == IOMMU_DOMAIN_IDENTITY) { -- - ret = iommu_request_dm_for_dev(dev); -- - if (ret) { -- - dmar_remove_one_dev_info(dev); -- - dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN; -- - domain_add_dev_info(si_domain, dev); -- - dev_info(dev, -- - "Device uses a private identity domain.\n"); -- - } -- - } -- - } else { -- - if (device_def_domain_type(dev) == IOMMU_DOMAIN_DMA) { -- - ret = iommu_request_dma_domain_for_dev(dev); -- - if (ret) { -- - dmar_remove_one_dev_info(dev); -- - dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN; -- - if (!get_private_domain_for_dev(dev)) { -- - dev_warn(dev, -- - "Failed to get a private domain.\n"); -- - ret = -ENOMEM; -- - goto unlink; -- - } -- - -- - dev_info(dev, -- - "Device uses a private dma domain.\n"); -- - } -- - } -- - } -- - ---- --- if (device_needs_bounce(dev)) { ---- --- dev_info(dev, "Use Intel IOMMU bounce page dma_ops\n"); ---- --- set_dma_ops(dev, &bounce_dma_ops); ---- --- } -- - -- - return 0; ---- --- -- - unlink: -- - iommu_device_unlink(&iommu->iommu, dev); -- - return ret; ++ + return &iommu->iommu; } -- - static void intel_iommu_remove_device(struct device *dev) ++ + static void intel_iommu_release_device(struct device *dev) { struct intel_iommu *iommu; u8 bus, devfn; @@@@@@@@@ -5863,11 -5863,11 -5812,7 -5812,7 -5676,19 -5863,11 -5809,7 -5812,7 +5673,19 @@@@@@@@@ dmar_remove_one_dev_info(dev); -- - iommu_group_remove_device(dev); ++++ +++ set_dma_ops(dev, NULL); ++++ +++} ++ ++ -- - iommu_device_unlink(&iommu->iommu, dev); ++++ +++static void intel_iommu_probe_finalize(struct device *dev) ++++ +++{ ++++ +++ struct iommu_domain *domain; ++ ++ ++++ +++ domain = iommu_get_domain_for_dev(dev); if (device_needs_bounce(dev)) ++++ +++ set_dma_ops(dev, &bounce_dma_ops); ++++ +++ else if (domain && domain->type == IOMMU_DOMAIN_DMA) ++++ +++ set_dma_ops(dev, &intel_dma_ops); ++++ +++ else set_dma_ops(dev, NULL); } @@@@@@@@@ -6198,8 -6198,8 -6143,8 -6143,8 -6037,9 -6198,8 -6140,8 -6143,8 +6034,9 @@@@@@@@@ const struct iommu_ops intel_iommu_ops .map = intel_iommu_map, .unmap = intel_iommu_unmap, .iova_to_phys = intel_iommu_iova_to_phys, -- - .add_device = intel_iommu_add_device, -- - .remove_device = intel_iommu_remove_device, ++ + .probe_device = intel_iommu_probe_device, ++++ +++ .probe_finalize = intel_iommu_probe_finalize, ++ + .release_device = intel_iommu_release_device, .get_resv_regions = intel_iommu_get_resv_regions, .put_resv_regions = generic_iommu_put_resv_regions, .apply_resv_region = intel_iommu_apply_resv_region, @@@@@@@@@ -6209,7 -6209,7 -6154,8 -6154,8 -6049,16 -6209,7 -6151,8 -6154,8 +6046,16 @@@@@@@@@ .dev_enable_feat = intel_iommu_dev_enable_feat, .dev_disable_feat = intel_iommu_dev_disable_feat, .is_attach_deferred = intel_iommu_is_attach_deferred, ++ + .def_domain_type = device_def_domain_type, .pgsize_bitmap = INTEL_IOMMU_PGSIZES, ++++ +++#ifdef CONFIG_INTEL_IOMMU_SVM ++++ +++ .cache_invalidate = intel_iommu_sva_invalidate, ++++ +++ .sva_bind_gpasid = intel_svm_bind_gpasid, ++++ +++ .sva_unbind_gpasid = intel_svm_unbind_gpasid, ++++ +++ .sva_bind = intel_svm_bind, ++++ +++ .sva_unbind = intel_svm_unbind, ++++ +++ .sva_get_pasid = intel_svm_get_pasid, ++++ +++#endif }; static void quirk_iommu_igfx(struct pci_dev *dev) diff --cc drivers/iommu/iommu.c index 03d6a26687bc4,2b471419e26c3,4050569188bec,a9e5618cde802,374b34fd6fac9,2b471419e26c3,abcd191181699,629d209b8e884..b5ea203f6c683 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@@@@@@@@ -79,6 -79,6 -80,19 -80,19 -80,19 -79,6 -80,20 -80,19 +80,20 @@@@@@@@@ static bool iommu_cmd_line_dma_api(void return !!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API); } --- -static int iommu_alloc_default_domain(struct device *dev); ++++++ +static int iommu_alloc_default_domain(struct iommu_group *group, ++++++ + struct device *dev); ++ + static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, ++ + unsigned type); ++ + static int __iommu_attach_device(struct iommu_domain *domain, ++ + struct device *dev); ++ + static int __iommu_attach_group(struct iommu_domain *domain, ++ + struct iommu_group *group); ++ + static void __iommu_detach_group(struct iommu_domain *domain, ++ + struct iommu_group *group); ++ + static int iommu_create_device_direct_mappings(struct iommu_group *group, ++ + struct device *dev); ++ + static struct iommu_group *iommu_group_get_for_dev(struct device *dev); ++ + #define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \ struct iommu_group_attribute iommu_group_attr_##_name = \ __ATTR(_name, _mode, _show, _store) @@@@@@@@@ -175,15 -174,15 -189,16 -188,13 -189,16 -174,15 -190,16 -189,16 +190,16 @@@@@@@@@ static void dev_iommu_free(struct devic dev->iommu = NULL; } -- - int iommu_probe_device(struct device *dev) ++ + static int __iommu_probe_device(struct device *dev, struct list_head *group_list) { const struct iommu_ops *ops = dev->bus->iommu_ops; ++ + struct iommu_device *iommu_dev; ++ + struct iommu_group *group; int ret; -- - WARN_ON(dev->iommu_group); + if (!ops) -- - return -EINVAL; ++ + + return -ENODEV; + if (!dev_iommu_get(dev)) return -ENOMEM; @@@@@@@@@ -205,27 -204,27 -241,66 -237,66 -241,66 -204,27 -242,66 -241,66 +242,66 @@@@@@@@@ err_free return ret; } -- - void iommu_release_device(struct device *dev) ++ + int iommu_probe_device(struct device *dev) { const struct iommu_ops *ops = dev->bus->iommu_ops; ++ + struct iommu_group *group; ++ + int ret; -- - if (dev->iommu_group) -- - ops->remove_device(dev); ++ + ret = __iommu_probe_device(dev, NULL); ++ + if (ret) ++ + goto err_out; ++ + ++++++ + group = iommu_group_get(dev); ++++++ + if (!group) ++++++ + goto err_release; ++++++ + ++ + /* ++ + * Try to allocate a default domain - needs support from the ++ + * IOMMU driver. There are still some drivers which don't ++ + * support default domains, so the return value is not yet ++ + * checked. ++ + */ --- - iommu_alloc_default_domain(dev); --- - --- - group = iommu_group_get(dev); --- - if (!group) --- - goto err_release; ++++++ + iommu_alloc_default_domain(group, dev); ++ + ++ + if (group->default_domain) ++ + ret = __iommu_attach_device(group->default_domain, dev); ++ + ++ + iommu_create_device_direct_mappings(group, dev); ++ + ++ + iommu_group_put(group); ++ + ++ + if (ret) ++ + goto err_release; ++ + ++ + if (ops->probe_finalize) ++ + ops->probe_finalize(dev); ++ + ++ + return 0; ++ + ++ + err_release: ++ + iommu_release_device(dev); ++ + ++ + err_out: ++ + return ret; -- - if (dev->iommu) { -- - module_put(ops->owner); -- - dev_iommu_free(dev); -- - } } -- - static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, -- - unsigned type); -- - static int __iommu_attach_device(struct iommu_domain *domain, -- - struct device *dev); -- - static int __iommu_attach_group(struct iommu_domain *domain, -- - struct iommu_group *group); -- - static void __iommu_detach_group(struct iommu_domain *domain, -- - struct iommu_group *group); ++ + void iommu_release_device(struct device *dev) ++ + { ++ + const struct iommu_ops *ops = dev->bus->iommu_ops; ++ + ++ + if (!dev->iommu) ++ + return; ++ + ++ + iommu_device_unlink(dev->iommu->iommu_dev, dev); ++ + iommu_group_remove_device(dev); ++ + ++ + ops->release_device(dev); ++ + ++ + module_put(ops->owner); ++ + dev_iommu_free(dev); ++ + } static int __init iommu_set_def_domain_type(char *str) { @@@@@@@@@ -752,11 -742,11 -819,9 -815,9 -819,9 -742,11 -820,9 -828,9 +829,9 @@@@@@@@@ rename dev->iommu_group = group; -- - iommu_group_create_direct_mappings(group, dev); -- - mutex_lock(&group->mutex); list_add_tail(&device->list, &group->devices); ------ if (group->domain) ++++++ if (group->domain && !iommu_is_attach_deferred(group->domain, dev)) ret = __iommu_attach_device(group->domain, dev); mutex_unlock(&group->mutex); if (ret) @@@@@@@@@ -1371,6 -1361,6 -1436,65 -1432,65 -1436,65 -1361,6 -1437,61 -1445,65 +1446,61 @@@@@@@@@ struct iommu_group *fsl_mc_device_group } EXPORT_SYMBOL_GPL(fsl_mc_device_group); ++ + static int iommu_get_def_domain_type(struct device *dev) ++ + { ++ + const struct iommu_ops *ops = dev->bus->iommu_ops; ++ + unsigned int type = 0; ++ + ++ + if (ops->def_domain_type) ++ + type = ops->def_domain_type(dev); ++ + ++ + return (type == 0) ? iommu_def_domain_type : type; ++ + } ++ + ++ + static int iommu_group_alloc_default_domain(struct bus_type *bus, ++ + struct iommu_group *group, ++ + unsigned int type) ++ + { ++ + struct iommu_domain *dom; ++ + ++ + dom = __iommu_domain_alloc(bus, type); ++ + if (!dom && type != IOMMU_DOMAIN_DMA) { ++ + dom = __iommu_domain_alloc(bus, IOMMU_DOMAIN_DMA); ++ + if (dom) ++ + pr_warn("Failed to allocate default IOMMU domain of type %u for group %s - Falling back to IOMMU_DOMAIN_DMA", ++ + type, group->name); ++ + } ++ + ++ + if (!dom) ++ + return -ENOMEM; ++ + ++ + group->default_domain = dom; ++ + if (!group->domain) ++ + group->domain = dom; ++ + ++ + if (!iommu_dma_strict) { ++ + int attr = 1; ++ + iommu_domain_set_attr(dom, ++ + DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, ++ + &attr); ++ + } ++ + ++ + return 0; ++ + } ++ + --- -static int iommu_alloc_default_domain(struct device *dev) ++++++ +static int iommu_alloc_default_domain(struct iommu_group *group, ++++++ + struct device *dev) ++ + { --- - struct iommu_group *group; ++ + unsigned int type; ++ + --- - group = iommu_group_get(dev); --- - if (!group) --- - return -ENODEV; --- - ++ + if (group->default_domain) ++ + return 0; ++ + ++ + type = iommu_get_def_domain_type(dev); ++ + ++ + return iommu_group_alloc_default_domain(dev->bus, group, type); ++ + } ++ + /** * iommu_group_get_for_dev - Find or create the IOMMU group for a device * @dev: target device @@@@@@@@@ -1401,44 -1391,44 -1525,17 -1521,17 -1525,17 -1391,44 -1522,17 -1534,17 +1531,17 @@@@@@@@@ static struct iommu_group *iommu_group_ if (IS_ERR(group)) return group; -- - /* -- - * Try to allocate a default domain - needs support from the -- - * IOMMU driver. -- - */ -- - if (!group->default_domain) { -- - struct iommu_domain *dom; -- - -- - dom = __iommu_domain_alloc(dev->bus, iommu_def_domain_type); -- - if (!dom && iommu_def_domain_type != IOMMU_DOMAIN_DMA) { -- - dom = __iommu_domain_alloc(dev->bus, IOMMU_DOMAIN_DMA); -- - if (dom) { -- - dev_warn(dev, -- - "failed to allocate default IOMMU domain of type %u; falling back to IOMMU_DOMAIN_DMA", -- - iommu_def_domain_type); -- - } -- - } -- - -- - group->default_domain = dom; -- - if (!group->domain) -- - group->domain = dom; -- - -- - if (dom && !iommu_dma_strict) { -- - int attr = 1; -- - iommu_domain_set_attr(dom, -- - DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, -- - &attr); -- - } -- - } -- - ret = iommu_group_add_device(group, dev); -- - if (ret) { -- - iommu_group_put(group); -- - return ERR_PTR(ret); -- - } ++ + if (ret) ++ + goto out_put_group; return group; ++ + ++ + out_put_group: ++ + iommu_group_put(group); ++ + ++ + return ERR_PTR(ret); } - EXPORT_SYMBOL_GPL(iommu_group_get_for_dev); - - EXPORT_SYMBOL(iommu_group_get_for_dev); struct iommu_domain *iommu_group_default_domain(struct iommu_group *group) { @@@@@@@@@ -1519,10 -1509,10 -1621,139 -1617,139 -1621,139 -1509,10 -1618,147 -1630,139 +1627,148 @@@@@@@@@ static int iommu_bus_notifier(struct no return 0; } ++ + struct __group_domain_type { ++ + struct device *dev; ++ + unsigned int type; ++ + }; ++ + ++ + static int probe_get_default_domain_type(struct device *dev, void *data) ++ + { ++ + const struct iommu_ops *ops = dev->bus->iommu_ops; ++ + struct __group_domain_type *gtype = data; ++ + unsigned int type = 0; ++ + ++ + if (ops->def_domain_type) ++ + type = ops->def_domain_type(dev); ++ + ++ + if (type) { ++ + if (gtype->type && gtype->type != type) { ++ + dev_warn(dev, "Device needs domain type %s, but device %s in the same iommu group requires type %s - using default\n", ++ + iommu_domain_type_str(type), ++ + dev_name(gtype->dev), ++ + iommu_domain_type_str(gtype->type)); ++ + gtype->type = 0; ++ + } ++ + ++ + if (!gtype->dev) { ++ + gtype->dev = dev; ++ + gtype->type = type; ++ + } ++ + } ++ + ++ + return 0; ++ + } ++ + ++ + static void probe_alloc_default_domain(struct bus_type *bus, ++ + struct iommu_group *group) ++ + { ++ + struct __group_domain_type gtype; ++ + ++ + memset(>ype, 0, sizeof(gtype)); ++ + ++ + /* Ask for default domain requirements of all devices in the group */ ++ + __iommu_group_for_each_dev(group, >ype, ++ + probe_get_default_domain_type); ++ + ++ + if (!gtype.type) ++ + gtype.type = iommu_def_domain_type; ++ + ++ + iommu_group_alloc_default_domain(bus, group, gtype.type); ++ + ++ + } ++ + ++ + static int iommu_group_do_dma_attach(struct device *dev, void *data) ++ + { ++ + struct iommu_domain *domain = data; --- - const struct iommu_ops *ops; --- - int ret; --- --- ret = __iommu_attach_device(domain, dev); -- -- ops = domain->ops; ++ + - ret = __iommu_attach_device(domain, dev); - - - ops = domain->ops; - - --- - if (ret == 0 && ops->probe_finalize) --- - ops->probe_finalize(dev); --- - --- - return ret; ++++++ + return __iommu_attach_device(domain, dev); ++ + } ++ + ++ + static int __iommu_group_dma_attach(struct iommu_group *group) ++ + { ++ + return __iommu_group_for_each_dev(group, group->default_domain, ++ + iommu_group_do_dma_attach); ++ + } ++ + ++++++ +static int iommu_group_do_probe_finalize(struct device *dev, void *data) ++++++ +{ ++++++ + struct iommu_domain *domain = data; ++++++ + ++++++ + if (domain->ops->probe_finalize) ++++++ + domain->ops->probe_finalize(dev); ++++++ + ++++++ + return 0; ++++++ +} ++++++ + ++++++ +static void __iommu_group_dma_finalize(struct iommu_group *group) ++++++ +{ ++++++ + __iommu_group_for_each_dev(group, group->default_domain, ++++++ + iommu_group_do_probe_finalize); ++++++ +} ++++++++ ++ + static int iommu_do_create_direct_mappings(struct device *dev, void *data) ++ + { ++ + struct iommu_group *group = data; ++ + ++ + iommu_create_device_direct_mappings(group, dev); ++ + ++ + return 0; ++ + } ++ + ++ + static int iommu_group_create_direct_mappings(struct iommu_group *group) ++ + { ++ + return __iommu_group_for_each_dev(group, group, ++ + iommu_do_create_direct_mappings); ++ + } ++ + ++ + int bus_iommu_probe(struct bus_type *bus) ++ + { ++ + struct iommu_group *group, *next; ++ + LIST_HEAD(group_list); ++ + int ret; ++ + ++ + /* ++ + * This code-path does not allocate the default domain when ++ + * creating the iommu group, so do it after the groups are ++ + * created. ++ + */ ++ + ret = bus_for_each_dev(bus, NULL, &group_list, probe_iommu_group); ++ + if (ret) ++ + return ret; ++ + ++ + list_for_each_entry_safe(group, next, &group_list, entry) { ++ + /* Remove item from the list */ ++ + list_del_init(&group->entry); ++ + ++ + mutex_lock(&group->mutex); ++ + ++ + /* Try to allocate default domain */ ++ + probe_alloc_default_domain(bus, group); ++ + ++ + if (!group->default_domain) { ++ + mutex_unlock(&group->mutex); ++ + continue; ++ + } ++ + ++ + iommu_group_create_direct_mappings(group); ++ + ++ + ret = __iommu_group_dma_attach(group); ++ + ++ + mutex_unlock(&group->mutex); ++ + ++ + if (ret) ++ + break; ++++++ + ++++++ + __iommu_group_dma_finalize(group); ++ + } ++ + ++ + return ret; ++ + } ++ + static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops) { -- - int err; struct notifier_block *nb; ++ + int err; nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL); if (!nb) @@@@@@@@@ -2301,71 -2295,71 -2536,71 -2532,71 -2536,6 -2295,71 -2541,71 -2541,71 +2547,6 @@@@@@@@@ struct iommu_resv_region *iommu_alloc_r } EXPORT_SYMBOL_GPL(iommu_alloc_resv_region); ---- ---static int ---- ---request_default_domain_for_dev(struct device *dev, unsigned long type) ---- ---{ ---- --- struct iommu_domain *domain; ---- --- struct iommu_group *group; ---- --- int ret; ---- --- ---- --- /* Device must already be in a group before calling this function */ ---- --- group = iommu_group_get(dev); ---- --- if (!group) ---- --- return -EINVAL; ---- --- ---- --- mutex_lock(&group->mutex); ---- --- ---- --- ret = 0; ---- --- if (group->default_domain && group->default_domain->type == type) ---- --- goto out; ---- --- ---- --- /* Don't change mappings of existing devices */ ---- --- ret = -EBUSY; ---- --- if (iommu_group_device_count(group) != 1) ---- --- goto out; ---- --- ---- --- ret = -ENOMEM; ---- --- domain = __iommu_domain_alloc(dev->bus, type); ---- --- if (!domain) ---- --- goto out; ---- --- ---- --- /* Attach the device to the domain */ ---- --- ret = __iommu_attach_group(domain, group); ---- --- if (ret) { ---- --- iommu_domain_free(domain); ---- --- goto out; ---- --- } ---- --- ---- --- /* Make the domain the default for this group */ ---- --- if (group->default_domain) ---- --- iommu_domain_free(group->default_domain); ---- --- group->default_domain = domain; ---- --- -- - iommu_group_create_direct_mappings(group, dev); -- -- iommu_create_device_direct_mappings(group, dev); ---- --- ---- --- dev_info(dev, "Using iommu %s mapping\n", ---- --- type == IOMMU_DOMAIN_DMA ? "dma" : "direct"); ---- --- ---- --- ret = 0; ---- ---out: ---- --- mutex_unlock(&group->mutex); ---- --- iommu_group_put(group); ---- --- ---- --- return ret; ---- ---} ---- --- ---- ---/* Request that a device is direct mapped by the IOMMU */ ---- ---int iommu_request_dm_for_dev(struct device *dev) ---- ---{ ---- --- return request_default_domain_for_dev(dev, IOMMU_DOMAIN_IDENTITY); ---- ---} ---- --- ---- ---/* Request that a device can't be direct mapped by the IOMMU */ ---- ---int iommu_request_dma_domain_for_dev(struct device *dev) ---- ---{ ---- --- return request_default_domain_for_dev(dev, IOMMU_DOMAIN_DMA); ---- ---} ---- --- void iommu_set_default_passthrough(bool cmd_line) { if (cmd_line)