git.maquefel.me Git - linux.git/blob

1 // SPDX-License-Identifier: GPL-2.0-only

2 /*

4 * Author: Joerg Roedel <jroedel@suse.de>

5 * Leo Duran <leo.duran@amd.com>

6 */

8 #define pr_fmt(fmt) "AMD-Vi: " fmt

9 #define dev_fmt(fmt) pr_fmt(fmt)

11 #include <linux/ratelimit.h>

12 #include <linux/pci.h>

13 #include <linux/acpi.h>

14 #include <linux/pci-ats.h>

15 #include <linux/bitmap.h>

16 #include <linux/slab.h>

17 #include <linux/debugfs.h>

18 #include <linux/scatterlist.h>

19 #include <linux/dma-map-ops.h>

20 #include <linux/dma-direct.h>

21 #include <linux/iommu-helper.h>

22 #include <linux/delay.h>

23 #include <linux/amd-iommu.h>

24 #include <linux/notifier.h>

25 #include <linux/export.h>

26 #include <linux/irq.h>

27 #include <linux/msi.h>

28 #include <linux/irqdomain.h>

29 #include <linux/percpu.h>

30 #include <linux/io-pgtable.h>

31 #include <linux/cc_platform.h>

32 #include <asm/irq_remapping.h>

33 #include <asm/io_apic.h>

34 #include <asm/apic.h>

35 #include <asm/hw_irq.h>

36 #include <asm/proto.h>

37 #include <asm/iommu.h>

38 #include <asm/gart.h>

39 #include <asm/dma.h>

41 #include "amd_iommu.h"

42 #include "../dma-iommu.h"

43 #include "../irq_remapping.h"

45 #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))

47 #define LOOP_TIMEOUT 100000

49 /* IO virtual address start page frame number */

50 #define IOVA_START_PFN (1)

51 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)

53 /* Reserved IOVA ranges */

54 #define MSI_RANGE_START (0xfee00000)

55 #define MSI_RANGE_END (0xfeefffff)

56 #define HT_RANGE_START (0xfd00000000ULL)

57 #define HT_RANGE_END (0xffffffffffULL)

59 #define DEFAULT_PGTABLE_LEVEL PAGE_MODE_3_LEVEL

61 static DEFINE_SPINLOCK(pd_bitmap_lock);

63 LIST_HEAD(ioapic_map);

64 LIST_HEAD(hpet_map);

65 LIST_HEAD(acpihid_map);

67 const struct iommu_ops amd_iommu_ops;

69 static ATOMIC_NOTIFIER_HEAD(ppr_notifier);

70 int amd_iommu_max_glx_val = -1;

72 /*

73 * general struct to manage commands send to an IOMMU

74 */

75 struct iommu_cmd {

76 u32 data[4];

77 };

79 struct kmem_cache *amd_iommu_irq_cache;

81 static void detach_device(struct device *dev);

82 static int domain_enable_v2(struct protection_domain *domain, int pasids);

84 /****************************************************************************

85 *

86 * Helper functions

87 *

88 ****************************************************************************/

90 static inline int get_acpihid_device_id(struct device *dev,

91 struct acpihid_map_entry **entry)

92 {

93 struct acpi_device *adev = ACPI_COMPANION(dev);

94 struct acpihid_map_entry *p;

96 if (!adev)

97 return -ENODEV;

99 list_for_each_entry(p, &acpihid_map, list) {

100 if (acpi_dev_hid_uid_match(adev, p->hid,

101 p->uid[0] ? p->uid : NULL)) {

102 if (entry)

103 *entry = p;

104 return p->devid;

105 }

106 }

107 return -EINVAL;

108 }

109

110 static inline int get_device_sbdf_id(struct device *dev)

111 {

112 int sbdf;

113

114 if (dev_is_pci(dev))

115 sbdf = get_pci_sbdf_id(to_pci_dev(dev));

116 else

117 sbdf = get_acpihid_device_id(dev, NULL);

118

119 return sbdf;

120 }

121

122 struct dev_table_entry *get_dev_table(struct amd_iommu *iommu)

123 {

124 struct dev_table_entry *dev_table;

125 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

126

127 BUG_ON(pci_seg == NULL);

128 dev_table = pci_seg->dev_table;

129 BUG_ON(dev_table == NULL);

130

131 return dev_table;

132 }

133

134 static inline u16 get_device_segment(struct device *dev)

135 {

136 u16 seg;

137

138 if (dev_is_pci(dev)) {

139 struct pci_dev *pdev = to_pci_dev(dev);

140

141 seg = pci_domain_nr(pdev->bus);

142 } else {

143 u32 devid = get_acpihid_device_id(dev, NULL);

144

145 seg = PCI_SBDF_TO_SEGID(devid);

146 }

147

148 return seg;

149 }

150

151 /* Writes the specific IOMMU for a device into the PCI segment rlookup table */

152 void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid)

153 {

154 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

155

156 pci_seg->rlookup_table[devid] = iommu;

157 }

158

159 static struct amd_iommu *__rlookup_amd_iommu(u16 seg, u16 devid)

160 {

161 struct amd_iommu_pci_seg *pci_seg;

162

163 for_each_pci_segment(pci_seg) {

164 if (pci_seg->id == seg)

165 return pci_seg->rlookup_table[devid];

166 }

167 return NULL;

168 }

169

170 static struct amd_iommu *rlookup_amd_iommu(struct device *dev)

171 {

172 u16 seg = get_device_segment(dev);

173 int devid = get_device_sbdf_id(dev);

174

175 if (devid < 0)

176 return NULL;

177 return __rlookup_amd_iommu(seg, PCI_SBDF_TO_DEVID(devid));

178 }

179

180 static struct protection_domain *to_pdomain(struct iommu_domain *dom)

181 {

182 return container_of(dom, struct protection_domain, domain);

183 }

184

185 static struct iommu_dev_data *alloc_dev_data(struct amd_iommu *iommu, u16 devid)

186 {

187 struct iommu_dev_data *dev_data;

188 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

189

190 dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);

191 if (!dev_data)

192 return NULL;

193

194 spin_lock_init(&dev_data->lock);

195 dev_data->devid = devid;

196 ratelimit_default_init(&dev_data->rs);

197

198 llist_add(&dev_data->dev_data_list, &pci_seg->dev_data_list);

199 return dev_data;

200 }

201

202 static struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid)

203 {

204 struct iommu_dev_data *dev_data;

205 struct llist_node *node;

206 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

207

208 if (llist_empty(&pci_seg->dev_data_list))

209 return NULL;

210

211 node = pci_seg->dev_data_list.first;

212 llist_for_each_entry(dev_data, node, dev_data_list) {

213 if (dev_data->devid == devid)

214 return dev_data;

215 }

216

217 return NULL;

218 }

219

220 static int clone_alias(struct pci_dev *pdev, u16 alias, void *data)

221 {

222 struct amd_iommu *iommu;

223 struct dev_table_entry *dev_table;

224 u16 devid = pci_dev_id(pdev);

225

226 if (devid == alias)

227 return 0;

228

229 iommu = rlookup_amd_iommu(&pdev->dev);

230 if (!iommu)

231 return 0;

232

233 amd_iommu_set_rlookup_table(iommu, alias);

234 dev_table = get_dev_table(iommu);

235 memcpy(dev_table[alias].data,

236 dev_table[devid].data,

237 sizeof(dev_table[alias].data));

238

239 return 0;

240 }

241

242 static void clone_aliases(struct amd_iommu *iommu, struct device *dev)

243 {

244 struct pci_dev *pdev;

245

246 if (!dev_is_pci(dev))

247 return;

248 pdev = to_pci_dev(dev);

249

250 /*

251 * The IVRS alias stored in the alias table may not be

252 * part of the PCI DMA aliases if it's bus differs

253 * from the original device.

254 */

255 clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], NULL);

256

257 pci_for_each_dma_alias(pdev, clone_alias, NULL);

258 }

259

260 static void setup_aliases(struct amd_iommu *iommu, struct device *dev)

261 {

262 struct pci_dev *pdev = to_pci_dev(dev);

263 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

264 u16 ivrs_alias;

265

266 /* For ACPI HID devices, there are no aliases */

267 if (!dev_is_pci(dev))

268 return;

269

270 /*

271 * Add the IVRS alias to the pci aliases if it is on the same

272 * bus. The IVRS table may know about a quirk that we don't.

273 */

274 ivrs_alias = pci_seg->alias_table[pci_dev_id(pdev)];

275 if (ivrs_alias != pci_dev_id(pdev) &&

276 PCI_BUS_NUM(ivrs_alias) == pdev->bus->number)

277 pci_add_dma_alias(pdev, ivrs_alias & 0xff, 1);

278

279 clone_aliases(iommu, dev);

280 }

281

282 static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid)

283 {

284 struct iommu_dev_data *dev_data;

285

286 dev_data = search_dev_data(iommu, devid);

287

288 if (dev_data == NULL) {

289 dev_data = alloc_dev_data(iommu, devid);

290 if (!dev_data)

291 return NULL;

292

293 if (translation_pre_enabled(iommu))

294 dev_data->defer_attach = true;

295 }

296

297 return dev_data;

298 }

299

300 /*

301 * Find or create an IOMMU group for a acpihid device.

302 */

303 static struct iommu_group *acpihid_device_group(struct device *dev)

304 {

305 struct acpihid_map_entry *p, *entry = NULL;

306 int devid;

307

308 devid = get_acpihid_device_id(dev, &entry);

309 if (devid < 0)

310 return ERR_PTR(devid);

311

312 list_for_each_entry(p, &acpihid_map, list) {

313 if ((devid == p->devid) && p->group)

314 entry->group = p->group;

315 }

316

317 if (!entry->group)

318 entry->group = generic_device_group(dev);

319 else

320 iommu_group_ref_get(entry->group);

321

322 return entry->group;

323 }

324

325 static bool pci_iommuv2_capable(struct pci_dev *pdev)

326 {

327 static const int caps[] = {

328 PCI_EXT_CAP_ID_PRI,

329 PCI_EXT_CAP_ID_PASID,

330 };

331 int i, pos;

332

333 if (!pci_ats_supported(pdev))

334 return false;

335

336 for (i = 0; i < 2; ++i) {

337 pos = pci_find_ext_capability(pdev, caps[i]);

338 if (pos == 0)

339 return false;

340 }

341

342 return true;

343 }

344

345 /*

346 * This function checks if the driver got a valid device from the caller to

347 * avoid dereferencing invalid pointers.

348 */

349 static bool check_device(struct device *dev)

350 {

351 struct amd_iommu_pci_seg *pci_seg;

352 struct amd_iommu *iommu;

353 int devid, sbdf;

354

355 if (!dev)

356 return false;

357

358 sbdf = get_device_sbdf_id(dev);

359 if (sbdf < 0)

360 return false;

361 devid = PCI_SBDF_TO_DEVID(sbdf);

362

363 iommu = rlookup_amd_iommu(dev);

364 if (!iommu)

365 return false;

366

367 /* Out of our scope? */

368 pci_seg = iommu->pci_seg;

369 if (devid > pci_seg->last_bdf)

370 return false;

371

372 return true;

373 }

374

375 static int iommu_init_device(struct amd_iommu *iommu, struct device *dev)

376 {

377 struct iommu_dev_data *dev_data;

378 int devid, sbdf;

379

380 if (dev_iommu_priv_get(dev))

381 return 0;

382

383 sbdf = get_device_sbdf_id(dev);

384 if (sbdf < 0)

385 return sbdf;

386

387 devid = PCI_SBDF_TO_DEVID(sbdf);

388 dev_data = find_dev_data(iommu, devid);

389 if (!dev_data)

390 return -ENOMEM;

391

392 dev_data->dev = dev;

393 setup_aliases(iommu, dev);

394

395 /*

396 * By default we use passthrough mode for IOMMUv2 capable device.

397 * But if amd_iommu=force_isolation is set (e.g. to debug DMA to

398 * invalid address), we ignore the capability for the device so

399 * it'll be forced to go into translation mode.

400 */

401 if ((iommu_default_passthrough() || !amd_iommu_force_isolation) &&

402 dev_is_pci(dev) && pci_iommuv2_capable(to_pci_dev(dev))) {

403 dev_data->iommu_v2 = iommu->is_iommu_v2;

404 }

405

406 dev_iommu_priv_set(dev, dev_data);

407

408 return 0;

409 }

410

411 static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev)

412 {

413 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

414 struct dev_table_entry *dev_table = get_dev_table(iommu);

415 int devid, sbdf;

416

417 sbdf = get_device_sbdf_id(dev);

418 if (sbdf < 0)

419 return;

420

421 devid = PCI_SBDF_TO_DEVID(sbdf);

422 pci_seg->rlookup_table[devid] = NULL;

423 memset(&dev_table[devid], 0, sizeof(struct dev_table_entry));

424

425 setup_aliases(iommu, dev);

426 }

427

428 static void amd_iommu_uninit_device(struct device *dev)

429 {

430 struct iommu_dev_data *dev_data;

431

432 dev_data = dev_iommu_priv_get(dev);

433 if (!dev_data)

434 return;

435

436 if (dev_data->domain)

437 detach_device(dev);

438

439 dev_iommu_priv_set(dev, NULL);

440

441 /*

442 * We keep dev_data around for unplugged devices and reuse it when the

443 * device is re-plugged - not doing so would introduce a ton of races.

444 */

445 }

446

447 /****************************************************************************

448 *

449 * Interrupt handling functions

450 *

451 ****************************************************************************/

452

453 static void dump_dte_entry(struct amd_iommu *iommu, u16 devid)

454 {

455 int i;

456 struct dev_table_entry *dev_table = get_dev_table(iommu);

457

458 for (i = 0; i < 4; ++i)

459 pr_err("DTE[%d]: %016llx\n", i, dev_table[devid].data[i]);

460 }

461

462 static void dump_command(unsigned long phys_addr)

463 {

464 struct iommu_cmd *cmd = iommu_phys_to_virt(phys_addr);

465 int i;

466

467 for (i = 0; i < 4; ++i)

468 pr_err("CMD[%d]: %08x\n", i, cmd->data[i]);

469 }

470

471 static void amd_iommu_report_rmp_hw_error(struct amd_iommu *iommu, volatile u32 *event)

472 {

473 struct iommu_dev_data *dev_data = NULL;

474 int devid, vmg_tag, flags;

475 struct pci_dev *pdev;

476 u64 spa;

477

478 devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;

479 vmg_tag = (event[1]) & 0xFFFF;

480 flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;

481 spa = ((u64)event[3] << 32) | (event[2] & 0xFFFFFFF8);

482

483 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),

484 devid & 0xff);

485 if (pdev)

486 dev_data = dev_iommu_priv_get(&pdev->dev);

487

488 if (dev_data) {

489 if (__ratelimit(&dev_data->rs)) {

490 pci_err(pdev, "Event logged [RMP_HW_ERROR vmg_tag=0x%04x, spa=0x%llx, flags=0x%04x]\n",

491 vmg_tag, spa, flags);

492 }

493 } else {

494 pr_err_ratelimited("Event logged [RMP_HW_ERROR device=%04x:%02x:%02x.%x, vmg_tag=0x%04x, spa=0x%llx, flags=0x%04x]\n",

495 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

496 vmg_tag, spa, flags);

497 }

498

499 if (pdev)

500 pci_dev_put(pdev);

501 }

502

503 static void amd_iommu_report_rmp_fault(struct amd_iommu *iommu, volatile u32 *event)

504 {

505 struct iommu_dev_data *dev_data = NULL;

506 int devid, flags_rmp, vmg_tag, flags;

507 struct pci_dev *pdev;

508 u64 gpa;

509

510 devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;

511 flags_rmp = (event[0] >> EVENT_FLAGS_SHIFT) & 0xFF;

512 vmg_tag = (event[1]) & 0xFFFF;

513 flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;

514 gpa = ((u64)event[3] << 32) | event[2];

515

516 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),

517 devid & 0xff);

518 if (pdev)

519 dev_data = dev_iommu_priv_get(&pdev->dev);

520

521 if (dev_data) {

522 if (__ratelimit(&dev_data->rs)) {

523 pci_err(pdev, "Event logged [RMP_PAGE_FAULT vmg_tag=0x%04x, gpa=0x%llx, flags_rmp=0x%04x, flags=0x%04x]\n",

524 vmg_tag, gpa, flags_rmp, flags);

525 }

526 } else {

527 pr_err_ratelimited("Event logged [RMP_PAGE_FAULT device=%04x:%02x:%02x.%x, vmg_tag=0x%04x, gpa=0x%llx, flags_rmp=0x%04x, flags=0x%04x]\n",

528 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

529 vmg_tag, gpa, flags_rmp, flags);

530 }

531

532 if (pdev)

533 pci_dev_put(pdev);

534 }

535

536 #define IS_IOMMU_MEM_TRANSACTION(flags) \

537 (((flags) & EVENT_FLAG_I) == 0)

538

539 #define IS_WRITE_REQUEST(flags) \

540 ((flags) & EVENT_FLAG_RW)

541

542 static void amd_iommu_report_page_fault(struct amd_iommu *iommu,

543 u16 devid, u16 domain_id,

544 u64 address, int flags)

545 {

546 struct iommu_dev_data *dev_data = NULL;

547 struct pci_dev *pdev;

548

549 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),

550 devid & 0xff);

551 if (pdev)

552 dev_data = dev_iommu_priv_get(&pdev->dev);

553

554 if (dev_data) {

555 /*

556 * If this is a DMA fault (for which the I(nterrupt)

557 * bit will be unset), allow report_iommu_fault() to

558 * prevent logging it.

559 */

560 if (IS_IOMMU_MEM_TRANSACTION(flags)) {

561 /* Device not attached to domain properly */

562 if (dev_data->domain == NULL) {

563 pr_err_ratelimited("Event logged [Device not attached to domain properly]\n");

564 pr_err_ratelimited(" device=%04x:%02x:%02x.%x domain=0x%04x\n",

565 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid),

566 PCI_FUNC(devid), domain_id);

567 goto out;

568 }

569

570 if (!report_iommu_fault(&dev_data->domain->domain,

571 &pdev->dev, address,

572 IS_WRITE_REQUEST(flags) ?

573 IOMMU_FAULT_WRITE :

574 IOMMU_FAULT_READ))

575 goto out;

576 }

577

578 if (__ratelimit(&dev_data->rs)) {

579 pci_err(pdev, "Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%llx flags=0x%04x]\n",

580 domain_id, address, flags);

581 }

582 } else {

583 pr_err_ratelimited("Event logged [IO_PAGE_FAULT device=%04x:%02x:%02x.%x domain=0x%04x address=0x%llx flags=0x%04x]\n",

584 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

585 domain_id, address, flags);

586 }

587

588 out:

589 if (pdev)

590 pci_dev_put(pdev);

591 }

592

593 static void iommu_print_event(struct amd_iommu *iommu, void *__evt)

594 {

595 struct device *dev = iommu->iommu.dev;

596 int type, devid, flags, tag;

597 volatile u32 *event = __evt;

598 int count = 0;

599 u64 address;

600 u32 pasid;

601

602 retry:

603 type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;

604 devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;

605 pasid = (event[0] & EVENT_DOMID_MASK_HI) |

606 (event[1] & EVENT_DOMID_MASK_LO);

607 flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;

608 address = (u64)(((u64)event[3]) << 32) | event[2];

609

610 if (type == 0) {

611 /* Did we hit the erratum? */

612 if (++count == LOOP_TIMEOUT) {

613 pr_err("No event written to event log\n");

614 return;

615 }

616 udelay(1);

617 goto retry;

618 }

619

620 if (type == EVENT_TYPE_IO_FAULT) {

621 amd_iommu_report_page_fault(iommu, devid, pasid, address, flags);

622 return;

623 }

624

625 switch (type) {

626 case EVENT_TYPE_ILL_DEV:

627 dev_err(dev, "Event logged [ILLEGAL_DEV_TABLE_ENTRY device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n",

628 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

629 pasid, address, flags);

630 dump_dte_entry(iommu, devid);

631 break;

632 case EVENT_TYPE_DEV_TAB_ERR:

633 dev_err(dev, "Event logged [DEV_TAB_HARDWARE_ERROR device=%04x:%02x:%02x.%x "

634 "address=0x%llx flags=0x%04x]\n",

635 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

636 address, flags);

637 break;

638 case EVENT_TYPE_PAGE_TAB_ERR:

639 dev_err(dev, "Event logged [PAGE_TAB_HARDWARE_ERROR device=%04x:%02x:%02x.%x pasid=0x%04x address=0x%llx flags=0x%04x]\n",

640 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

641 pasid, address, flags);

642 break;

643 case EVENT_TYPE_ILL_CMD:

644 dev_err(dev, "Event logged [ILLEGAL_COMMAND_ERROR address=0x%llx]\n", address);

645 dump_command(address);

646 break;

647 case EVENT_TYPE_CMD_HARD_ERR:

648 dev_err(dev, "Event logged [COMMAND_HARDWARE_ERROR address=0x%llx flags=0x%04x]\n",

649 address, flags);

650 break;

651 case EVENT_TYPE_IOTLB_INV_TO:

652 dev_err(dev, "Event logged [IOTLB_INV_TIMEOUT device=%04x:%02x:%02x.%x address=0x%llx]\n",

653 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

654 address);

655 break;

656 case EVENT_TYPE_INV_DEV_REQ:

657 dev_err(dev, "Event logged [INVALID_DEVICE_REQUEST device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n",

658 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

659 pasid, address, flags);

660 break;

661 case EVENT_TYPE_RMP_FAULT:

662 amd_iommu_report_rmp_fault(iommu, event);

663 break;

664 case EVENT_TYPE_RMP_HW_ERR:

665 amd_iommu_report_rmp_hw_error(iommu, event);

666 break;

667 case EVENT_TYPE_INV_PPR_REQ:

668 pasid = PPR_PASID(*((u64 *)__evt));

669 tag = event[1] & 0x03FF;

670 dev_err(dev, "Event logged [INVALID_PPR_REQUEST device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x tag=0x%03x]\n",

671 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),

672 pasid, address, flags, tag);

673 break;

674 default:

675 dev_err(dev, "Event logged [UNKNOWN event[0]=0x%08x event[1]=0x%08x event[2]=0x%08x event[3]=0x%08x\n",

676 event[0], event[1], event[2], event[3]);

677 }

678

679 /*

680 * To detect the hardware errata 732 we need to clear the

681 * entry back to zero. This issue does not exist on SNP

682 * enabled system. Also this buffer is not writeable on

683 * SNP enabled system.

684 */

685 if (!amd_iommu_snp_en)

686 memset(__evt, 0, 4 * sizeof(u32));

687 }

688

689 static void iommu_poll_events(struct amd_iommu *iommu)

690 {

691 u32 head, tail;

692

693 head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);

694 tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);

695

696 while (head != tail) {

697 iommu_print_event(iommu, iommu->evt_buf + head);

698 head = (head + EVENT_ENTRY_SIZE) % EVT_BUFFER_SIZE;

699 }

700

701 writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);

702 }

703

704 static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw)

705 {

706 struct amd_iommu_fault fault;

707

708 if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {

709 pr_err_ratelimited("Unknown PPR request received\n");

710 return;

711 }

712

713 fault.address = raw[1];

714 fault.pasid = PPR_PASID(raw[0]);

715 fault.sbdf = PCI_SEG_DEVID_TO_SBDF(iommu->pci_seg->id, PPR_DEVID(raw[0]));

716 fault.tag = PPR_TAG(raw[0]);

717 fault.flags = PPR_FLAGS(raw[0]);

718

719 atomic_notifier_call_chain(&ppr_notifier, 0, &fault);

720 }

721

722 static void iommu_poll_ppr_log(struct amd_iommu *iommu)

723 {

724 u32 head, tail;

725

726 if (iommu->ppr_log == NULL)

727 return;

728

729 head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);

730 tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);

731

732 while (head != tail) {

733 volatile u64 *raw;

734 u64 entry[2];

735 int i;

736

737 raw = (u64 *)(iommu->ppr_log + head);

738

739 /*

740 * Hardware bug: Interrupt may arrive before the entry is

741 * written to memory. If this happens we need to wait for the

742 * entry to arrive.

743 */

744 for (i = 0; i < LOOP_TIMEOUT; ++i) {

745 if (PPR_REQ_TYPE(raw[0]) != 0)

746 break;

747 udelay(1);

748 }

749

750 /* Avoid memcpy function-call overhead */

751 entry[0] = raw[0];

752 entry[1] = raw[1];

753

754 /*

755 * To detect the hardware errata 733 we need to clear the

756 * entry back to zero. This issue does not exist on SNP

757 * enabled system. Also this buffer is not writeable on

758 * SNP enabled system.

759 */

760 if (!amd_iommu_snp_en)

761 raw[0] = raw[1] = 0UL;

762

763 /* Update head pointer of hardware ring-buffer */

764 head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;

765 writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);

766

767 /* Handle PPR entry */

768 iommu_handle_ppr_entry(iommu, entry);

769

770 /* Refresh ring-buffer information */

771 head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);

772 tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);

773 }

774 }

775

776 #ifdef CONFIG_IRQ_REMAP

777 static int (*iommu_ga_log_notifier)(u32);

778

779 int amd_iommu_register_ga_log_notifier(int (*notifier)(u32))

780 {

781 iommu_ga_log_notifier = notifier;

782

783 return 0;

784 }

785 EXPORT_SYMBOL(amd_iommu_register_ga_log_notifier);

786

787 static void iommu_poll_ga_log(struct amd_iommu *iommu)

788 {

789 u32 head, tail;

790

791 if (iommu->ga_log == NULL)

792 return;

793

794 head = readl(iommu->mmio_base + MMIO_GA_HEAD_OFFSET);

795 tail = readl(iommu->mmio_base + MMIO_GA_TAIL_OFFSET);

796

797 while (head != tail) {

798 volatile u64 *raw;

799 u64 log_entry;

800

801 raw = (u64 *)(iommu->ga_log + head);

802

803 /* Avoid memcpy function-call overhead */

804 log_entry = *raw;

805

806 /* Update head pointer of hardware ring-buffer */

807 head = (head + GA_ENTRY_SIZE) % GA_LOG_SIZE;

808 writel(head, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);

809

810 /* Handle GA entry */

811 switch (GA_REQ_TYPE(log_entry)) {

812 case GA_GUEST_NR:

813 if (!iommu_ga_log_notifier)

814 break;

815

816 pr_debug("%s: devid=%#x, ga_tag=%#x\n",

817 __func__, GA_DEVID(log_entry),

818 GA_TAG(log_entry));

819

820 if (iommu_ga_log_notifier(GA_TAG(log_entry)) != 0)

821 pr_err("GA log notifier failed.\n");

822 break;

823 default:

824 break;

825 }

826 }

827 }

828

829 static void

830 amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu)

831 {

832 if (!irq_remapping_enabled || !dev_is_pci(dev) ||

833 !pci_dev_has_default_msi_parent_domain(to_pci_dev(dev)))

834 return;

835

836 dev_set_msi_domain(dev, iommu->ir_domain);

837 }

838

839 #else /* CONFIG_IRQ_REMAP */

840 static inline void

841 amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu) { }

842 #endif /* !CONFIG_IRQ_REMAP */

843

844 #define AMD_IOMMU_INT_MASK \

845 (MMIO_STATUS_EVT_OVERFLOW_INT_MASK | \

846 MMIO_STATUS_EVT_INT_MASK | \

847 MMIO_STATUS_PPR_INT_MASK | \

848 MMIO_STATUS_GALOG_INT_MASK)

849

850 irqreturn_t amd_iommu_int_thread(int irq, void *data)

851 {

852 struct amd_iommu *iommu = (struct amd_iommu *) data;

853 u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);

854

855 while (status & AMD_IOMMU_INT_MASK) {

856 /* Enable interrupt sources again */

857 writel(AMD_IOMMU_INT_MASK,

858 iommu->mmio_base + MMIO_STATUS_OFFSET);

859

860 if (status & MMIO_STATUS_EVT_INT_MASK) {

861 pr_devel("Processing IOMMU Event Log\n");

862 iommu_poll_events(iommu);

863 }

864

865 if (status & MMIO_STATUS_PPR_INT_MASK) {

866 pr_devel("Processing IOMMU PPR Log\n");

867 iommu_poll_ppr_log(iommu);

868 }

869

870 #ifdef CONFIG_IRQ_REMAP

871 if (status & MMIO_STATUS_GALOG_INT_MASK) {

872 pr_devel("Processing IOMMU GA Log\n");

873 iommu_poll_ga_log(iommu);

874 }

875 #endif

876

877 if (status & MMIO_STATUS_EVT_OVERFLOW_INT_MASK) {

878 pr_info_ratelimited("IOMMU event log overflow\n");

879 amd_iommu_restart_event_logging(iommu);

880 }

881

882 /*

883 * Hardware bug: ERBT1312

884 * When re-enabling interrupt (by writing 1

885 * to clear the bit), the hardware might also try to set

886 * the interrupt bit in the event status register.

887 * In this scenario, the bit will be set, and disable

888 * subsequent interrupts.

889 *

890 * Workaround: The IOMMU driver should read back the

891 * status register and check if the interrupt bits are cleared.

892 * If not, driver will need to go through the interrupt handler

893 * again and re-clear the bits

894 */

895 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);

896 }

897 return IRQ_HANDLED;

898 }

899

900 irqreturn_t amd_iommu_int_handler(int irq, void *data)

901 {

902 return IRQ_WAKE_THREAD;

903 }

904

905 /****************************************************************************

906 *

907 * IOMMU command queuing functions

908 *

909 ****************************************************************************/

910

911 static int wait_on_sem(struct amd_iommu *iommu, u64 data)

912 {

913 int i = 0;

914

915 while (*iommu->cmd_sem != data && i < LOOP_TIMEOUT) {

916 udelay(1);

917 i += 1;

918 }

919

920 if (i == LOOP_TIMEOUT) {

921 pr_alert("Completion-Wait loop timed out\n");

922 return -EIO;

923 }

924

925 return 0;

926 }

927

928 static void copy_cmd_to_buffer(struct amd_iommu *iommu,

929 struct iommu_cmd *cmd)

930 {

931 u8 *target;

932 u32 tail;

933

934 /* Copy command to buffer */

935 tail = iommu->cmd_buf_tail;

936 target = iommu->cmd_buf + tail;

937 memcpy(target, cmd, sizeof(*cmd));

938

939 tail = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;

940 iommu->cmd_buf_tail = tail;

941

942 /* Tell the IOMMU about it */

943 writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);

944 }

945

946 static void build_completion_wait(struct iommu_cmd *cmd,

947 struct amd_iommu *iommu,

948 u64 data)

949 {

950 u64 paddr = iommu_virt_to_phys((void *)iommu->cmd_sem);

951

952 memset(cmd, 0, sizeof(*cmd));

953 cmd->data[0] = lower_32_bits(paddr) | CMD_COMPL_WAIT_STORE_MASK;

954 cmd->data[1] = upper_32_bits(paddr);

955 cmd->data[2] = lower_32_bits(data);

956 cmd->data[3] = upper_32_bits(data);

957 CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);

958 }

959

960 static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)

961 {

962 memset(cmd, 0, sizeof(*cmd));

963 cmd->data[0] = devid;

964 CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);

965 }

966

967 /*

968 * Builds an invalidation address which is suitable for one page or multiple

969 * pages. Sets the size bit (S) as needed is more than one page is flushed.

970 */

971 static inline u64 build_inv_address(u64 address, size_t size)

972 {

973 u64 pages, end, msb_diff;

974

975 pages = iommu_num_pages(address, size, PAGE_SIZE);

976

977 if (pages == 1)

978 return address & PAGE_MASK;

979

980 end = address + size - 1;

981

982 /*

983 * msb_diff would hold the index of the most significant bit that

984 * flipped between the start and end.

985 */

986 msb_diff = fls64(end ^ address) - 1;

987

988 /*

989 * Bits 63:52 are sign extended. If for some reason bit 51 is different

990 * between the start and the end, invalidate everything.

991 */

992 if (unlikely(msb_diff > 51)) {

993 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;

994 } else {

995 /*

996 * The msb-bit must be clear on the address. Just set all the

997 * lower bits.

998 */

999 address |= (1ull << msb_diff) - 1;

1000 }

1001

1002 /* Clear bits 11:0 */

1003 address &= PAGE_MASK;

1004

1005 /* Set the size bit - we flush more than one 4kb page */

1006 return address | CMD_INV_IOMMU_PAGES_SIZE_MASK;

1007 }

1008

1009 static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,

1010 size_t size, u16 domid, int pde)

1011 {

1012 u64 inv_address = build_inv_address(address, size);

1013

1014 memset(cmd, 0, sizeof(*cmd));

1015 cmd->data[1] |= domid;

1016 cmd->data[2] = lower_32_bits(inv_address);

1017 cmd->data[3] = upper_32_bits(inv_address);

1018 CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);

1019 if (pde) /* PDE bit - we want to flush everything, not only the PTEs */

1020 cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;

1021 }

1022

1023 static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,

1024 u64 address, size_t size)

1025 {

1026 u64 inv_address = build_inv_address(address, size);

1027

1028 memset(cmd, 0, sizeof(*cmd));

1029 cmd->data[0] = devid;

1030 cmd->data[0] |= (qdep & 0xff) << 24;

1031 cmd->data[1] = devid;

1032 cmd->data[2] = lower_32_bits(inv_address);

1033 cmd->data[3] = upper_32_bits(inv_address);

1034 CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);

1035 }

1036

1037 static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, u32 pasid,

1038 u64 address, bool size)

1039 {

1040 memset(cmd, 0, sizeof(*cmd));

1041

1042 address &= ~(0xfffULL);

1043

1044 cmd->data[0] = pasid;

1045 cmd->data[1] = domid;

1046 cmd->data[2] = lower_32_bits(address);

1047 cmd->data[3] = upper_32_bits(address);

1048 cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;

1049 cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;

1050 if (size)

1051 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;

1052 CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);

1053 }

1054

1055 static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, u32 pasid,

1056 int qdep, u64 address, bool size)

1057 {

1058 memset(cmd, 0, sizeof(*cmd));

1059

1060 address &= ~(0xfffULL);

1061

1062 cmd->data[0] = devid;

1063 cmd->data[0] |= ((pasid >> 8) & 0xff) << 16;

1064 cmd->data[0] |= (qdep & 0xff) << 24;

1065 cmd->data[1] = devid;

1066 cmd->data[1] |= (pasid & 0xff) << 16;

1067 cmd->data[2] = lower_32_bits(address);

1068 cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;

1069 cmd->data[3] = upper_32_bits(address);

1070 if (size)

1071 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;

1072 CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);

1073 }

1074

1075 static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, u32 pasid,

1076 int status, int tag, bool gn)

1077 {

1078 memset(cmd, 0, sizeof(*cmd));

1079

1080 cmd->data[0] = devid;

1081 if (gn) {

1082 cmd->data[1] = pasid;

1083 cmd->data[2] = CMD_INV_IOMMU_PAGES_GN_MASK;

1084 }

1085 cmd->data[3] = tag & 0x1ff;

1086 cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT;

1087

1088 CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR);

1089 }

1090

1091 static void build_inv_all(struct iommu_cmd *cmd)

1092 {

1093 memset(cmd, 0, sizeof(*cmd));

1094 CMD_SET_TYPE(cmd, CMD_INV_ALL);

1095 }

1096

1097 static void build_inv_irt(struct iommu_cmd *cmd, u16 devid)

1098 {

1099 memset(cmd, 0, sizeof(*cmd));

1100 cmd->data[0] = devid;

1101 CMD_SET_TYPE(cmd, CMD_INV_IRT);

1102 }

1103

1104 /*

1105 * Writes the command to the IOMMUs command buffer and informs the

1106 * hardware about the new command.

1107 */

1108 static int __iommu_queue_command_sync(struct amd_iommu *iommu,

1109 struct iommu_cmd *cmd,

1110 bool sync)

1111 {

1112 unsigned int count = 0;

1113 u32 left, next_tail;

1114

1115 next_tail = (iommu->cmd_buf_tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;

1116 again:

1117 left = (iommu->cmd_buf_head - next_tail) % CMD_BUFFER_SIZE;

1118

1119 if (left <= 0x20) {

1120 /* Skip udelay() the first time around */

1121 if (count++) {

1122 if (count == LOOP_TIMEOUT) {

1123 pr_err("Command buffer timeout\n");

1124 return -EIO;

1125 }

1126

1127 udelay(1);

1128 }

1129

1130 /* Update head and recheck remaining space */

1131 iommu->cmd_buf_head = readl(iommu->mmio_base +

1132 MMIO_CMD_HEAD_OFFSET);

1133

1134 goto again;

1135 }

1136

1137 copy_cmd_to_buffer(iommu, cmd);

1138

1139 /* Do we need to make sure all commands are processed? */

1140 iommu->need_sync = sync;

1141

1142 return 0;

1143 }

1144

1145 static int iommu_queue_command_sync(struct amd_iommu *iommu,

1146 struct iommu_cmd *cmd,

1147 bool sync)

1148 {

1149 unsigned long flags;

1150 int ret;

1151

1152 raw_spin_lock_irqsave(&iommu->lock, flags);

1153 ret = __iommu_queue_command_sync(iommu, cmd, sync);

1154 raw_spin_unlock_irqrestore(&iommu->lock, flags);

1155

1156 return ret;

1157 }

1158

1159 static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)

1160 {

1161 return iommu_queue_command_sync(iommu, cmd, true);

1162 }

1163

1164 /*

1165 * This function queues a completion wait command into the command

1166 * buffer of an IOMMU

1167 */

1168 static int iommu_completion_wait(struct amd_iommu *iommu)

1169 {

1170 struct iommu_cmd cmd;

1171 unsigned long flags;

1172 int ret;

1173 u64 data;

1174

1175 if (!iommu->need_sync)

1176 return 0;

1177

1178 data = atomic64_add_return(1, &iommu->cmd_sem_val);

1179 build_completion_wait(&cmd, iommu, data);

1180

1181 raw_spin_lock_irqsave(&iommu->lock, flags);

1182

1183 ret = __iommu_queue_command_sync(iommu, &cmd, false);

1184 if (ret)

1185 goto out_unlock;

1186

1187 ret = wait_on_sem(iommu, data);

1188

1189 out_unlock:

1190 raw_spin_unlock_irqrestore(&iommu->lock, flags);

1191

1192 return ret;

1193 }

1194

1195 static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)

1196 {

1197 struct iommu_cmd cmd;

1198

1199 build_inv_dte(&cmd, devid);

1200

1201 return iommu_queue_command(iommu, &cmd);

1202 }

1203

1204 static void amd_iommu_flush_dte_all(struct amd_iommu *iommu)

1205 {

1206 u32 devid;

1207 u16 last_bdf = iommu->pci_seg->last_bdf;

1208

1209 for (devid = 0; devid <= last_bdf; ++devid)

1210 iommu_flush_dte(iommu, devid);

1211

1212 iommu_completion_wait(iommu);

1213 }

1214

1215 /*

1216 * This function uses heavy locking and may disable irqs for some time. But

1217 * this is no issue because it is only called during resume.

1218 */

1219 static void amd_iommu_flush_tlb_all(struct amd_iommu *iommu)

1220 {

1221 u32 dom_id;

1222 u16 last_bdf = iommu->pci_seg->last_bdf;

1223

1224 for (dom_id = 0; dom_id <= last_bdf; ++dom_id) {

1225 struct iommu_cmd cmd;

1226 build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,

1227 dom_id, 1);

1228 iommu_queue_command(iommu, &cmd);

1229 }

1230

1231 iommu_completion_wait(iommu);

1232 }

1233

1234 static void amd_iommu_flush_tlb_domid(struct amd_iommu *iommu, u32 dom_id)

1235 {

1236 struct iommu_cmd cmd;

1237

1238 build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,

1239 dom_id, 1);

1240 iommu_queue_command(iommu, &cmd);

1241

1242 iommu_completion_wait(iommu);

1243 }

1244

1245 static void amd_iommu_flush_all(struct amd_iommu *iommu)

1246 {

1247 struct iommu_cmd cmd;

1248

1249 build_inv_all(&cmd);

1250

1251 iommu_queue_command(iommu, &cmd);

1252 iommu_completion_wait(iommu);

1253 }

1254

1255 static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)

1256 {

1257 struct iommu_cmd cmd;

1258

1259 build_inv_irt(&cmd, devid);

1260

1261 iommu_queue_command(iommu, &cmd);

1262 }

1263

1264 static void amd_iommu_flush_irt_all(struct amd_iommu *iommu)

1265 {

1266 u32 devid;

1267 u16 last_bdf = iommu->pci_seg->last_bdf;

1268

1269 if (iommu->irtcachedis_enabled)

1270 return;

1271

1272 for (devid = 0; devid <= last_bdf; devid++)

1273 iommu_flush_irt(iommu, devid);

1274

1275 iommu_completion_wait(iommu);

1276 }

1277

1278 void iommu_flush_all_caches(struct amd_iommu *iommu)

1279 {

1280 if (iommu_feature(iommu, FEATURE_IA)) {

1281 amd_iommu_flush_all(iommu);

1282 } else {

1283 amd_iommu_flush_dte_all(iommu);

1284 amd_iommu_flush_irt_all(iommu);

1285 amd_iommu_flush_tlb_all(iommu);

1286 }

1287 }

1288

1289 /*

1290 * Command send function for flushing on-device TLB

1291 */

1292 static int device_flush_iotlb(struct iommu_dev_data *dev_data,

1293 u64 address, size_t size)

1294 {

1295 struct amd_iommu *iommu;

1296 struct iommu_cmd cmd;

1297 int qdep;

1298

1299 qdep = dev_data->ats.qdep;

1300 iommu = rlookup_amd_iommu(dev_data->dev);

1301 if (!iommu)

1302 return -EINVAL;

1303

1304 build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);

1305

1306 return iommu_queue_command(iommu, &cmd);

1307 }

1308

1309 static int device_flush_dte_alias(struct pci_dev *pdev, u16 alias, void *data)

1310 {

1311 struct amd_iommu *iommu = data;

1312

1313 return iommu_flush_dte(iommu, alias);

1314 }

1315

1316 /*

1317 * Command send function for invalidating a device table entry

1318 */

1319 static int device_flush_dte(struct iommu_dev_data *dev_data)

1320 {

1321 struct amd_iommu *iommu;

1322 struct pci_dev *pdev = NULL;

1323 struct amd_iommu_pci_seg *pci_seg;

1324 u16 alias;

1325 int ret;

1326

1327 iommu = rlookup_amd_iommu(dev_data->dev);

1328 if (!iommu)

1329 return -EINVAL;

1330

1331 if (dev_is_pci(dev_data->dev))

1332 pdev = to_pci_dev(dev_data->dev);

1333

1334 if (pdev)

1335 ret = pci_for_each_dma_alias(pdev,

1336 device_flush_dte_alias, iommu);

1337 else

1338 ret = iommu_flush_dte(iommu, dev_data->devid);

1339 if (ret)

1340 return ret;

1341

1342 pci_seg = iommu->pci_seg;

1343 alias = pci_seg->alias_table[dev_data->devid];

1344 if (alias != dev_data->devid) {

1345 ret = iommu_flush_dte(iommu, alias);

1346 if (ret)

1347 return ret;

1348 }

1349

1350 if (dev_data->ats.enabled)

1351 ret = device_flush_iotlb(dev_data, 0, ~0UL);

1352

1353 return ret;

1354 }

1355

1356 /*

1357 * TLB invalidation function which is called from the mapping functions.

1358 * It invalidates a single PTE if the range to flush is within a single

1359 * page. Otherwise it flushes the whole TLB of the IOMMU.

1360 */

1361 static void __domain_flush_pages(struct protection_domain *domain,

1362 u64 address, size_t size, int pde)

1363 {

1364 struct iommu_dev_data *dev_data;

1365 struct iommu_cmd cmd;

1366 int ret = 0, i;

1367

1368 build_inv_iommu_pages(&cmd, address, size, domain->id, pde);

1369

1370 for (i = 0; i < amd_iommu_get_num_iommus(); ++i) {

1371 if (!domain->dev_iommu[i])

1372 continue;

1373

1374 /*

1375 * Devices of this domain are behind this IOMMU

1376 * We need a TLB flush

1377 */

1378 ret |= iommu_queue_command(amd_iommus[i], &cmd);

1379 }

1380

1381 list_for_each_entry(dev_data, &domain->dev_list, list) {

1382

1383 if (!dev_data->ats.enabled)

1384 continue;

1385

1386 ret |= device_flush_iotlb(dev_data, address, size);

1387 }

1388

1389 WARN_ON(ret);

1390 }

1391

1392 static void domain_flush_pages(struct protection_domain *domain,

1393 u64 address, size_t size, int pde)

1394 {

1395 if (likely(!amd_iommu_np_cache)) {

1396 __domain_flush_pages(domain, address, size, pde);

1397 return;

1398 }

1399

1400 /*

1401 * When NpCache is on, we infer that we run in a VM and use a vIOMMU.

1402 * In such setups it is best to avoid flushes of ranges which are not

1403 * naturally aligned, since it would lead to flushes of unmodified

1404 * PTEs. Such flushes would require the hypervisor to do more work than

1405 * necessary. Therefore, perform repeated flushes of aligned ranges

1406 * until you cover the range. Each iteration flushes the smaller

1407 * between the natural alignment of the address that we flush and the

1408 * greatest naturally aligned region that fits in the range.

1409 */

1410 while (size != 0) {

1411 int addr_alignment = __ffs(address);

1412 int size_alignment = __fls(size);

1413 int min_alignment;

1414 size_t flush_size;

1415

1416 /*

1417 * size is always non-zero, but address might be zero, causing

1418 * addr_alignment to be negative. As the casting of the

1419 * argument in __ffs(address) to long might trim the high bits

1420 * of the address on x86-32, cast to long when doing the check.

1421 */

1422 if (likely((unsigned long)address != 0))

1423 min_alignment = min(addr_alignment, size_alignment);

1424 else

1425 min_alignment = size_alignment;

1426

1427 flush_size = 1ul << min_alignment;

1428

1429 __domain_flush_pages(domain, address, flush_size, pde);

1430 address += flush_size;

1431 size -= flush_size;

1432 }

1433 }

1434

1435 /* Flush the whole IO/TLB for a given protection domain - including PDE */

1436 void amd_iommu_domain_flush_tlb_pde(struct protection_domain *domain)

1437 {

1438 domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);

1439 }

1440

1441 void amd_iommu_domain_flush_complete(struct protection_domain *domain)

1442 {

1443 int i;

1444

1445 for (i = 0; i < amd_iommu_get_num_iommus(); ++i) {

1446 if (domain && !domain->dev_iommu[i])

1447 continue;

1448

1449 /*

1450 * Devices of this domain are behind this IOMMU

1451 * We need to wait for completion of all commands.

1452 */

1453 iommu_completion_wait(amd_iommus[i]);

1454 }

1455 }

1456

1457 /* Flush the not present cache if it exists */

1458 static void domain_flush_np_cache(struct protection_domain *domain,

1459 dma_addr_t iova, size_t size)

1460 {

1461 if (unlikely(amd_iommu_np_cache)) {

1462 unsigned long flags;

1463

1464 spin_lock_irqsave(&domain->lock, flags);

1465 domain_flush_pages(domain, iova, size, 1);

1466 amd_iommu_domain_flush_complete(domain);

1467 spin_unlock_irqrestore(&domain->lock, flags);

1468 }

1469 }

1470

1471

1472 /*

1473 * This function flushes the DTEs for all devices in domain

1474 */

1475 static void domain_flush_devices(struct protection_domain *domain)

1476 {

1477 struct iommu_dev_data *dev_data;

1478

1479 list_for_each_entry(dev_data, &domain->dev_list, list)

1480 device_flush_dte(dev_data);

1481 }

1482

1483 /****************************************************************************

1484 *

1485 * The next functions belong to the domain allocation. A domain is

1486 * allocated for every IOMMU as the default domain. If device isolation

1487 * is enabled, every device get its own domain. The most important thing

1488 * about domains is the page table mapping the DMA address space they

1489 * contain.

1490 *

1491 ****************************************************************************/

1492

1493 static u16 domain_id_alloc(void)

1494 {

1495 int id;

1496

1497 spin_lock(&pd_bitmap_lock);

1498 id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);

1499 BUG_ON(id == 0);

1500 if (id > 0 && id < MAX_DOMAIN_ID)

1501 __set_bit(id, amd_iommu_pd_alloc_bitmap);

1502 else

1503 id = 0;

1504 spin_unlock(&pd_bitmap_lock);

1505

1506 return id;

1507 }

1508

1509 static void domain_id_free(int id)

1510 {

1511 spin_lock(&pd_bitmap_lock);

1512 if (id > 0 && id < MAX_DOMAIN_ID)

1513 __clear_bit(id, amd_iommu_pd_alloc_bitmap);

1514 spin_unlock(&pd_bitmap_lock);

1515 }

1516

1517 static void free_gcr3_tbl_level1(u64 *tbl)

1518 {

1519 u64 *ptr;

1520 int i;

1521

1522 for (i = 0; i < 512; ++i) {

1523 if (!(tbl[i] & GCR3_VALID))

1524 continue;

1525

1526 ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK);

1527

1528 free_page((unsigned long)ptr);

1529 }

1530 }

1531

1532 static void free_gcr3_tbl_level2(u64 *tbl)

1533 {

1534 u64 *ptr;

1535 int i;

1536

1537 for (i = 0; i < 512; ++i) {

1538 if (!(tbl[i] & GCR3_VALID))

1539 continue;

1540

1541 ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK);

1542

1543 free_gcr3_tbl_level1(ptr);

1544 }

1545 }

1546

1547 static void free_gcr3_table(struct protection_domain *domain)

1548 {

1549 if (domain->glx == 2)

1550 free_gcr3_tbl_level2(domain->gcr3_tbl);

1551 else if (domain->glx == 1)

1552 free_gcr3_tbl_level1(domain->gcr3_tbl);

1553 else

1554 BUG_ON(domain->glx != 0);

1555

1556 free_page((unsigned long)domain->gcr3_tbl);

1557 }

1558

1559 static void set_dte_entry(struct amd_iommu *iommu, u16 devid,

1560 struct protection_domain *domain, bool ats, bool ppr)

1561 {

1562 u64 pte_root = 0;

1563 u64 flags = 0;

1564 u32 old_domid;

1565 struct dev_table_entry *dev_table = get_dev_table(iommu);

1566

1567 if (domain->iop.mode != PAGE_MODE_NONE)

1568 pte_root = iommu_virt_to_phys(domain->iop.root);

1569

1570 pte_root |= (domain->iop.mode & DEV_ENTRY_MODE_MASK)

1571 << DEV_ENTRY_MODE_SHIFT;

1572

1573 pte_root |= DTE_FLAG_IR | DTE_FLAG_IW | DTE_FLAG_V;

1574

1575 /*

1576 * When SNP is enabled, Only set TV bit when IOMMU

1577 * page translation is in use.

1578 */

1579 if (!amd_iommu_snp_en || (domain->id != 0))

1580 pte_root |= DTE_FLAG_TV;

1581

1582 flags = dev_table[devid].data[1];

1583

1584 if (ats)

1585 flags |= DTE_FLAG_IOTLB;

1586

1587 if (ppr) {

1588 if (iommu_feature(iommu, FEATURE_EPHSUP))

1589 pte_root |= 1ULL << DEV_ENTRY_PPR;

1590 }

1591

1592 if (domain->flags & PD_IOMMUV2_MASK) {

1593 u64 gcr3 = iommu_virt_to_phys(domain->gcr3_tbl);

1594 u64 glx = domain->glx;

1595 u64 tmp;

1596

1597 pte_root |= DTE_FLAG_GV;

1598 pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;

1599

1600 /* First mask out possible old values for GCR3 table */

1601 tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;

1602 flags &= ~tmp;

1603

1604 tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;

1605 flags &= ~tmp;

1606

1607 /* Encode GCR3 table into DTE */

1608 tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;

1609 pte_root |= tmp;

1610

1611 tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;

1612 flags |= tmp;

1613

1614 tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;

1615 flags |= tmp;

1616

1617 if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL) {

1618 dev_table[devid].data[2] |=

1619 ((u64)GUEST_PGTABLE_5_LEVEL << DTE_GPT_LEVEL_SHIFT);

1620 }

1621

1622 if (domain->flags & PD_GIOV_MASK)

1623 pte_root |= DTE_FLAG_GIOV;

1624 }

1625

1626 flags &= ~DEV_DOMID_MASK;

1627 flags |= domain->id;

1628

1629 old_domid = dev_table[devid].data[1] & DEV_DOMID_MASK;

1630 dev_table[devid].data[1] = flags;

1631 dev_table[devid].data[0] = pte_root;

1632

1633 /*

1634 * A kdump kernel might be replacing a domain ID that was copied from

1635 * the previous kernel--if so, it needs to flush the translation cache

1636 * entries for the old domain ID that is being overwritten

1637 */

1638 if (old_domid) {

1639 amd_iommu_flush_tlb_domid(iommu, old_domid);

1640 }

1641 }

1642

1643 static void clear_dte_entry(struct amd_iommu *iommu, u16 devid)

1644 {

1645 struct dev_table_entry *dev_table = get_dev_table(iommu);

1646

1647 /* remove entry from the device table seen by the hardware */

1648 dev_table[devid].data[0] = DTE_FLAG_V;

1649

1650 if (!amd_iommu_snp_en)

1651 dev_table[devid].data[0] |= DTE_FLAG_TV;

1652

1653 dev_table[devid].data[1] &= DTE_FLAG_MASK;

1654

1655 amd_iommu_apply_erratum_63(iommu, devid);

1656 }

1657

1658 static void do_attach(struct iommu_dev_data *dev_data,

1659 struct protection_domain *domain)

1660 {

1661 struct amd_iommu *iommu;

1662 bool ats;

1663

1664 iommu = rlookup_amd_iommu(dev_data->dev);

1665 if (!iommu)

1666 return;

1667 ats = dev_data->ats.enabled;

1668

1669 /* Update data structures */

1670 dev_data->domain = domain;

1671 list_add(&dev_data->list, &domain->dev_list);

1672

1673 /* Update NUMA Node ID */

1674 if (domain->nid == NUMA_NO_NODE)

1675 domain->nid = dev_to_node(dev_data->dev);

1676

1677 /* Do reference counting */

1678 domain->dev_iommu[iommu->index] += 1;

1679 domain->dev_cnt += 1;

1680

1681 /* Update device table */

1682 set_dte_entry(iommu, dev_data->devid, domain,

1683 ats, dev_data->iommu_v2);

1684 clone_aliases(iommu, dev_data->dev);

1685

1686 device_flush_dte(dev_data);

1687 }

1688

1689 static void do_detach(struct iommu_dev_data *dev_data)

1690 {

1691 struct protection_domain *domain = dev_data->domain;

1692 struct amd_iommu *iommu;

1693

1694 iommu = rlookup_amd_iommu(dev_data->dev);

1695 if (!iommu)

1696 return;

1697

1698 /* Update data structures */

1699 dev_data->domain = NULL;

1700 list_del(&dev_data->list);

1701 clear_dte_entry(iommu, dev_data->devid);

1702 clone_aliases(iommu, dev_data->dev);

1703

1704 /* Flush the DTE entry */

1705 device_flush_dte(dev_data);

1706

1707 /* Flush IOTLB */

1708 amd_iommu_domain_flush_tlb_pde(domain);

1709

1710 /* Wait for the flushes to finish */

1711 amd_iommu_domain_flush_complete(domain);

1712

1713 /* decrease reference counters - needs to happen after the flushes */

1714 domain->dev_iommu[iommu->index] -= 1;

1715 domain->dev_cnt -= 1;

1716 }

1717

1718 static void pdev_iommuv2_disable(struct pci_dev *pdev)

1719 {

1720 pci_disable_ats(pdev);

1721 pci_disable_pri(pdev);

1722 pci_disable_pasid(pdev);

1723 }

1724

1725 static int pdev_pri_ats_enable(struct pci_dev *pdev)

1726 {

1727 int ret;

1728

1729 /* Only allow access to user-accessible pages */

1730 ret = pci_enable_pasid(pdev, 0);

1731 if (ret)

1732 return ret;

1733

1734 /* First reset the PRI state of the device */

1735 ret = pci_reset_pri(pdev);

1736 if (ret)

1737 goto out_err_pasid;

1738

1739 /* Enable PRI */

1740 /* FIXME: Hardcode number of outstanding requests for now */

1741 ret = pci_enable_pri(pdev, 32);

1742 if (ret)

1743 goto out_err_pasid;

1744

1745 ret = pci_enable_ats(pdev, PAGE_SHIFT);

1746 if (ret)

1747 goto out_err_pri;

1748

1749 return 0;

1750

1751 out_err_pri:

1752 pci_disable_pri(pdev);

1753

1754 out_err_pasid:

1755 pci_disable_pasid(pdev);

1756

1757 return ret;

1758 }

1759

1760 /*

1761 * If a device is not yet associated with a domain, this function makes the

1762 * device visible in the domain

1763 */

1764 static int attach_device(struct device *dev,

1765 struct protection_domain *domain)

1766 {

1767 struct iommu_dev_data *dev_data;

1768 struct pci_dev *pdev;

1769 unsigned long flags;

1770 int ret;

1771

1772 spin_lock_irqsave(&domain->lock, flags);

1773

1774 dev_data = dev_iommu_priv_get(dev);

1775

1776 spin_lock(&dev_data->lock);

1777

1778 ret = -EBUSY;

1779 if (dev_data->domain != NULL)

1780 goto out;

1781

1782 if (!dev_is_pci(dev))

1783 goto skip_ats_check;

1784

1785 pdev = to_pci_dev(dev);

1786 if (domain->flags & PD_IOMMUV2_MASK) {

1787 struct iommu_domain *def_domain = iommu_get_dma_domain(dev);

1788

1789 ret = -EINVAL;

1790

1791 /*

1792 * In case of using AMD_IOMMU_V1 page table mode and the device

1793 * is enabling for PPR/ATS support (using v2 table),

1794 * we need to make sure that the domain type is identity map.

1795 */

1796 if ((amd_iommu_pgtable == AMD_IOMMU_V1) &&

1797 def_domain->type != IOMMU_DOMAIN_IDENTITY) {

1798 goto out;

1799 }

1800

1801 if (dev_data->iommu_v2) {

1802 if (pdev_pri_ats_enable(pdev) != 0)

1803 goto out;

1804

1805 dev_data->ats.enabled = true;

1806 dev_data->ats.qdep = pci_ats_queue_depth(pdev);

1807 dev_data->pri_tlp = pci_prg_resp_pasid_required(pdev);

1808 }

1809 } else if (amd_iommu_iotlb_sup &&

1810 pci_enable_ats(pdev, PAGE_SHIFT) == 0) {

1811 dev_data->ats.enabled = true;

1812 dev_data->ats.qdep = pci_ats_queue_depth(pdev);

1813 }

1814

1815 skip_ats_check:

1816 ret = 0;

1817

1818 do_attach(dev_data, domain);

1819

1820 /*

1821 * We might boot into a crash-kernel here. The crashed kernel

1822 * left the caches in the IOMMU dirty. So we have to flush

1823 * here to evict all dirty stuff.

1824 */

1825 amd_iommu_domain_flush_tlb_pde(domain);

1826

1827 amd_iommu_domain_flush_complete(domain);

1828

1829 out:

1830 spin_unlock(&dev_data->lock);

1831

1832 spin_unlock_irqrestore(&domain->lock, flags);

1833

1834 return ret;

1835 }

1836

1837 /*

1838 * Removes a device from a protection domain (with devtable_lock held)

1839 */

1840 static void detach_device(struct device *dev)

1841 {

1842 struct protection_domain *domain;

1843 struct iommu_dev_data *dev_data;

1844 unsigned long flags;

1845

1846 dev_data = dev_iommu_priv_get(dev);

1847 domain = dev_data->domain;

1848

1849 spin_lock_irqsave(&domain->lock, flags);

1850

1851 spin_lock(&dev_data->lock);

1852

1853 /*

1854 * First check if the device is still attached. It might already

1855 * be detached from its domain because the generic

1856 * iommu_detach_group code detached it and we try again here in

1857 * our alias handling.

1858 */

1859 if (WARN_ON(!dev_data->domain))

1860 goto out;

1861

1862 do_detach(dev_data);

1863

1864 if (!dev_is_pci(dev))

1865 goto out;

1866

1867 if (domain->flags & PD_IOMMUV2_MASK && dev_data->iommu_v2)

1868 pdev_iommuv2_disable(to_pci_dev(dev));

1869 else if (dev_data->ats.enabled)

1870 pci_disable_ats(to_pci_dev(dev));

1871

1872 dev_data->ats.enabled = false;

1873

1874 out:

1875 spin_unlock(&dev_data->lock);

1876

1877 spin_unlock_irqrestore(&domain->lock, flags);

1878 }

1879

1880 static struct iommu_device *amd_iommu_probe_device(struct device *dev)

1881 {

1882 struct iommu_device *iommu_dev;

1883 struct amd_iommu *iommu;

1884 int ret;

1885

1886 if (!check_device(dev))

1887 return ERR_PTR(-ENODEV);

1888

1889 iommu = rlookup_amd_iommu(dev);

1890 if (!iommu)

1891 return ERR_PTR(-ENODEV);

1892

1893 /* Not registered yet? */

1894 if (!iommu->iommu.ops)

1895 return ERR_PTR(-ENODEV);

1896

1897 if (dev_iommu_priv_get(dev))

1898 return &iommu->iommu;

1899

1900 ret = iommu_init_device(iommu, dev);

1901 if (ret) {

1902 if (ret != -ENOTSUPP)

1903 dev_err(dev, "Failed to initialize - trying to proceed anyway\n");

1904 iommu_dev = ERR_PTR(ret);

1905 iommu_ignore_device(iommu, dev);

1906 } else {

1907 amd_iommu_set_pci_msi_domain(dev, iommu);

1908 iommu_dev = &iommu->iommu;

1909 }

1910

1911 iommu_completion_wait(iommu);

1912

1913 return iommu_dev;

1914 }

1915

1916 static void amd_iommu_probe_finalize(struct device *dev)

1917 {

1918 /* Domains are initialized for this device - have a look what we ended up with */

1919 set_dma_ops(dev, NULL);

1920 iommu_setup_dma_ops(dev, 0, U64_MAX);

1921 }

1922

1923 static void amd_iommu_release_device(struct device *dev)

1924 {

1925 struct amd_iommu *iommu;

1926

1927 if (!check_device(dev))

1928 return;

1929

1930 iommu = rlookup_amd_iommu(dev);

1931 if (!iommu)

1932 return;

1933

1934 amd_iommu_uninit_device(dev);

1935 iommu_completion_wait(iommu);

1936 }

1937

1938 static struct iommu_group *amd_iommu_device_group(struct device *dev)

1939 {

1940 if (dev_is_pci(dev))

1941 return pci_device_group(dev);

1942

1943 return acpihid_device_group(dev);

1944 }

1945

1946 /*****************************************************************************

1947 *

1948 * The next functions belong to the dma_ops mapping/unmapping code.

1949 *

1950 *****************************************************************************/

1951

1952 static void update_device_table(struct protection_domain *domain)

1953 {

1954 struct iommu_dev_data *dev_data;

1955

1956 list_for_each_entry(dev_data, &domain->dev_list, list) {

1957 struct amd_iommu *iommu = rlookup_amd_iommu(dev_data->dev);

1958

1959 if (!iommu)

1960 continue;

1961 set_dte_entry(iommu, dev_data->devid, domain,

1962 dev_data->ats.enabled, dev_data->iommu_v2);

1963 clone_aliases(iommu, dev_data->dev);

1964 }

1965 }

1966

1967 void amd_iommu_update_and_flush_device_table(struct protection_domain *domain)

1968 {

1969 update_device_table(domain);

1970 domain_flush_devices(domain);

1971 }

1972

1973 void amd_iommu_domain_update(struct protection_domain *domain)

1974 {

1975 /* Update device table */

1976 amd_iommu_update_and_flush_device_table(domain);

1977

1978 /* Flush domain TLB(s) and wait for completion */

1979 amd_iommu_domain_flush_tlb_pde(domain);

1980 amd_iommu_domain_flush_complete(domain);

1981 }

1982

1983 /*****************************************************************************

1984 *

1985 * The following functions belong to the exported interface of AMD IOMMU

1986 *

1987 * This interface allows access to lower level functions of the IOMMU

1988 * like protection domain handling and assignement of devices to domains

1989 * which is not possible with the dma_ops interface.

1990 *

1991 *****************************************************************************/

1992

1993 static void cleanup_domain(struct protection_domain *domain)

1994 {

1995 struct iommu_dev_data *entry;

1996 unsigned long flags;

1997

1998 spin_lock_irqsave(&domain->lock, flags);

1999

2000 while (!list_empty(&domain->dev_list)) {

2001 entry = list_first_entry(&domain->dev_list,

2002 struct iommu_dev_data, list);

2003 BUG_ON(!entry->domain);

2004 do_detach(entry);

2005 }

2006

2007 spin_unlock_irqrestore(&domain->lock, flags);

2008 }

2009

2010 static void protection_domain_free(struct protection_domain *domain)

2011 {

2012 if (!domain)

2013 return;

2014

2015 if (domain->iop.pgtbl_cfg.tlb)

2016 free_io_pgtable_ops(&domain->iop.iop.ops);

2017

2018 if (domain->id)

2019 domain_id_free(domain->id);

2020

2021 kfree(domain);

2022 }

2023

2024 static int protection_domain_init_v1(struct protection_domain *domain, int mode)

2025 {

2026 u64 *pt_root = NULL;

2027

2028 BUG_ON(mode < PAGE_MODE_NONE || mode > PAGE_MODE_6_LEVEL);

2029

2030 spin_lock_init(&domain->lock);

2031 domain->id = domain_id_alloc();

2032 if (!domain->id)

2033 return -ENOMEM;

2034 INIT_LIST_HEAD(&domain->dev_list);

2035

2036 if (mode != PAGE_MODE_NONE) {

2037 pt_root = (void *)get_zeroed_page(GFP_KERNEL);

2038 if (!pt_root) {

2039 domain_id_free(domain->id);

2040 return -ENOMEM;

2041 }

2042 }

2043

2044 amd_iommu_domain_set_pgtable(domain, pt_root, mode);

2045

2046 return 0;

2047 }

2048

2049 static int protection_domain_init_v2(struct protection_domain *domain)

2050 {

2051 spin_lock_init(&domain->lock);

2052 domain->id = domain_id_alloc();

2053 if (!domain->id)

2054 return -ENOMEM;

2055 INIT_LIST_HEAD(&domain->dev_list);

2056

2057 domain->flags |= PD_GIOV_MASK;

2058

2059 domain->domain.pgsize_bitmap = AMD_IOMMU_PGSIZES_V2;

2060

2061 if (domain_enable_v2(domain, 1)) {

2062 domain_id_free(domain->id);

2063 return -ENOMEM;

2064 }

2065

2066 return 0;

2067 }

2068

2069 static struct protection_domain *protection_domain_alloc(unsigned int type)

2070 {

2071 struct io_pgtable_ops *pgtbl_ops;

2072 struct protection_domain *domain;

2073 int pgtable = amd_iommu_pgtable;

2074 int mode = DEFAULT_PGTABLE_LEVEL;

2075 int ret;

2076

2077 domain = kzalloc(sizeof(*domain), GFP_KERNEL);

2078 if (!domain)

2079 return NULL;

2080

2081 /*

2082 * Force IOMMU v1 page table when iommu=pt and

2083 * when allocating domain for pass-through devices.

2084 */

2085 if (type == IOMMU_DOMAIN_IDENTITY) {

2086 pgtable = AMD_IOMMU_V1;

2087 mode = PAGE_MODE_NONE;

2088 } else if (type == IOMMU_DOMAIN_UNMANAGED) {

2089 pgtable = AMD_IOMMU_V1;

2090 }

2091

2092 switch (pgtable) {

2093 case AMD_IOMMU_V1:

2094 ret = protection_domain_init_v1(domain, mode);

2095 break;

2096 case AMD_IOMMU_V2:

2097 ret = protection_domain_init_v2(domain);

2098 break;

2099 default:

2100 ret = -EINVAL;

2101 }

2102

2103 if (ret)

2104 goto out_err;

2105

2106 /* No need to allocate io pgtable ops in passthrough mode */

2107 if (type == IOMMU_DOMAIN_IDENTITY)

2108 return domain;

2109

2110 domain->nid = NUMA_NO_NODE;

2111

2112 pgtbl_ops = alloc_io_pgtable_ops(pgtable, &domain->iop.pgtbl_cfg, domain);

2113 if (!pgtbl_ops) {

2114 domain_id_free(domain->id);

2115 goto out_err;

2116 }

2117

2118 return domain;

2119 out_err:

2120 kfree(domain);

2121 return NULL;

2122 }

2123

2124 static struct iommu_domain *amd_iommu_domain_alloc(unsigned type)

2125 {

2126 struct protection_domain *domain;

2127

2128 /*

2129 * Since DTE[Mode]=0 is prohibited on SNP-enabled system,

2130 * default to use IOMMU_DOMAIN_DMA[_FQ].

2131 */

2132 if (amd_iommu_snp_en && (type == IOMMU_DOMAIN_IDENTITY))

2133 return NULL;

2134

2135 domain = protection_domain_alloc(type);

2136 if (!domain)

2137 return NULL;

2138

2139 domain->domain.geometry.aperture_start = 0;

2140 domain->domain.geometry.aperture_end = ~0ULL;

2141 domain->domain.geometry.force_aperture = true;

2142

2143 return &domain->domain;

2144 }

2145

2146 static void amd_iommu_domain_free(struct iommu_domain *dom)

2147 {

2148 struct protection_domain *domain;

2149

2150 domain = to_pdomain(dom);

2151

2152 if (domain->dev_cnt > 0)

2153 cleanup_domain(domain);

2154

2155 BUG_ON(domain->dev_cnt != 0);

2156

2157 if (!dom)

2158 return;

2159

2160 if (domain->flags & PD_IOMMUV2_MASK)

2161 free_gcr3_table(domain);

2162

2163 protection_domain_free(domain);

2164 }

2165

2166 static int amd_iommu_attach_device(struct iommu_domain *dom,

2167 struct device *dev)

2168 {

2169 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);

2170 struct protection_domain *domain = to_pdomain(dom);

2171 struct amd_iommu *iommu = rlookup_amd_iommu(dev);

2172 int ret;

2173

2174 /*

2175 * Skip attach device to domain if new domain is same as

2176 * devices current domain

2177 */

2178 if (dev_data->domain == domain)

2179 return 0;

2180

2181 dev_data->defer_attach = false;

2182

2183 if (dev_data->domain)

2184 detach_device(dev);

2185

2186 ret = attach_device(dev, domain);

2187

2188 #ifdef CONFIG_IRQ_REMAP

2189 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) {

2190 if (dom->type == IOMMU_DOMAIN_UNMANAGED)

2191 dev_data->use_vapic = 1;

2192 else

2193 dev_data->use_vapic = 0;

2194 }

2195 #endif

2196

2197 iommu_completion_wait(iommu);

2198

2199 return ret;

2200 }

2201

2202 static void amd_iommu_iotlb_sync_map(struct iommu_domain *dom,

2203 unsigned long iova, size_t size)

2204 {

2205 struct protection_domain *domain = to_pdomain(dom);

2206 struct io_pgtable_ops *ops = &domain->iop.iop.ops;

2207

2208 if (ops->map_pages)

2209 domain_flush_np_cache(domain, iova, size);

2210 }

2211

2212 static int amd_iommu_map_pages(struct iommu_domain *dom, unsigned long iova,

2213 phys_addr_t paddr, size_t pgsize, size_t pgcount,

2214 int iommu_prot, gfp_t gfp, size_t *mapped)

2215 {

2216 struct protection_domain *domain = to_pdomain(dom);

2217 struct io_pgtable_ops *ops = &domain->iop.iop.ops;

2218 int prot = 0;

2219 int ret = -EINVAL;

2220

2221 if ((amd_iommu_pgtable == AMD_IOMMU_V1) &&

2222 (domain->iop.mode == PAGE_MODE_NONE))

2223 return -EINVAL;

2224

2225 if (iommu_prot & IOMMU_READ)

2226 prot |= IOMMU_PROT_IR;

2227 if (iommu_prot & IOMMU_WRITE)

2228 prot |= IOMMU_PROT_IW;

2229

2230 if (ops->map_pages) {

2231 ret = ops->map_pages(ops, iova, paddr, pgsize,

2232 pgcount, prot, gfp, mapped);

2233 }

2234

2235 return ret;

2236 }

2237

2238 static void amd_iommu_iotlb_gather_add_page(struct iommu_domain *domain,

2239 struct iommu_iotlb_gather *gather,

2240 unsigned long iova, size_t size)

2241 {

2242 /*

2243 * AMD's IOMMU can flush as many pages as necessary in a single flush.

2244 * Unless we run in a virtual machine, which can be inferred according

2245 * to whether "non-present cache" is on, it is probably best to prefer

2246 * (potentially) too extensive TLB flushing (i.e., more misses) over

2247 * mutliple TLB flushes (i.e., more flushes). For virtual machines the

2248 * hypervisor needs to synchronize the host IOMMU PTEs with those of

2249 * the guest, and the trade-off is different: unnecessary TLB flushes

2250 * should be avoided.

2251 */

2252 if (amd_iommu_np_cache &&

2253 iommu_iotlb_gather_is_disjoint(gather, iova, size))

2254 iommu_iotlb_sync(domain, gather);

2255

2256 iommu_iotlb_gather_add_range(gather, iova, size);

2257 }

2258

2259 static size_t amd_iommu_unmap_pages(struct iommu_domain *dom, unsigned long iova,

2260 size_t pgsize, size_t pgcount,

2261 struct iommu_iotlb_gather *gather)

2262 {

2263 struct protection_domain *domain = to_pdomain(dom);

2264 struct io_pgtable_ops *ops = &domain->iop.iop.ops;

2265 size_t r;

2266

2267 if ((amd_iommu_pgtable == AMD_IOMMU_V1) &&

2268 (domain->iop.mode == PAGE_MODE_NONE))

2269 return 0;

2270

2271 r = (ops->unmap_pages) ? ops->unmap_pages(ops, iova, pgsize, pgcount, NULL) : 0;

2272

2273 if (r)

2274 amd_iommu_iotlb_gather_add_page(dom, gather, iova, r);

2275

2276 return r;

2277 }

2278

2279 static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,

2280 dma_addr_t iova)

2281 {

2282 struct protection_domain *domain = to_pdomain(dom);

2283 struct io_pgtable_ops *ops = &domain->iop.iop.ops;

2284

2285 return ops->iova_to_phys(ops, iova);

2286 }

2287

2288 static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap)

2289 {

2290 switch (cap) {

2291 case IOMMU_CAP_CACHE_COHERENCY:

2292 return true;

2293 case IOMMU_CAP_NOEXEC:

2294 return false;

2295 case IOMMU_CAP_PRE_BOOT_PROTECTION:

2296 return amdr_ivrs_remap_support;

2297 case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:

2298 return true;

2299 default:

2300 break;

2301 }

2302

2303 return false;

2304 }

2305

2306 static void amd_iommu_get_resv_regions(struct device *dev,

2307 struct list_head *head)

2308 {

2309 struct iommu_resv_region *region;

2310 struct unity_map_entry *entry;

2311 struct amd_iommu *iommu;

2312 struct amd_iommu_pci_seg *pci_seg;

2313 int devid, sbdf;

2314

2315 sbdf = get_device_sbdf_id(dev);

2316 if (sbdf < 0)

2317 return;

2318

2319 devid = PCI_SBDF_TO_DEVID(sbdf);

2320 iommu = rlookup_amd_iommu(dev);

2321 if (!iommu)

2322 return;

2323 pci_seg = iommu->pci_seg;

2324

2325 list_for_each_entry(entry, &pci_seg->unity_map, list) {

2326 int type, prot = 0;

2327 size_t length;

2328

2329 if (devid < entry->devid_start || devid > entry->devid_end)

2330 continue;

2331

2332 type = IOMMU_RESV_DIRECT;

2333 length = entry->address_end - entry->address_start;

2334 if (entry->prot & IOMMU_PROT_IR)

2335 prot |= IOMMU_READ;

2336 if (entry->prot & IOMMU_PROT_IW)

2337 prot |= IOMMU_WRITE;

2338 if (entry->prot & IOMMU_UNITY_MAP_FLAG_EXCL_RANGE)

2339 /* Exclusion range */

2340 type = IOMMU_RESV_RESERVED;

2341

2342 region = iommu_alloc_resv_region(entry->address_start,

2343 length, prot, type,

2344 GFP_KERNEL);

2345 if (!region) {

2346 dev_err(dev, "Out of memory allocating dm-regions\n");

2347 return;

2348 }

2349 list_add_tail(&region->list, head);

2350 }

2351

2352 region = iommu_alloc_resv_region(MSI_RANGE_START,

2353 MSI_RANGE_END - MSI_RANGE_START + 1,

2354 0, IOMMU_RESV_MSI, GFP_KERNEL);

2355 if (!region)

2356 return;

2357 list_add_tail(&region->list, head);

2358

2359 region = iommu_alloc_resv_region(HT_RANGE_START,

2360 HT_RANGE_END - HT_RANGE_START + 1,

2361 0, IOMMU_RESV_RESERVED, GFP_KERNEL);

2362 if (!region)

2363 return;

2364 list_add_tail(&region->list, head);

2365 }

2366

2367 bool amd_iommu_is_attach_deferred(struct device *dev)

2368 {

2369 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);

2370

2371 return dev_data->defer_attach;

2372 }

2373 EXPORT_SYMBOL_GPL(amd_iommu_is_attach_deferred);

2374

2375 static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain)

2376 {

2377 struct protection_domain *dom = to_pdomain(domain);

2378 unsigned long flags;

2379

2380 spin_lock_irqsave(&dom->lock, flags);

2381 amd_iommu_domain_flush_tlb_pde(dom);

2382 amd_iommu_domain_flush_complete(dom);

2383 spin_unlock_irqrestore(&dom->lock, flags);

2384 }

2385

2386 static void amd_iommu_iotlb_sync(struct iommu_domain *domain,

2387 struct iommu_iotlb_gather *gather)

2388 {

2389 struct protection_domain *dom = to_pdomain(domain);

2390 unsigned long flags;

2391

2392 spin_lock_irqsave(&dom->lock, flags);

2393 domain_flush_pages(dom, gather->start, gather->end - gather->start, 1);

2394 amd_iommu_domain_flush_complete(dom);

2395 spin_unlock_irqrestore(&dom->lock, flags);

2396 }

2397

2398 static int amd_iommu_def_domain_type(struct device *dev)

2399 {

2400 struct iommu_dev_data *dev_data;

2401

2402 dev_data = dev_iommu_priv_get(dev);

2403 if (!dev_data)

2404 return 0;

2405

2406 /*

2407 * Do not identity map IOMMUv2 capable devices when:

2408 * - memory encryption is active, because some of those devices

2409 * (AMD GPUs) don't have the encryption bit in their DMA-mask

2410 * and require remapping.

2411 * - SNP is enabled, because it prohibits DTE[Mode]=0.

2412 */

2413 if (dev_data->iommu_v2 &&

2414 !cc_platform_has(CC_ATTR_MEM_ENCRYPT) &&

2415 !amd_iommu_snp_en) {

2416 return IOMMU_DOMAIN_IDENTITY;

2417 }

2418

2419 return 0;

2420 }

2421

2422 static bool amd_iommu_enforce_cache_coherency(struct iommu_domain *domain)

2423 {

2424 /* IOMMU_PTE_FC is always set */

2425 return true;

2426 }

2427

2428 const struct iommu_ops amd_iommu_ops = {

2429 .capable = amd_iommu_capable,

2430 .domain_alloc = amd_iommu_domain_alloc,

2431 .probe_device = amd_iommu_probe_device,

2432 .release_device = amd_iommu_release_device,

2433 .probe_finalize = amd_iommu_probe_finalize,

2434 .device_group = amd_iommu_device_group,

2435 .get_resv_regions = amd_iommu_get_resv_regions,

2436 .is_attach_deferred = amd_iommu_is_attach_deferred,

2437 .pgsize_bitmap = AMD_IOMMU_PGSIZES,

2438 .def_domain_type = amd_iommu_def_domain_type,

2439 .default_domain_ops = &(const struct iommu_domain_ops) {

2440 .attach_dev = amd_iommu_attach_device,

2441 .map_pages = amd_iommu_map_pages,

2442 .unmap_pages = amd_iommu_unmap_pages,

2443 .iotlb_sync_map = amd_iommu_iotlb_sync_map,

2444 .iova_to_phys = amd_iommu_iova_to_phys,

2445 .flush_iotlb_all = amd_iommu_flush_iotlb_all,

2446 .iotlb_sync = amd_iommu_iotlb_sync,

2447 .free = amd_iommu_domain_free,

2448 .enforce_cache_coherency = amd_iommu_enforce_cache_coherency,

2449 }

2450 };

2451

2452 /*****************************************************************************

2453 *

2454 * The next functions do a basic initialization of IOMMU for pass through

2455 * mode

2456 *

2457 * In passthrough mode the IOMMU is initialized and enabled but not used for

2458 * DMA-API translation.

2459 *

2460 *****************************************************************************/

2461

2462 /* IOMMUv2 specific functions */

2463 int amd_iommu_register_ppr_notifier(struct notifier_block *nb)

2464 {

2465 return atomic_notifier_chain_register(&ppr_notifier, nb);

2466 }

2467 EXPORT_SYMBOL(amd_iommu_register_ppr_notifier);

2468

2469 int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb)

2470 {

2471 return atomic_notifier_chain_unregister(&ppr_notifier, nb);

2472 }

2473 EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);

2474

2475 void amd_iommu_domain_direct_map(struct iommu_domain *dom)

2476 {

2477 struct protection_domain *domain = to_pdomain(dom);

2478 unsigned long flags;

2479

2480 spin_lock_irqsave(&domain->lock, flags);

2481

2482 if (domain->iop.pgtbl_cfg.tlb)

2483 free_io_pgtable_ops(&domain->iop.iop.ops);

2484

2485 spin_unlock_irqrestore(&domain->lock, flags);

2486 }

2487 EXPORT_SYMBOL(amd_iommu_domain_direct_map);

2488

2489 /* Note: This function expects iommu_domain->lock to be held prior calling the function. */

2490 static int domain_enable_v2(struct protection_domain *domain, int pasids)

2491 {

2492 int levels;

2493

2494 /* Number of GCR3 table levels required */

2495 for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9)

2496 levels += 1;

2497

2498 if (levels > amd_iommu_max_glx_val)

2499 return -EINVAL;

2500

2501 domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC);

2502 if (domain->gcr3_tbl == NULL)

2503 return -ENOMEM;

2504

2505 domain->glx = levels;

2506 domain->flags |= PD_IOMMUV2_MASK;

2507

2508 amd_iommu_domain_update(domain);

2509

2510 return 0;

2511 }

2512

2513 int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)

2514 {

2515 struct protection_domain *pdom = to_pdomain(dom);

2516 unsigned long flags;

2517 int ret;

2518

2519 spin_lock_irqsave(&pdom->lock, flags);

2520

2521 /*

2522 * Save us all sanity checks whether devices already in the

2523 * domain support IOMMUv2. Just force that the domain has no

2524 * devices attached when it is switched into IOMMUv2 mode.

2525 */

2526 ret = -EBUSY;

2527 if (pdom->dev_cnt > 0 || pdom->flags & PD_IOMMUV2_MASK)

2528 goto out;

2529

2530 if (!pdom->gcr3_tbl)

2531 ret = domain_enable_v2(pdom, pasids);

2532

2533 out:

2534 spin_unlock_irqrestore(&pdom->lock, flags);

2535 return ret;

2536 }

2537 EXPORT_SYMBOL(amd_iommu_domain_enable_v2);

2538

2539 static int __flush_pasid(struct protection_domain *domain, u32 pasid,

2540 u64 address, bool size)

2541 {

2542 struct iommu_dev_data *dev_data;

2543 struct iommu_cmd cmd;

2544 int i, ret;

2545

2546 if (!(domain->flags & PD_IOMMUV2_MASK))

2547 return -EINVAL;

2548

2549 build_inv_iommu_pasid(&cmd, domain->id, pasid, address, size);

2550

2551 /*

2552 * IOMMU TLB needs to be flushed before Device TLB to

2553 * prevent device TLB refill from IOMMU TLB

2554 */

2555 for (i = 0; i < amd_iommu_get_num_iommus(); ++i) {

2556 if (domain->dev_iommu[i] == 0)

2557 continue;

2558

2559 ret = iommu_queue_command(amd_iommus[i], &cmd);

2560 if (ret != 0)

2561 goto out;

2562 }

2563

2564 /* Wait until IOMMU TLB flushes are complete */

2565 amd_iommu_domain_flush_complete(domain);

2566

2567 /* Now flush device TLBs */

2568 list_for_each_entry(dev_data, &domain->dev_list, list) {

2569 struct amd_iommu *iommu;

2570 int qdep;

2571

2572 /*

2573 There might be non-IOMMUv2 capable devices in an IOMMUv2

2574 * domain.

2575 */

2576 if (!dev_data->ats.enabled)

2577 continue;

2578

2579 qdep = dev_data->ats.qdep;

2580 iommu = rlookup_amd_iommu(dev_data->dev);

2581 if (!iommu)

2582 continue;

2583 build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid,

2584 qdep, address, size);

2585

2586 ret = iommu_queue_command(iommu, &cmd);

2587 if (ret != 0)

2588 goto out;

2589 }

2590

2591 /* Wait until all device TLBs are flushed */

2592 amd_iommu_domain_flush_complete(domain);

2593

2594 ret = 0;

2595

2596 out:

2597

2598 return ret;

2599 }

2600

2601 static int __amd_iommu_flush_page(struct protection_domain *domain, u32 pasid,

2602 u64 address)

2603 {

2604 return __flush_pasid(domain, pasid, address, false);

2605 }

2606

2607 int amd_iommu_flush_page(struct iommu_domain *dom, u32 pasid,

2608 u64 address)

2609 {

2610 struct protection_domain *domain = to_pdomain(dom);

2611 unsigned long flags;

2612 int ret;

2613

2614 spin_lock_irqsave(&domain->lock, flags);

2615 ret = __amd_iommu_flush_page(domain, pasid, address);

2616 spin_unlock_irqrestore(&domain->lock, flags);

2617

2618 return ret;

2619 }

2620 EXPORT_SYMBOL(amd_iommu_flush_page);

2621

2622 static int __amd_iommu_flush_tlb(struct protection_domain *domain, u32 pasid)

2623 {

2624 return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,

2625 true);

2626 }

2627

2628 int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid)

2629 {

2630 struct protection_domain *domain = to_pdomain(dom);

2631 unsigned long flags;

2632 int ret;

2633

2634 spin_lock_irqsave(&domain->lock, flags);

2635 ret = __amd_iommu_flush_tlb(domain, pasid);

2636 spin_unlock_irqrestore(&domain->lock, flags);

2637

2638 return ret;

2639 }

2640 EXPORT_SYMBOL(amd_iommu_flush_tlb);

2641

2642 static u64 *__get_gcr3_pte(u64 *root, int level, u32 pasid, bool alloc)

2643 {

2644 int index;

2645 u64 *pte;

2646

2647 while (true) {

2648

2649 index = (pasid >> (9 * level)) & 0x1ff;

2650 pte = &root[index];

2651

2652 if (level == 0)

2653 break;

2654

2655 if (!(*pte & GCR3_VALID)) {

2656 if (!alloc)

2657 return NULL;

2658

2659 root = (void *)get_zeroed_page(GFP_ATOMIC);

2660 if (root == NULL)

2661 return NULL;

2662

2663 *pte = iommu_virt_to_phys(root) | GCR3_VALID;

2664 }

2665

2666 root = iommu_phys_to_virt(*pte & PAGE_MASK);

2667

2668 level -= 1;

2669 }

2670

2671 return pte;

2672 }

2673

2674 static int __set_gcr3(struct protection_domain *domain, u32 pasid,

2675 unsigned long cr3)

2676 {

2677 u64 *pte;

2678

2679 if (domain->iop.mode != PAGE_MODE_NONE)

2680 return -EINVAL;

2681

2682 pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);

2683 if (pte == NULL)

2684 return -ENOMEM;

2685

2686 *pte = (cr3 & PAGE_MASK) | GCR3_VALID;

2687

2688 return __amd_iommu_flush_tlb(domain, pasid);

2689 }

2690

2691 static int __clear_gcr3(struct protection_domain *domain, u32 pasid)

2692 {

2693 u64 *pte;

2694

2695 if (domain->iop.mode != PAGE_MODE_NONE)

2696 return -EINVAL;

2697

2698 pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);

2699 if (pte == NULL)

2700 return 0;

2701

2702 *pte = 0;

2703

2704 return __amd_iommu_flush_tlb(domain, pasid);

2705 }

2706

2707 int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, u32 pasid,

2708 unsigned long cr3)

2709 {

2710 struct protection_domain *domain = to_pdomain(dom);

2711 unsigned long flags;

2712 int ret;

2713

2714 spin_lock_irqsave(&domain->lock, flags);

2715 ret = __set_gcr3(domain, pasid, cr3);

2716 spin_unlock_irqrestore(&domain->lock, flags);

2717

2718 return ret;

2719 }

2720 EXPORT_SYMBOL(amd_iommu_domain_set_gcr3);

2721

2722 int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, u32 pasid)

2723 {

2724 struct protection_domain *domain = to_pdomain(dom);

2725 unsigned long flags;

2726 int ret;

2727

2728 spin_lock_irqsave(&domain->lock, flags);

2729 ret = __clear_gcr3(domain, pasid);

2730 spin_unlock_irqrestore(&domain->lock, flags);

2731

2732 return ret;

2733 }

2734 EXPORT_SYMBOL(amd_iommu_domain_clear_gcr3);

2735

2736 int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid,

2737 int status, int tag)

2738 {

2739 struct iommu_dev_data *dev_data;

2740 struct amd_iommu *iommu;

2741 struct iommu_cmd cmd;

2742

2743 dev_data = dev_iommu_priv_get(&pdev->dev);

2744 iommu = rlookup_amd_iommu(&pdev->dev);

2745 if (!iommu)

2746 return -ENODEV;

2747

2748 build_complete_ppr(&cmd, dev_data->devid, pasid, status,

2749 tag, dev_data->pri_tlp);

2750

2751 return iommu_queue_command(iommu, &cmd);

2752 }

2753 EXPORT_SYMBOL(amd_iommu_complete_ppr);

2754

2755 int amd_iommu_device_info(struct pci_dev *pdev,

2756 struct amd_iommu_device_info *info)

2757 {

2758 int max_pasids;

2759 int pos;

2760

2761 if (pdev == NULL || info == NULL)

2762 return -EINVAL;

2763

2764 if (!amd_iommu_v2_supported())

2765 return -EINVAL;

2766

2767 memset(info, 0, sizeof(*info));

2768

2769 if (pci_ats_supported(pdev))

2770 info->flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;

2771

2772 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);

2773 if (pos)

2774 info->flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP;

2775

2776 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);

2777 if (pos) {

2778 int features;

2779

2780 max_pasids = 1 << (9 * (amd_iommu_max_glx_val + 1));

2781 max_pasids = min(max_pasids, (1 << 20));

2782

2783 info->flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP;

2784 info->max_pasids = min(pci_max_pasids(pdev), max_pasids);

2785

2786 features = pci_pasid_features(pdev);

2787 if (features & PCI_PASID_CAP_EXEC)

2788 info->flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP;

2789 if (features & PCI_PASID_CAP_PRIV)

2790 info->flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP;

2791 }

2792

2793 return 0;

2794 }

2795 EXPORT_SYMBOL(amd_iommu_device_info);

2796

2797 #ifdef CONFIG_IRQ_REMAP

2798

2799 /*****************************************************************************

2800 *

2801 * Interrupt Remapping Implementation

2802 *

2803 *****************************************************************************/

2804

2805 static struct irq_chip amd_ir_chip;

2806 static DEFINE_SPINLOCK(iommu_table_lock);

2807

2808 static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid)

2809 {

2810 int ret;

2811 u64 data;

2812 unsigned long flags;

2813 struct iommu_cmd cmd, cmd2;

2814

2815 if (iommu->irtcachedis_enabled)

2816 return;

2817

2818 build_inv_irt(&cmd, devid);

2819 data = atomic64_add_return(1, &iommu->cmd_sem_val);

2820 build_completion_wait(&cmd2, iommu, data);

2821

2822 raw_spin_lock_irqsave(&iommu->lock, flags);

2823 ret = __iommu_queue_command_sync(iommu, &cmd, true);

2824 if (ret)

2825 goto out;

2826 ret = __iommu_queue_command_sync(iommu, &cmd2, false);

2827 if (ret)

2828 goto out;

2829 wait_on_sem(iommu, data);

2830 out:

2831 raw_spin_unlock_irqrestore(&iommu->lock, flags);

2832 }

2833

2834 static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid,

2835 struct irq_remap_table *table)

2836 {

2837 u64 dte;

2838 struct dev_table_entry *dev_table = get_dev_table(iommu);

2839

2840 dte = dev_table[devid].data[2];

2841 dte &= ~DTE_IRQ_PHYS_ADDR_MASK;

2842 dte |= iommu_virt_to_phys(table->table);

2843 dte |= DTE_IRQ_REMAP_INTCTL;

2844 dte |= DTE_INTTABLEN;

2845 dte |= DTE_IRQ_REMAP_ENABLE;

2846

2847 dev_table[devid].data[2] = dte;

2848 }

2849

2850 static struct irq_remap_table *get_irq_table(struct amd_iommu *iommu, u16 devid)

2851 {

2852 struct irq_remap_table *table;

2853 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

2854

2855 if (WARN_ONCE(!pci_seg->rlookup_table[devid],

2856 "%s: no iommu for devid %x:%x\n",

2857 __func__, pci_seg->id, devid))

2858 return NULL;

2859

2860 table = pci_seg->irq_lookup_table[devid];

2861 if (WARN_ONCE(!table, "%s: no table for devid %x:%x\n",

2862 __func__, pci_seg->id, devid))

2863 return NULL;

2864

2865 return table;

2866 }

2867

2868 static struct irq_remap_table *__alloc_irq_table(void)

2869 {

2870 struct irq_remap_table *table;

2871

2872 table = kzalloc(sizeof(*table), GFP_KERNEL);

2873 if (!table)

2874 return NULL;

2875

2876 table->table = kmem_cache_alloc(amd_iommu_irq_cache, GFP_KERNEL);

2877 if (!table->table) {

2878 kfree(table);

2879 return NULL;

2880 }

2881 raw_spin_lock_init(&table->lock);

2882

2883 if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir))

2884 memset(table->table, 0,

2885 MAX_IRQS_PER_TABLE * sizeof(u32));

2886 else

2887 memset(table->table, 0,

2888 (MAX_IRQS_PER_TABLE * (sizeof(u64) * 2)));

2889 return table;

2890 }

2891

2892 static void set_remap_table_entry(struct amd_iommu *iommu, u16 devid,

2893 struct irq_remap_table *table)

2894 {

2895 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;

2896

2897 pci_seg->irq_lookup_table[devid] = table;

2898 set_dte_irq_entry(iommu, devid, table);

2899 iommu_flush_dte(iommu, devid);

2900 }

2901

2902 static int set_remap_table_entry_alias(struct pci_dev *pdev, u16 alias,

2903 void *data)

2904 {

2905 struct irq_remap_table *table = data;

2906 struct amd_iommu_pci_seg *pci_seg;

2907 struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev);

2908

2909 if (!iommu)

2910 return -EINVAL;

2911

2912 pci_seg = iommu->pci_seg;

2913 pci_seg->irq_lookup_table[alias] = table;

2914 set_dte_irq_entry(iommu, alias, table);

2915 iommu_flush_dte(pci_seg->rlookup_table[alias], alias);

2916

2917 return 0;

2918 }

2919

2920 static struct irq_remap_table *alloc_irq_table(struct amd_iommu *iommu,

2921 u16 devid, struct pci_dev *pdev)

2922 {

2923 struct irq_remap_table *table = NULL;

2924 struct irq_remap_table *new_table = NULL;

2925 struct amd_iommu_pci_seg *pci_seg;

2926 unsigned long flags;

2927 u16 alias;

2928

2929 spin_lock_irqsave(&iommu_table_lock, flags);

2930

2931 pci_seg = iommu->pci_seg;

2932 table = pci_seg->irq_lookup_table[devid];

2933 if (table)

2934 goto out_unlock;

2935

2936 alias = pci_seg->alias_table[devid];

2937 table = pci_seg->irq_lookup_table[alias];

2938 if (table) {

2939 set_remap_table_entry(iommu, devid, table);

2940 goto out_wait;

2941 }

2942 spin_unlock_irqrestore(&iommu_table_lock, flags);

2943

2944 /* Nothing there yet, allocate new irq remapping table */

2945 new_table = __alloc_irq_table();

2946 if (!new_table)

2947 return NULL;

2948

2949 spin_lock_irqsave(&iommu_table_lock, flags);

2950

2951 table = pci_seg->irq_lookup_table[devid];

2952 if (table)

2953 goto out_unlock;

2954

2955 table = pci_seg->irq_lookup_table[alias];

2956 if (table) {

2957 set_remap_table_entry(iommu, devid, table);

2958 goto out_wait;

2959 }

2960

2961 table = new_table;

2962 new_table = NULL;

2963

2964 if (pdev)

2965 pci_for_each_dma_alias(pdev, set_remap_table_entry_alias,

2966 table);

2967 else

2968 set_remap_table_entry(iommu, devid, table);

2969

2970 if (devid != alias)

2971 set_remap_table_entry(iommu, alias, table);

2972

2973 out_wait:

2974 iommu_completion_wait(iommu);

2975

2976 out_unlock:

2977 spin_unlock_irqrestore(&iommu_table_lock, flags);

2978

2979 if (new_table) {

2980 kmem_cache_free(amd_iommu_irq_cache, new_table->table);

2981 kfree(new_table);

2982 }

2983 return table;

2984 }

2985

2986 static int alloc_irq_index(struct amd_iommu *iommu, u16 devid, int count,

2987 bool align, struct pci_dev *pdev)

2988 {

2989 struct irq_remap_table *table;

2990 int index, c, alignment = 1;

2991 unsigned long flags;

2992

2993 table = alloc_irq_table(iommu, devid, pdev);

2994 if (!table)

2995 return -ENODEV;

2996

2997 if (align)

2998 alignment = roundup_pow_of_two(count);

2999

3000 raw_spin_lock_irqsave(&table->lock, flags);

3001

3002 /* Scan table for free entries */

3003 for (index = ALIGN(table->min_index, alignment), c = 0;

3004 index < MAX_IRQS_PER_TABLE;) {

3005 if (!iommu->irte_ops->is_allocated(table, index)) {

3006 c += 1;

3007 } else {

3008 c = 0;

3009 index = ALIGN(index + 1, alignment);

3010 continue;

3011 }

3012

3013 if (c == count) {

3014 for (; c != 0; --c)

3015 iommu->irte_ops->set_allocated(table, index - c + 1);

3016

3017 index -= count - 1;

3018 goto out;

3019 }

3020

3021 index++;

3022 }

3023

3024 index = -ENOSPC;

3025

3026 out:

3027 raw_spin_unlock_irqrestore(&table->lock, flags);

3028

3029 return index;

3030 }

3031

3032 static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,

3033 struct irte_ga *irte)

3034 {

3035 bool ret;

3036 struct irq_remap_table *table;

3037 unsigned long flags;

3038 struct irte_ga *entry;

3039

3040 table = get_irq_table(iommu, devid);

3041 if (!table)

3042 return -ENOMEM;

3043

3044 raw_spin_lock_irqsave(&table->lock, flags);

3045

3046 entry = (struct irte_ga *)table->table;

3047 entry = &entry[index];

3048

3049 ret = cmpxchg_double(&entry->lo.val, &entry->hi.val,

3050 entry->lo.val, entry->hi.val,

3051 irte->lo.val, irte->hi.val);

3052 /*

3053 * We use cmpxchg16 to atomically update the 128-bit IRTE,

3054 * and it cannot be updated by the hardware or other processors

3055 * behind us, so the return value of cmpxchg16 should be the

3056 * same as the old value.

3057 */

3058 WARN_ON(!ret);

3059

3060 raw_spin_unlock_irqrestore(&table->lock, flags);

3061

3062 iommu_flush_irt_and_complete(iommu, devid);

3063

3064 return 0;

3065 }

3066

3067 static int modify_irte(struct amd_iommu *iommu,

3068 u16 devid, int index, union irte *irte)

3069 {

3070 struct irq_remap_table *table;

3071 unsigned long flags;

3072

3073 table = get_irq_table(iommu, devid);

3074 if (!table)

3075 return -ENOMEM;

3076

3077 raw_spin_lock_irqsave(&table->lock, flags);

3078 table->table[index] = irte->val;

3079 raw_spin_unlock_irqrestore(&table->lock, flags);

3080

3081 iommu_flush_irt_and_complete(iommu, devid);

3082

3083 return 0;

3084 }

3085

3086 static void free_irte(struct amd_iommu *iommu, u16 devid, int index)

3087 {

3088 struct irq_remap_table *table;

3089 unsigned long flags;

3090

3091 table = get_irq_table(iommu, devid);

3092 if (!table)

3093 return;

3094

3095 raw_spin_lock_irqsave(&table->lock, flags);

3096 iommu->irte_ops->clear_allocated(table, index);

3097 raw_spin_unlock_irqrestore(&table->lock, flags);

3098

3099 iommu_flush_irt_and_complete(iommu, devid);

3100 }

3101

3102 static void irte_prepare(void *entry,

3103 u32 delivery_mode, bool dest_mode,

3104 u8 vector, u32 dest_apicid, int devid)

3105 {

3106 union irte *irte = (union irte *) entry;

3107

3108 irte->val = 0;

3109 irte->fields.vector = vector;

3110 irte->fields.int_type = delivery_mode;

3111 irte->fields.destination = dest_apicid;

3112 irte->fields.dm = dest_mode;

3113 irte->fields.valid = 1;

3114 }

3115

3116 static void irte_ga_prepare(void *entry,

3117 u32 delivery_mode, bool dest_mode,

3118 u8 vector, u32 dest_apicid, int devid)

3119 {

3120 struct irte_ga *irte = (struct irte_ga *) entry;

3121

3122 irte->lo.val = 0;

3123 irte->hi.val = 0;

3124 irte->lo.fields_remap.int_type = delivery_mode;

3125 irte->lo.fields_remap.dm = dest_mode;

3126 irte->hi.fields.vector = vector;

3127 irte->lo.fields_remap.destination = APICID_TO_IRTE_DEST_LO(dest_apicid);

3128 irte->hi.fields.destination = APICID_TO_IRTE_DEST_HI(dest_apicid);

3129 irte->lo.fields_remap.valid = 1;

3130 }

3131

3132 static void irte_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)

3133 {

3134 union irte *irte = (union irte *) entry;

3135

3136 irte->fields.valid = 1;

3137 modify_irte(iommu, devid, index, irte);

3138 }

3139

3140 static void irte_ga_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)

3141 {

3142 struct irte_ga *irte = (struct irte_ga *) entry;

3143

3144 irte->lo.fields_remap.valid = 1;

3145 modify_irte_ga(iommu, devid, index, irte);

3146 }

3147

3148 static void irte_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)

3149 {

3150 union irte *irte = (union irte *) entry;

3151

3152 irte->fields.valid = 0;

3153 modify_irte(iommu, devid, index, irte);

3154 }

3155

3156 static void irte_ga_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)

3157 {

3158 struct irte_ga *irte = (struct irte_ga *) entry;

3159

3160 irte->lo.fields_remap.valid = 0;

3161 modify_irte_ga(iommu, devid, index, irte);

3162 }

3163

3164 static void irte_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,

3165 u8 vector, u32 dest_apicid)

3166 {

3167 union irte *irte = (union irte *) entry;

3168

3169 irte->fields.vector = vector;

3170 irte->fields.destination = dest_apicid;

3171 modify_irte(iommu, devid, index, irte);

3172 }

3173

3174 static void irte_ga_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,

3175 u8 vector, u32 dest_apicid)

3176 {

3177 struct irte_ga *irte = (struct irte_ga *) entry;

3178

3179 if (!irte->lo.fields_remap.guest_mode) {

3180 irte->hi.fields.vector = vector;

3181 irte->lo.fields_remap.destination =

3182 APICID_TO_IRTE_DEST_LO(dest_apicid);

3183 irte->hi.fields.destination =

3184 APICID_TO_IRTE_DEST_HI(dest_apicid);

3185 modify_irte_ga(iommu, devid, index, irte);

3186 }

3187 }

3188

3189 #define IRTE_ALLOCATED (~1U)

3190 static void irte_set_allocated(struct irq_remap_table *table, int index)

3191 {

3192 table->table[index] = IRTE_ALLOCATED;

3193 }

3194

3195 static void irte_ga_set_allocated(struct irq_remap_table *table, int index)

3196 {

3197 struct irte_ga *ptr = (struct irte_ga *)table->table;

3198 struct irte_ga *irte = &ptr[index];

3199

3200 memset(&irte->lo.val, 0, sizeof(u64));

3201 memset(&irte->hi.val, 0, sizeof(u64));

3202 irte->hi.fields.vector = 0xff;

3203 }

3204

3205 static bool irte_is_allocated(struct irq_remap_table *table, int index)

3206 {

3207 union irte *ptr = (union irte *)table->table;

3208 union irte *irte = &ptr[index];

3209

3210 return irte->val != 0;

3211 }

3212

3213 static bool irte_ga_is_allocated(struct irq_remap_table *table, int index)

3214 {

3215 struct irte_ga *ptr = (struct irte_ga *)table->table;

3216 struct irte_ga *irte = &ptr[index];

3217

3218 return irte->hi.fields.vector != 0;

3219 }

3220

3221 static void irte_clear_allocated(struct irq_remap_table *table, int index)

3222 {

3223 table->table[index] = 0;

3224 }

3225

3226 static void irte_ga_clear_allocated(struct irq_remap_table *table, int index)

3227 {

3228 struct irte_ga *ptr = (struct irte_ga *)table->table;

3229 struct irte_ga *irte = &ptr[index];

3230

3231 memset(&irte->lo.val, 0, sizeof(u64));

3232 memset(&irte->hi.val, 0, sizeof(u64));

3233 }

3234

3235 static int get_devid(struct irq_alloc_info *info)

3236 {

3237 switch (info->type) {

3238 case X86_IRQ_ALLOC_TYPE_IOAPIC:

3239 return get_ioapic_devid(info->devid);

3240 case X86_IRQ_ALLOC_TYPE_HPET:

3241 return get_hpet_devid(info->devid);

3242 case X86_IRQ_ALLOC_TYPE_PCI_MSI:

3243 case X86_IRQ_ALLOC_TYPE_PCI_MSIX:

3244 return get_device_sbdf_id(msi_desc_to_dev(info->desc));

3245 default:

3246 WARN_ON_ONCE(1);

3247 return -1;

3248 }

3249 }

3250

3251 struct irq_remap_ops amd_iommu_irq_ops = {

3252 .prepare = amd_iommu_prepare,

3253 .enable = amd_iommu_enable,

3254 .disable = amd_iommu_disable,

3255 .reenable = amd_iommu_reenable,

3256 .enable_faulting = amd_iommu_enable_faulting,

3257 };

3258

3259 static void fill_msi_msg(struct msi_msg *msg, u32 index)

3260 {

3261 msg->data = index;

3262 msg->address_lo = 0;

3263 msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW;

3264 msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH;

3265 }

3266

3267 static void irq_remapping_prepare_irte(struct amd_ir_data *data,

3268 struct irq_cfg *irq_cfg,

3269 struct irq_alloc_info *info,

3270 int devid, int index, int sub_handle)

3271 {

3272 struct irq_2_irte *irte_info = &data->irq_2_irte;

3273 struct amd_iommu *iommu = data->iommu;

3274

3275 if (!iommu)

3276 return;

3277

3278 data->irq_2_irte.devid = devid;

3279 data->irq_2_irte.index = index + sub_handle;

3280 iommu->irte_ops->prepare(data->entry, apic->delivery_mode,

3281 apic->dest_mode_logical, irq_cfg->vector,

3282 irq_cfg->dest_apicid, devid);

3283

3284 switch (info->type) {

3285 case X86_IRQ_ALLOC_TYPE_IOAPIC:

3286 case X86_IRQ_ALLOC_TYPE_HPET:

3287 case X86_IRQ_ALLOC_TYPE_PCI_MSI:

3288 case X86_IRQ_ALLOC_TYPE_PCI_MSIX:

3289 fill_msi_msg(&data->msi_entry, irte_info->index);

3290 break;

3291

3292 default:

3293 BUG_ON(1);

3294 break;

3295 }

3296 }

3297

3298 struct amd_irte_ops irte_32_ops = {

3299 .prepare = irte_prepare,

3300 .activate = irte_activate,

3301 .deactivate = irte_deactivate,

3302 .set_affinity = irte_set_affinity,

3303 .set_allocated = irte_set_allocated,

3304 .is_allocated = irte_is_allocated,

3305 .clear_allocated = irte_clear_allocated,

3306 };

3307

3308 struct amd_irte_ops irte_128_ops = {

3309 .prepare = irte_ga_prepare,

3310 .activate = irte_ga_activate,

3311 .deactivate = irte_ga_deactivate,

3312 .set_affinity = irte_ga_set_affinity,

3313 .set_allocated = irte_ga_set_allocated,

3314 .is_allocated = irte_ga_is_allocated,

3315 .clear_allocated = irte_ga_clear_allocated,

3316 };

3317

3318 static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,

3319 unsigned int nr_irqs, void *arg)

3320 {

3321 struct irq_alloc_info *info = arg;

3322 struct irq_data *irq_data;

3323 struct amd_ir_data *data = NULL;

3324 struct amd_iommu *iommu;

3325 struct irq_cfg *cfg;

3326 int i, ret, devid, seg, sbdf;

3327 int index;

3328

3329 if (!info)

3330 return -EINVAL;

3331 if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_PCI_MSI)

3332 return -EINVAL;

3333

3334 sbdf = get_devid(info);

3335 if (sbdf < 0)

3336 return -EINVAL;

3337

3338 seg = PCI_SBDF_TO_SEGID(sbdf);

3339 devid = PCI_SBDF_TO_DEVID(sbdf);

3340 iommu = __rlookup_amd_iommu(seg, devid);

3341 if (!iommu)

3342 return -EINVAL;

3343

3344 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);

3345 if (ret < 0)

3346 return ret;

3347

3348 if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) {

3349 struct irq_remap_table *table;

3350

3351 table = alloc_irq_table(iommu, devid, NULL);

3352 if (table) {

3353 if (!table->min_index) {

3354 /*

3355 * Keep the first 32 indexes free for IOAPIC

3356 * interrupts.

3357 */

3358 table->min_index = 32;

3359 for (i = 0; i < 32; ++i)

3360 iommu->irte_ops->set_allocated(table, i);

3361 }

3362 WARN_ON(table->min_index != 32);

3363 index = info->ioapic.pin;

3364 } else {

3365 index = -ENOMEM;

3366 }

3367 } else if (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI ||

3368 info->type == X86_IRQ_ALLOC_TYPE_PCI_MSIX) {

3369 bool align = (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI);

3370

3371 index = alloc_irq_index(iommu, devid, nr_irqs, align,

3372 msi_desc_to_pci_dev(info->desc));

3373 } else {

3374 index = alloc_irq_index(iommu, devid, nr_irqs, false, NULL);

3375 }

3376

3377 if (index < 0) {

3378 pr_warn("Failed to allocate IRTE\n");

3379 ret = index;

3380 goto out_free_parent;

3381 }

3382

3383 for (i = 0; i < nr_irqs; i++) {

3384 irq_data = irq_domain_get_irq_data(domain, virq + i);

3385 cfg = irq_data ? irqd_cfg(irq_data) : NULL;

3386 if (!cfg) {

3387 ret = -EINVAL;

3388 goto out_free_data;

3389 }

3390

3391 ret = -ENOMEM;

3392 data = kzalloc(sizeof(*data), GFP_KERNEL);

3393 if (!data)

3394 goto out_free_data;

3395

3396 if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir))

3397 data->entry = kzalloc(sizeof(union irte), GFP_KERNEL);

3398 else

3399 data->entry = kzalloc(sizeof(struct irte_ga),

3400 GFP_KERNEL);

3401 if (!data->entry) {

3402 kfree(data);

3403 goto out_free_data;

3404 }

3405

3406 data->iommu = iommu;

3407 irq_data->hwirq = (devid << 16) + i;

3408 irq_data->chip_data = data;

3409 irq_data->chip = &amd_ir_chip;

3410 irq_remapping_prepare_irte(data, cfg, info, devid, index, i);

3411 irq_set_status_flags(virq + i, IRQ_MOVE_PCNTXT);

3412 }

3413

3414 return 0;

3415

3416 out_free_data:

3417 for (i--; i >= 0; i--) {

3418 irq_data = irq_domain_get_irq_data(domain, virq + i);

3419 if (irq_data)

3420 kfree(irq_data->chip_data);

3421 }

3422 for (i = 0; i < nr_irqs; i++)

3423 free_irte(iommu, devid, index + i);

3424 out_free_parent:

3425 irq_domain_free_irqs_common(domain, virq, nr_irqs);

3426 return ret;

3427 }

3428

3429 static void irq_remapping_free(struct irq_domain *domain, unsigned int virq,

3430 unsigned int nr_irqs)

3431 {

3432 struct irq_2_irte *irte_info;

3433 struct irq_data *irq_data;

3434 struct amd_ir_data *data;

3435 int i;

3436

3437 for (i = 0; i < nr_irqs; i++) {

3438 irq_data = irq_domain_get_irq_data(domain, virq + i);

3439 if (irq_data && irq_data->chip_data) {

3440 data = irq_data->chip_data;

3441 irte_info = &data->irq_2_irte;

3442 free_irte(data->iommu, irte_info->devid, irte_info->index);

3443 kfree(data->entry);

3444 kfree(data);

3445 }

3446 }

3447 irq_domain_free_irqs_common(domain, virq, nr_irqs);

3448 }

3449

3450 static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu,

3451 struct amd_ir_data *ir_data,

3452 struct irq_2_irte *irte_info,

3453 struct irq_cfg *cfg);

3454

3455 static int irq_remapping_activate(struct irq_domain *domain,

3456 struct irq_data *irq_data, bool reserve)

3457 {

3458 struct amd_ir_data *data = irq_data->chip_data;

3459 struct irq_2_irte *irte_info = &data->irq_2_irte;

3460 struct amd_iommu *iommu = data->iommu;

3461 struct irq_cfg *cfg = irqd_cfg(irq_data);

3462

3463 if (!iommu)

3464 return 0;

3465

3466 iommu->irte_ops->activate(iommu, data->entry, irte_info->devid,

3467 irte_info->index);

3468 amd_ir_update_irte(irq_data, iommu, data, irte_info, cfg);

3469 return 0;

3470 }

3471

3472 static void irq_remapping_deactivate(struct irq_domain *domain,

3473 struct irq_data *irq_data)

3474 {

3475 struct amd_ir_data *data = irq_data->chip_data;

3476 struct irq_2_irte *irte_info = &data->irq_2_irte;

3477 struct amd_iommu *iommu = data->iommu;

3478

3479 if (iommu)

3480 iommu->irte_ops->deactivate(iommu, data->entry, irte_info->devid,

3481 irte_info->index);

3482 }

3483

3484 static int irq_remapping_select(struct irq_domain *d, struct irq_fwspec *fwspec,

3485 enum irq_domain_bus_token bus_token)

3486 {

3487 struct amd_iommu *iommu;

3488 int devid = -1;

3489

3490 if (!amd_iommu_irq_remap)

3491 return 0;

3492

3493 if (x86_fwspec_is_ioapic(fwspec))

3494 devid = get_ioapic_devid(fwspec->param[0]);

3495 else if (x86_fwspec_is_hpet(fwspec))

3496 devid = get_hpet_devid(fwspec->param[0]);

3497

3498 if (devid < 0)

3499 return 0;

3500 iommu = __rlookup_amd_iommu((devid >> 16), (devid & 0xffff));

3501

3502 return iommu && iommu->ir_domain == d;

3503 }

3504

3505 static const struct irq_domain_ops amd_ir_domain_ops = {

3506 .select = irq_remapping_select,

3507 .alloc = irq_remapping_alloc,

3508 .free = irq_remapping_free,

3509 .activate = irq_remapping_activate,

3510 .deactivate = irq_remapping_deactivate,

3511 };

3512

3513 int amd_iommu_activate_guest_mode(void *data)

3514 {

3515 struct amd_ir_data *ir_data = (struct amd_ir_data *)data;

3516 struct irte_ga *entry = (struct irte_ga *) ir_data->entry;

3517 u64 valid;

3518

3519 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) ||

3520 !entry || entry->lo.fields_vapic.guest_mode)

3521 return 0;

3522

3523 valid = entry->lo.fields_vapic.valid;

3524

3525 entry->lo.val = 0;

3526 entry->hi.val = 0;

3527

3528 entry->lo.fields_vapic.valid = valid;

3529 entry->lo.fields_vapic.guest_mode = 1;

3530 entry->lo.fields_vapic.ga_log_intr = 1;

3531 entry->hi.fields.ga_root_ptr = ir_data->ga_root_ptr;

3532 entry->hi.fields.vector = ir_data->ga_vector;

3533 entry->lo.fields_vapic.ga_tag = ir_data->ga_tag;

3534

3535 return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,

3536 ir_data->irq_2_irte.index, entry);

3537 }

3538 EXPORT_SYMBOL(amd_iommu_activate_guest_mode);

3539

3540 int amd_iommu_deactivate_guest_mode(void *data)

3541 {

3542 struct amd_ir_data *ir_data = (struct amd_ir_data *)data;

3543 struct irte_ga *entry = (struct irte_ga *) ir_data->entry;

3544 struct irq_cfg *cfg = ir_data->cfg;

3545 u64 valid;

3546

3547 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) ||

3548 !entry || !entry->lo.fields_vapic.guest_mode)

3549 return 0;

3550

3551 valid = entry->lo.fields_remap.valid;

3552

3553 entry->lo.val = 0;

3554 entry->hi.val = 0;

3555

3556 entry->lo.fields_remap.valid = valid;

3557 entry->lo.fields_remap.dm = apic->dest_mode_logical;

3558 entry->lo.fields_remap.int_type = apic->delivery_mode;

3559 entry->hi.fields.vector = cfg->vector;

3560 entry->lo.fields_remap.destination =

3561 APICID_TO_IRTE_DEST_LO(cfg->dest_apicid);

3562 entry->hi.fields.destination =

3563 APICID_TO_IRTE_DEST_HI(cfg->dest_apicid);

3564

3565 return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,

3566 ir_data->irq_2_irte.index, entry);

3567 }

3568 EXPORT_SYMBOL(amd_iommu_deactivate_guest_mode);

3569

3570 static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info)

3571 {

3572 int ret;

3573 struct amd_iommu_pi_data *pi_data = vcpu_info;

3574 struct vcpu_data *vcpu_pi_info = pi_data->vcpu_data;

3575 struct amd_ir_data *ir_data = data->chip_data;

3576 struct irq_2_irte *irte_info = &ir_data->irq_2_irte;

3577 struct iommu_dev_data *dev_data;

3578

3579 if (ir_data->iommu == NULL)

3580 return -EINVAL;

3581

3582 dev_data = search_dev_data(ir_data->iommu, irte_info->devid);

3583

3584 /* Note:

3585 * This device has never been set up for guest mode.

3586 * we should not modify the IRTE

3587 */

3588 if (!dev_data || !dev_data->use_vapic)

3589 return 0;

3590

3591 ir_data->cfg = irqd_cfg(data);

3592 pi_data->ir_data = ir_data;

3593

3594 /* Note:

3595 * SVM tries to set up for VAPIC mode, but we are in

3596 * legacy mode. So, we force legacy mode instead.

3597 */

3598 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) {

3599 pr_debug("%s: Fall back to using intr legacy remap\n",

3600 __func__);

3601 pi_data->is_guest_mode = false;

3602 }

3603

3604 pi_data->prev_ga_tag = ir_data->cached_ga_tag;

3605 if (pi_data->is_guest_mode) {

3606 ir_data->ga_root_ptr = (pi_data->base >> 12);

3607 ir_data->ga_vector = vcpu_pi_info->vector;

3608 ir_data->ga_tag = pi_data->ga_tag;

3609 ret = amd_iommu_activate_guest_mode(ir_data);

3610 if (!ret)

3611 ir_data->cached_ga_tag = pi_data->ga_tag;

3612 } else {

3613 ret = amd_iommu_deactivate_guest_mode(ir_data);

3614

3615 /*

3616 * This communicates the ga_tag back to the caller

3617 * so that it can do all the necessary clean up.

3618 */

3619 if (!ret)

3620 ir_data->cached_ga_tag = 0;

3621 }

3622

3623 return ret;

3624 }

3625

3626

3627 static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu,

3628 struct amd_ir_data *ir_data,

3629 struct irq_2_irte *irte_info,

3630 struct irq_cfg *cfg)

3631 {

3632

3633 /*

3634 * Atomically updates the IRTE with the new destination, vector

3635 * and flushes the interrupt entry cache.

3636 */

3637 iommu->irte_ops->set_affinity(iommu, ir_data->entry, irte_info->devid,

3638 irte_info->index, cfg->vector,

3639 cfg->dest_apicid);

3640 }

3641

3642 static int amd_ir_set_affinity(struct irq_data *data,

3643 const struct cpumask *mask, bool force)

3644 {

3645 struct amd_ir_data *ir_data = data->chip_data;

3646 struct irq_2_irte *irte_info = &ir_data->irq_2_irte;

3647 struct irq_cfg *cfg = irqd_cfg(data);

3648 struct irq_data *parent = data->parent_data;

3649 struct amd_iommu *iommu = ir_data->iommu;

3650 int ret;

3651

3652 if (!iommu)

3653 return -ENODEV;

3654

3655 ret = parent->chip->irq_set_affinity(parent, mask, force);

3656 if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)

3657 return ret;

3658

3659 amd_ir_update_irte(data, iommu, ir_data, irte_info, cfg);

3660 /*

3661 * After this point, all the interrupts will start arriving

3662 * at the new destination. So, time to cleanup the previous

3663 * vector allocation.

3664 */

3665 send_cleanup_vector(cfg);

3666

3667 return IRQ_SET_MASK_OK_DONE;

3668 }

3669

3670 static void ir_compose_msi_msg(struct irq_data *irq_data, struct msi_msg *msg)

3671 {

3672 struct amd_ir_data *ir_data = irq_data->chip_data;

3673

3674 *msg = ir_data->msi_entry;

3675 }

3676

3677 static struct irq_chip amd_ir_chip = {

3678 .name = "AMD-IR",

3679 .irq_ack = apic_ack_irq,

3680 .irq_set_affinity = amd_ir_set_affinity,

3681 .irq_set_vcpu_affinity = amd_ir_set_vcpu_affinity,

3682 .irq_compose_msi_msg = ir_compose_msi_msg,

3683 };

3684

3685 static const struct msi_parent_ops amdvi_msi_parent_ops = {

3686 .supported_flags = X86_VECTOR_MSI_FLAGS_SUPPORTED |

3687 MSI_FLAG_MULTI_PCI_MSI |

3688 MSI_FLAG_PCI_IMS,

3689 .prefix = "IR-",

3690 .init_dev_msi_info = msi_parent_init_dev_msi_info,

3691 };

3692

3693 static const struct msi_parent_ops virt_amdvi_msi_parent_ops = {

3694 .supported_flags = X86_VECTOR_MSI_FLAGS_SUPPORTED |

3695 MSI_FLAG_MULTI_PCI_MSI,

3696 .prefix = "vIR-",

3697 .init_dev_msi_info = msi_parent_init_dev_msi_info,

3698 };

3699

3700 int amd_iommu_create_irq_domain(struct amd_iommu *iommu)

3701 {

3702 struct fwnode_handle *fn;

3703

3704 fn = irq_domain_alloc_named_id_fwnode("AMD-IR", iommu->index);

3705 if (!fn)

3706 return -ENOMEM;

3707 iommu->ir_domain = irq_domain_create_hierarchy(arch_get_ir_parent_domain(), 0, 0,

3708 fn, &amd_ir_domain_ops, iommu);

3709 if (!iommu->ir_domain) {

3710 irq_domain_free_fwnode(fn);

3711 return -ENOMEM;

3712 }

3713

3714 irq_domain_update_bus_token(iommu->ir_domain, DOMAIN_BUS_AMDVI);

3715 iommu->ir_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT |

3716 IRQ_DOMAIN_FLAG_ISOLATED_MSI;

3717

3718 if (amd_iommu_np_cache)

3719 iommu->ir_domain->msi_parent_ops = &virt_amdvi_msi_parent_ops;

3720 else

3721 iommu->ir_domain->msi_parent_ops = &amdvi_msi_parent_ops;

3722

3723 return 0;

3724 }

3725

3726 int amd_iommu_update_ga(int cpu, bool is_run, void *data)

3727 {

3728 struct amd_ir_data *ir_data = (struct amd_ir_data *)data;

3729 struct irte_ga *entry = (struct irte_ga *) ir_data->entry;

3730

3731 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) ||

3732 !entry || !entry->lo.fields_vapic.guest_mode)

3733 return 0;

3734

3735 if (!ir_data->iommu)

3736 return -ENODEV;

3737

3738 if (cpu >= 0) {

3739 entry->lo.fields_vapic.destination =

3740 APICID_TO_IRTE_DEST_LO(cpu);

3741 entry->hi.fields.destination =

3742 APICID_TO_IRTE_DEST_HI(cpu);

3743 }

3744 entry->lo.fields_vapic.is_run = is_run;

3745

3746 return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,

3747 ir_data->irq_2_irte.index, entry);

3748 }

3749 EXPORT_SYMBOL(amd_iommu_update_ga);

3750 #endif