git.maquefel.me Git - linux.git/blob

1 // SPDX-License-Identifier: GPL-2.0

2 /*

3 * Driver for HiSilicon PCIe tune and trace device

4 *

6 * Author: Yicong Yang <yangyicong@hisilicon.com>

7 */

9 #include <linux/bitfield.h>

10 #include <linux/bitops.h>

11 #include <linux/cpuhotplug.h>

12 #include <linux/delay.h>

13 #include <linux/dma-mapping.h>

14 #include <linux/interrupt.h>

15 #include <linux/io.h>

16 #include <linux/iommu.h>

17 #include <linux/iopoll.h>

18 #include <linux/module.h>

19 #include <linux/sysfs.h>

20 #include <linux/vmalloc.h>

22 #include "hisi_ptt.h"

24 /* Dynamic CPU hotplug state used by PTT */

25 static enum cpuhp_state hisi_ptt_pmu_online;

27 static bool hisi_ptt_wait_tuning_finish(struct hisi_ptt *hisi_ptt)

28 {

29 u32 val;

31 return !readl_poll_timeout(hisi_ptt->iobase + HISI_PTT_TUNING_INT_STAT,

32 val, !(val & HISI_PTT_TUNING_INT_STAT_MASK),

33 HISI_PTT_WAIT_POLL_INTERVAL_US,

34 HISI_PTT_WAIT_TUNE_TIMEOUT_US);

35 }

37 static ssize_t hisi_ptt_tune_attr_show(struct device *dev,

38 struct device_attribute *attr,

39 char *buf)

40 {

41 struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev));

42 struct dev_ext_attribute *ext_attr;

43 struct hisi_ptt_tune_desc *desc;

44 u32 reg;

45 u16 val;

47 ext_attr = container_of(attr, struct dev_ext_attribute, attr);

48 desc = ext_attr->var;

50 mutex_lock(&hisi_ptt->tune_lock);

52 reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_CTRL);

53 reg &= ~(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB);

54 reg |= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB,

55 desc->event_code);

56 writel(reg, hisi_ptt->iobase + HISI_PTT_TUNING_CTRL);

58 /* Write all 1 to indicates it's the read process */

59 writel(~0U, hisi_ptt->iobase + HISI_PTT_TUNING_DATA);

61 if (!hisi_ptt_wait_tuning_finish(hisi_ptt)) {

62 mutex_unlock(&hisi_ptt->tune_lock);

63 return -ETIMEDOUT;

64 }

66 reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_DATA);

67 reg &= HISI_PTT_TUNING_DATA_VAL_MASK;

68 val = FIELD_GET(HISI_PTT_TUNING_DATA_VAL_MASK, reg);

70 mutex_unlock(&hisi_ptt->tune_lock);

71 return sysfs_emit(buf, "%u\n", val);

72 }

74 static ssize_t hisi_ptt_tune_attr_store(struct device *dev,

75 struct device_attribute *attr,

76 const char *buf, size_t count)

77 {

78 struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev));

79 struct dev_ext_attribute *ext_attr;

80 struct hisi_ptt_tune_desc *desc;

81 u32 reg;

82 u16 val;

84 ext_attr = container_of(attr, struct dev_ext_attribute, attr);

85 desc = ext_attr->var;

87 if (kstrtou16(buf, 10, &val))

88 return -EINVAL;

90 mutex_lock(&hisi_ptt->tune_lock);

92 reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_CTRL);

93 reg &= ~(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB);

94 reg |= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB,

95 desc->event_code);

96 writel(reg, hisi_ptt->iobase + HISI_PTT_TUNING_CTRL);

97 writel(FIELD_PREP(HISI_PTT_TUNING_DATA_VAL_MASK, val),

98 hisi_ptt->iobase + HISI_PTT_TUNING_DATA);

100 if (!hisi_ptt_wait_tuning_finish(hisi_ptt)) {

101 mutex_unlock(&hisi_ptt->tune_lock);

102 return -ETIMEDOUT;

103 }

104

105 mutex_unlock(&hisi_ptt->tune_lock);

106 return count;

107 }

108

109 #define HISI_PTT_TUNE_ATTR(_name, _val, _show, _store) \

110 static struct hisi_ptt_tune_desc _name##_desc = { \

111 .name = #_name, \

112 .event_code = (_val), \

113 }; \

114 static struct dev_ext_attribute hisi_ptt_##_name##_attr = { \

115 .attr = __ATTR(_name, 0600, _show, _store), \

116 .var = &_name##_desc, \

117 }

118

119 #define HISI_PTT_TUNE_ATTR_COMMON(_name, _val) \

120 HISI_PTT_TUNE_ATTR(_name, _val, \

121 hisi_ptt_tune_attr_show, \

122 hisi_ptt_tune_attr_store)

123

124 /*

125 * The value of the tuning event are composed of two parts: main event code

126 * in BIT[0,15] and subevent code in BIT[16,23]. For example, qox_tx_cpl is

127 * a subevent of 'Tx path QoS control' which for tuning the weight of Tx

128 * completion TLPs. See hisi_ptt.rst documentation for more information.

129 */

130 #define HISI_PTT_TUNE_QOS_TX_CPL (0x4 | (3 << 16))

131 #define HISI_PTT_TUNE_QOS_TX_NP (0x4 | (4 << 16))

132 #define HISI_PTT_TUNE_QOS_TX_P (0x4 | (5 << 16))

133 #define HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL (0x5 | (6 << 16))

134 #define HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL (0x5 | (7 << 16))

135

136 HISI_PTT_TUNE_ATTR_COMMON(qos_tx_cpl, HISI_PTT_TUNE_QOS_TX_CPL);

137 HISI_PTT_TUNE_ATTR_COMMON(qos_tx_np, HISI_PTT_TUNE_QOS_TX_NP);

138 HISI_PTT_TUNE_ATTR_COMMON(qos_tx_p, HISI_PTT_TUNE_QOS_TX_P);

139 HISI_PTT_TUNE_ATTR_COMMON(rx_alloc_buf_level, HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL);

140 HISI_PTT_TUNE_ATTR_COMMON(tx_alloc_buf_level, HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL);

141

142 static struct attribute *hisi_ptt_tune_attrs[] = {

143 &hisi_ptt_qos_tx_cpl_attr.attr.attr,

144 &hisi_ptt_qos_tx_np_attr.attr.attr,

145 &hisi_ptt_qos_tx_p_attr.attr.attr,

146 &hisi_ptt_rx_alloc_buf_level_attr.attr.attr,

147 &hisi_ptt_tx_alloc_buf_level_attr.attr.attr,

148 NULL,

149 };

150

151 static struct attribute_group hisi_ptt_tune_group = {

152 .name = "tune",

153 .attrs = hisi_ptt_tune_attrs,

154 };

155

156 static u16 hisi_ptt_get_filter_val(u16 devid, bool is_port)

157 {

158 if (is_port)

159 return BIT(HISI_PCIE_CORE_PORT_ID(devid & 0xff));

160

161 return devid;

162 }

163

164 static bool hisi_ptt_wait_trace_hw_idle(struct hisi_ptt *hisi_ptt)

165 {

166 u32 val;

167

168 return !readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_STS,

169 val, val & HISI_PTT_TRACE_IDLE,

170 HISI_PTT_WAIT_POLL_INTERVAL_US,

171 HISI_PTT_WAIT_TRACE_TIMEOUT_US);

172 }

173

174 static void hisi_ptt_wait_dma_reset_done(struct hisi_ptt *hisi_ptt)

175 {

176 u32 val;

177

178 readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS,

179 val, !val, HISI_PTT_RESET_POLL_INTERVAL_US,

180 HISI_PTT_RESET_TIMEOUT_US);

181 }

182

183 static void hisi_ptt_trace_end(struct hisi_ptt *hisi_ptt)

184 {

185 writel(0, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);

186 hisi_ptt->trace_ctrl.started = false;

187 }

188

189 static int hisi_ptt_trace_start(struct hisi_ptt *hisi_ptt)

190 {

191 struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl;

192 u32 val;

193 int i;

194

195 /* Check device idle before start trace */

196 if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt)) {

197 pci_err(hisi_ptt->pdev, "Failed to start trace, the device is still busy\n");

198 return -EBUSY;

199 }

200

201 ctrl->started = true;

202

203 /* Reset the DMA before start tracing */

204 val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);

205 val |= HISI_PTT_TRACE_CTRL_RST;

206 writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);

207

208 hisi_ptt_wait_dma_reset_done(hisi_ptt);

209

210 val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);

211 val &= ~HISI_PTT_TRACE_CTRL_RST;

212 writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);

213

214 /* Reset the index of current buffer */

215 hisi_ptt->trace_ctrl.buf_index = 0;

216

217 /* Zero the trace buffers */

218 for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; i++)

219 memset(ctrl->trace_buf[i].addr, 0, HISI_PTT_TRACE_BUF_SIZE);

220

221 /* Clear the interrupt status */

222 writel(HISI_PTT_TRACE_INT_STAT_MASK, hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT);

223 writel(0, hisi_ptt->iobase + HISI_PTT_TRACE_INT_MASK);

224

225 /* Set the trace control register */

226 val = FIELD_PREP(HISI_PTT_TRACE_CTRL_TYPE_SEL, ctrl->type);

227 val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_RXTX_SEL, ctrl->direction);

228 val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_DATA_FORMAT, ctrl->format);

229 val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_TARGET_SEL, hisi_ptt->trace_ctrl.filter);

230 if (!hisi_ptt->trace_ctrl.is_port)

231 val |= HISI_PTT_TRACE_CTRL_FILTER_MODE;

232

233 /* Start the Trace */

234 val |= HISI_PTT_TRACE_CTRL_EN;

235 writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);

236

237 return 0;

238 }

239

240 static int hisi_ptt_update_aux(struct hisi_ptt *hisi_ptt, int index, bool stop)

241 {

242 struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl;

243 struct perf_output_handle *handle = &ctrl->handle;

244 struct perf_event *event = handle->event;

245 struct hisi_ptt_pmu_buf *buf;

246 size_t size;

247 void *addr;

248

249 buf = perf_get_aux(handle);

250 if (!buf || !handle->size)

251 return -EINVAL;

252

253 addr = ctrl->trace_buf[ctrl->buf_index].addr;

254

255 /*

256 * If we're going to stop, read the size of already traced data from

257 * HISI_PTT_TRACE_WR_STS. Otherwise we're coming from the interrupt,

258 * the data size is always HISI_PTT_TRACE_BUF_SIZE.

259 */

260 if (stop) {

261 u32 reg;

262

263 reg = readl(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS);

264 size = FIELD_GET(HISI_PTT_TRACE_WR_STS_WRITE, reg);

265 } else {

266 size = HISI_PTT_TRACE_BUF_SIZE;

267 }

268

269 memcpy(buf->base + buf->pos, addr, size);

270 buf->pos += size;

271

272 /*

273 * Just commit the traced data if we're going to stop. Otherwise if the

274 * resident AUX buffer cannot contain the data of next trace buffer,

275 * apply a new one.

276 */

277 if (stop) {

278 perf_aux_output_end(handle, buf->pos);

279 } else if (buf->length - buf->pos < HISI_PTT_TRACE_BUF_SIZE) {

280 perf_aux_output_end(handle, buf->pos);

281

282 buf = perf_aux_output_begin(handle, event);

283 if (!buf)

284 return -EINVAL;

285

286 buf->pos = handle->head % buf->length;

287 if (buf->length - buf->pos < HISI_PTT_TRACE_BUF_SIZE) {

288 perf_aux_output_end(handle, 0);

289 return -EINVAL;

290 }

291 }

292

293 return 0;

294 }

295

296 static irqreturn_t hisi_ptt_isr(int irq, void *context)

297 {

298 struct hisi_ptt *hisi_ptt = context;

299 u32 status, buf_idx;

300

301 status = readl(hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT);

302 if (!(status & HISI_PTT_TRACE_INT_STAT_MASK))

303 return IRQ_NONE;

304

305 buf_idx = ffs(status) - 1;

306

307 /* Clear the interrupt status of buffer @buf_idx */

308 writel(status, hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT);

309

310 /*

311 * Update the AUX buffer and cache the current buffer index,

312 * as we need to know this and save the data when the trace

313 * is ended out of the interrupt handler. End the trace

314 * if the updating fails.

315 */

316 if (hisi_ptt_update_aux(hisi_ptt, buf_idx, false))

317 hisi_ptt_trace_end(hisi_ptt);

318 else

319 hisi_ptt->trace_ctrl.buf_index = (buf_idx + 1) % HISI_PTT_TRACE_BUF_CNT;

320

321 return IRQ_HANDLED;

322 }

323

324 static void hisi_ptt_irq_free_vectors(void *pdev)

325 {

326 pci_free_irq_vectors(pdev);

327 }

328

329 static int hisi_ptt_register_irq(struct hisi_ptt *hisi_ptt)

330 {

331 struct pci_dev *pdev = hisi_ptt->pdev;

332 int ret;

333

334 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);

335 if (ret < 0) {

336 pci_err(pdev, "failed to allocate irq vector, ret = %d\n", ret);

337 return ret;

338 }

339

340 ret = devm_add_action_or_reset(&pdev->dev, hisi_ptt_irq_free_vectors, pdev);

341 if (ret < 0)

342 return ret;

343

344 ret = devm_request_threaded_irq(&pdev->dev,

345 pci_irq_vector(pdev, HISI_PTT_TRACE_DMA_IRQ),

346 NULL, hisi_ptt_isr, 0,

347 DRV_NAME, hisi_ptt);

348 if (ret) {

349 pci_err(pdev, "failed to request irq %d, ret = %d\n",

350 pci_irq_vector(pdev, HISI_PTT_TRACE_DMA_IRQ), ret);

351 return ret;

352 }

353

354 return 0;

355 }

356

357 static void hisi_ptt_del_free_filter(struct hisi_ptt *hisi_ptt,

358 struct hisi_ptt_filter_desc *filter)

359 {

360 if (filter->is_port)

361 hisi_ptt->port_mask &= ~hisi_ptt_get_filter_val(filter->devid, true);

362

363 list_del(&filter->list);

364 kfree(filter->name);

365 kfree(filter);

366 }

367

368 static struct hisi_ptt_filter_desc *

369 hisi_ptt_alloc_add_filter(struct hisi_ptt *hisi_ptt, u16 devid, bool is_port)

370 {

371 struct hisi_ptt_filter_desc *filter;

372 u8 devfn = devid & 0xff;

373 char *filter_name;

374

375 filter_name = kasprintf(GFP_KERNEL, "%04x:%02x:%02x.%d", pci_domain_nr(hisi_ptt->pdev->bus),

376 PCI_BUS_NUM(devid), PCI_SLOT(devfn), PCI_FUNC(devfn));

377 if (!filter_name) {

378 pci_err(hisi_ptt->pdev, "failed to allocate name for filter %04x:%02x:%02x.%d\n",

379 pci_domain_nr(hisi_ptt->pdev->bus), PCI_BUS_NUM(devid),

380 PCI_SLOT(devfn), PCI_FUNC(devfn));

381 return NULL;

382 }

383

384 filter = kzalloc(sizeof(*filter), GFP_KERNEL);

385 if (!filter) {

386 pci_err(hisi_ptt->pdev, "failed to add filter for %s\n",

387 filter_name);

388 kfree(filter_name);

389 return NULL;

390 }

391

392 filter->name = filter_name;

393 filter->is_port = is_port;

394 filter->devid = devid;

395

396 if (filter->is_port) {

397 list_add_tail(&filter->list, &hisi_ptt->port_filters);

398

399 /* Update the available port mask */

400 hisi_ptt->port_mask |= hisi_ptt_get_filter_val(filter->devid, true);

401 } else {

402 list_add_tail(&filter->list, &hisi_ptt->req_filters);

403 }

404

405 return filter;

406 }

407

408 static void hisi_ptt_update_filters(struct work_struct *work)

409 {

410 struct delayed_work *delayed_work = to_delayed_work(work);

411 struct hisi_ptt_filter_update_info info;

412 struct hisi_ptt_filter_desc *filter;

413 struct hisi_ptt *hisi_ptt;

414

415 hisi_ptt = container_of(delayed_work, struct hisi_ptt, work);

416

417 if (!mutex_trylock(&hisi_ptt->filter_lock)) {

418 schedule_delayed_work(&hisi_ptt->work, HISI_PTT_WORK_DELAY_MS);

419 return;

420 }

421

422 while (kfifo_get(&hisi_ptt->filter_update_kfifo, &info)) {

423 if (info.is_add) {

424 /*

425 * Notify the users if failed to add this filter, others

426 * still work and available. See the comments in

427 * hisi_ptt_init_filters().

428 */

429 filter = hisi_ptt_alloc_add_filter(hisi_ptt, info.devid, info.is_port);

430 if (!filter)

431 continue;

432 } else {

433 struct hisi_ptt_filter_desc *tmp;

434 struct list_head *target_list;

435

436 target_list = info.is_port ? &hisi_ptt->port_filters :

437 &hisi_ptt->req_filters;

438

439 list_for_each_entry_safe(filter, tmp, target_list, list)

440 if (filter->devid == info.devid) {

441 hisi_ptt_del_free_filter(hisi_ptt, filter);

442 break;

443 }

444 }

445 }

446

447 mutex_unlock(&hisi_ptt->filter_lock);

448 }

449

450 /*

451 * A PCI bus notifier is used here for dynamically updating the filter

452 * list.

453 */

454 static int hisi_ptt_notifier_call(struct notifier_block *nb, unsigned long action,

455 void *data)

456 {

457 struct hisi_ptt *hisi_ptt = container_of(nb, struct hisi_ptt, hisi_ptt_nb);

458 struct hisi_ptt_filter_update_info info;

459 struct pci_dev *pdev, *root_port;

460 struct device *dev = data;

461 u32 port_devid;

462

463 pdev = to_pci_dev(dev);

464 root_port = pcie_find_root_port(pdev);

465 if (!root_port)

466 return 0;

467

468 port_devid = PCI_DEVID(root_port->bus->number, root_port->devfn);

469 if (port_devid < hisi_ptt->lower_bdf ||

470 port_devid > hisi_ptt->upper_bdf)

471 return 0;

472

473 info.is_port = pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT;

474 info.devid = PCI_DEVID(pdev->bus->number, pdev->devfn);

475

476 switch (action) {

477 case BUS_NOTIFY_ADD_DEVICE:

478 info.is_add = true;

479 break;

480 case BUS_NOTIFY_DEL_DEVICE:

481 info.is_add = false;

482 break;

483 default:

484 return 0;

485 }

486

487 /*

488 * The FIFO size is 16 which is sufficient for almost all the cases,

489 * since each PCIe core will have most 8 Root Ports (typically only

490 * 1~4 Root Ports). On failure log the failed filter and let user

491 * handle it.

492 */

493 if (kfifo_in_spinlocked(&hisi_ptt->filter_update_kfifo, &info, 1,

494 &hisi_ptt->filter_update_lock))

495 schedule_delayed_work(&hisi_ptt->work, 0);

496 else

497 pci_warn(hisi_ptt->pdev,

498 "filter update fifo overflow for target %s\n",

499 pci_name(pdev));

500

501 return 0;

502 }

503

504 static int hisi_ptt_init_filters(struct pci_dev *pdev, void *data)

505 {

506 struct pci_dev *root_port = pcie_find_root_port(pdev);

507 struct hisi_ptt_filter_desc *filter;

508 struct hisi_ptt *hisi_ptt = data;

509 u32 port_devid;

510

511 if (!root_port)

512 return 0;

513

514 port_devid = PCI_DEVID(root_port->bus->number, root_port->devfn);

515 if (port_devid < hisi_ptt->lower_bdf ||

516 port_devid > hisi_ptt->upper_bdf)

517 return 0;

518

519 /*

520 * We won't fail the probe if filter allocation failed here. The filters

521 * should be partial initialized and users would know which filter fails

522 * through the log. Other functions of PTT device are still available.

523 */

524 filter = hisi_ptt_alloc_add_filter(hisi_ptt, PCI_DEVID(pdev->bus->number, pdev->devfn),

525 pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT);

526 if (!filter)

527 return -ENOMEM;

528

529 return 0;

530 }

531

532 static void hisi_ptt_release_filters(void *data)

533 {

534 struct hisi_ptt_filter_desc *filter, *tmp;

535 struct hisi_ptt *hisi_ptt = data;

536

537 list_for_each_entry_safe(filter, tmp, &hisi_ptt->req_filters, list)

538 hisi_ptt_del_free_filter(hisi_ptt, filter);

539

540 list_for_each_entry_safe(filter, tmp, &hisi_ptt->port_filters, list)

541 hisi_ptt_del_free_filter(hisi_ptt, filter);

542 }

543

544 static int hisi_ptt_config_trace_buf(struct hisi_ptt *hisi_ptt)

545 {

546 struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl;

547 struct device *dev = &hisi_ptt->pdev->dev;

548 int i;

549

550 ctrl->trace_buf = devm_kcalloc(dev, HISI_PTT_TRACE_BUF_CNT,

551 sizeof(*ctrl->trace_buf), GFP_KERNEL);

552 if (!ctrl->trace_buf)

553 return -ENOMEM;

554

555 for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; ++i) {

556 ctrl->trace_buf[i].addr = dmam_alloc_coherent(dev, HISI_PTT_TRACE_BUF_SIZE,

557 &ctrl->trace_buf[i].dma,

558 GFP_KERNEL);

559 if (!ctrl->trace_buf[i].addr)

560 return -ENOMEM;

561 }

562

563 /* Configure the trace DMA buffer */

564 for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; i++) {

565 writel(lower_32_bits(ctrl->trace_buf[i].dma),

566 hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_BASE_LO_0 +

567 i * HISI_PTT_TRACE_ADDR_STRIDE);

568 writel(upper_32_bits(ctrl->trace_buf[i].dma),

569 hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_BASE_HI_0 +

570 i * HISI_PTT_TRACE_ADDR_STRIDE);

571 }

572 writel(HISI_PTT_TRACE_BUF_SIZE, hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_SIZE);

573

574 return 0;

575 }

576

577 static int hisi_ptt_init_ctrls(struct hisi_ptt *hisi_ptt)

578 {

579 struct pci_dev *pdev = hisi_ptt->pdev;

580 struct pci_bus *bus;

581 int ret;

582 u32 reg;

583

584 INIT_DELAYED_WORK(&hisi_ptt->work, hisi_ptt_update_filters);

585 INIT_KFIFO(hisi_ptt->filter_update_kfifo);

586 spin_lock_init(&hisi_ptt->filter_update_lock);

587

588 INIT_LIST_HEAD(&hisi_ptt->port_filters);

589 INIT_LIST_HEAD(&hisi_ptt->req_filters);

590 mutex_init(&hisi_ptt->filter_lock);

591

592 ret = hisi_ptt_config_trace_buf(hisi_ptt);

593 if (ret)

594 return ret;

595

596 /*

597 * The device range register provides the information about the root

598 * ports which the RCiEP can control and trace. The RCiEP and the root

599 * ports which it supports are on the same PCIe core, with same domain

600 * number but maybe different bus number. The device range register

601 * will tell us which root ports we can support, Bit[31:16] indicates

602 * the upper BDF numbers of the root port, while Bit[15:0] indicates

603 * the lower.

604 */

605 reg = readl(hisi_ptt->iobase + HISI_PTT_DEVICE_RANGE);

606 hisi_ptt->upper_bdf = FIELD_GET(HISI_PTT_DEVICE_RANGE_UPPER, reg);

607 hisi_ptt->lower_bdf = FIELD_GET(HISI_PTT_DEVICE_RANGE_LOWER, reg);

608

609 bus = pci_find_bus(pci_domain_nr(pdev->bus), PCI_BUS_NUM(hisi_ptt->upper_bdf));

610 if (bus)

611 pci_walk_bus(bus, hisi_ptt_init_filters, hisi_ptt);

612

613 ret = devm_add_action_or_reset(&pdev->dev, hisi_ptt_release_filters, hisi_ptt);

614 if (ret)

615 return ret;

616

617 hisi_ptt->trace_ctrl.on_cpu = -1;

618 return 0;

619 }

620

621 static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,

622 char *buf)

623 {

624 struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev));

625 const cpumask_t *cpumask = cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev));

626

627 return cpumap_print_to_pagebuf(true, buf, cpumask);

628 }

629 static DEVICE_ATTR_RO(cpumask);

630

631 static struct attribute *hisi_ptt_cpumask_attrs[] = {

632 &dev_attr_cpumask.attr,

633 NULL

634 };

635

636 static const struct attribute_group hisi_ptt_cpumask_attr_group = {

637 .attrs = hisi_ptt_cpumask_attrs,

638 };

639

640 /*

641 * Bit 19 indicates the filter type, 1 for Root Port filter and 0 for Requester

642 * filter. Bit[15:0] indicates the filter value, for Root Port filter it's

643 * a bit mask of desired ports and for Requester filter it's the Requester ID

644 * of the desired PCIe function. Bit[18:16] is reserved for extension.

645 *

646 * See hisi_ptt.rst documentation for detailed information.

647 */

648 PMU_FORMAT_ATTR(filter, "config:0-19");

649 PMU_FORMAT_ATTR(direction, "config:20-23");

650 PMU_FORMAT_ATTR(type, "config:24-31");

651 PMU_FORMAT_ATTR(format, "config:32-35");

652

653 static struct attribute *hisi_ptt_pmu_format_attrs[] = {

654 &format_attr_filter.attr,

655 &format_attr_direction.attr,

656 &format_attr_type.attr,

657 &format_attr_format.attr,

658 NULL

659 };

660

661 static struct attribute_group hisi_ptt_pmu_format_group = {

662 .name = "format",

663 .attrs = hisi_ptt_pmu_format_attrs,

664 };

665

666 static const struct attribute_group *hisi_ptt_pmu_groups[] = {

667 &hisi_ptt_cpumask_attr_group,

668 &hisi_ptt_pmu_format_group,

669 &hisi_ptt_tune_group,

670 NULL

671 };

672

673 static int hisi_ptt_trace_valid_direction(u32 val)

674 {

675 /*

676 * The direction values have different effects according to the data

677 * format (specified in the parentheses). TLP set A/B means different

678 * set of TLP types. See hisi_ptt.rst documentation for more details.

679 */

680 static const u32 hisi_ptt_trace_available_direction[] = {

681 0, /* inbound(4DW) or reserved(8DW) */

682 1, /* outbound(4DW) */

683 2, /* {in, out}bound(4DW) or inbound(8DW), TLP set A */

684 3, /* {in, out}bound(4DW) or inbound(8DW), TLP set B */

685 };

686 int i;

687

688 for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_available_direction); i++) {

689 if (val == hisi_ptt_trace_available_direction[i])

690 return 0;

691 }

692

693 return -EINVAL;

694 }

695

696 static int hisi_ptt_trace_valid_type(u32 val)

697 {

698 /* Different types can be set simultaneously */

699 static const u32 hisi_ptt_trace_available_type[] = {

700 1, /* posted_request */

701 2, /* non-posted_request */

702 4, /* completion */

703 };

704 int i;

705

706 if (!val)

707 return -EINVAL;

708

709 /*

710 * Walk the available list and clear the valid bits of

711 * the config. If there is any resident bit after the

712 * walk then the config is invalid.

713 */

714 for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_available_type); i++)

715 val &= ~hisi_ptt_trace_available_type[i];

716

717 if (val)

718 return -EINVAL;

719

720 return 0;

721 }

722

723 static int hisi_ptt_trace_valid_format(u32 val)

724 {

725 static const u32 hisi_ptt_trace_availble_format[] = {

726 0, /* 4DW */

727 1, /* 8DW */

728 };

729 int i;

730

731 for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_availble_format); i++) {

732 if (val == hisi_ptt_trace_availble_format[i])

733 return 0;

734 }

735

736 return -EINVAL;

737 }

738

739 static int hisi_ptt_trace_valid_filter(struct hisi_ptt *hisi_ptt, u64 config)

740 {

741 unsigned long val, port_mask = hisi_ptt->port_mask;

742 struct hisi_ptt_filter_desc *filter;

743 int ret = 0;

744

745 hisi_ptt->trace_ctrl.is_port = FIELD_GET(HISI_PTT_PMU_FILTER_IS_PORT, config);

746 val = FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK, config);

747

748 /*

749 * Port filters are defined as bit mask. For port filters, check

750 * the bits in the @val are within the range of hisi_ptt->port_mask

751 * and whether it's empty or not, otherwise user has specified

752 * some unsupported root ports.

753 *

754 * For Requester ID filters, walk the available filter list to see

755 * whether we have one matched.

756 */

757 mutex_lock(&hisi_ptt->filter_lock);

758 if (!hisi_ptt->trace_ctrl.is_port) {

759 list_for_each_entry(filter, &hisi_ptt->req_filters, list) {

760 if (val == hisi_ptt_get_filter_val(filter->devid, filter->is_port))

761 goto out;

762 }

763 } else if (bitmap_subset(&val, &port_mask, BITS_PER_LONG)) {

764 goto out;

765 }

766

767 ret = -EINVAL;

768 out:

769 mutex_unlock(&hisi_ptt->filter_lock);

770 return ret;

771 }

772

773 static void hisi_ptt_pmu_init_configs(struct hisi_ptt *hisi_ptt, struct perf_event *event)

774 {

775 struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl;

776 u32 val;

777

778 val = FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK, event->attr.config);

779 hisi_ptt->trace_ctrl.filter = val;

780

781 val = FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK, event->attr.config);

782 ctrl->direction = val;

783

784 val = FIELD_GET(HISI_PTT_PMU_TYPE_MASK, event->attr.config);

785 ctrl->type = val;

786

787 val = FIELD_GET(HISI_PTT_PMU_FORMAT_MASK, event->attr.config);

788 ctrl->format = val;

789 }

790

791 static int hisi_ptt_pmu_event_init(struct perf_event *event)

792 {

793 struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu);

794 int ret;

795 u32 val;

796

797 if (event->cpu < 0) {

798 dev_dbg(event->pmu->dev, "Per-task mode not supported\n");

799 return -EOPNOTSUPP;

800 }

801

802 if (event->attr.type != hisi_ptt->hisi_ptt_pmu.type)

803 return -ENOENT;

804

805 ret = hisi_ptt_trace_valid_filter(hisi_ptt, event->attr.config);

806 if (ret < 0)

807 return ret;

808

809 val = FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK, event->attr.config);

810 ret = hisi_ptt_trace_valid_direction(val);

811 if (ret < 0)

812 return ret;

813

814 val = FIELD_GET(HISI_PTT_PMU_TYPE_MASK, event->attr.config);

815 ret = hisi_ptt_trace_valid_type(val);

816 if (ret < 0)

817 return ret;

818

819 val = FIELD_GET(HISI_PTT_PMU_FORMAT_MASK, event->attr.config);

820 return hisi_ptt_trace_valid_format(val);

821 }

822

823 static void *hisi_ptt_pmu_setup_aux(struct perf_event *event, void **pages,

824 int nr_pages, bool overwrite)

825 {

826 struct hisi_ptt_pmu_buf *buf;

827 struct page **pagelist;

828 int i;

829

830 if (overwrite) {

831 dev_warn(event->pmu->dev, "Overwrite mode is not supported\n");

832 return NULL;

833 }

834

835 /* If the pages size less than buffers, we cannot start trace */

836 if (nr_pages < HISI_PTT_TRACE_TOTAL_BUF_SIZE / PAGE_SIZE)

837 return NULL;

838

839 buf = kzalloc(sizeof(*buf), GFP_KERNEL);

840 if (!buf)

841 return NULL;

842

843 pagelist = kcalloc(nr_pages, sizeof(*pagelist), GFP_KERNEL);

844 if (!pagelist)

845 goto err;

846

847 for (i = 0; i < nr_pages; i++)

848 pagelist[i] = virt_to_page(pages[i]);

849

850 buf->base = vmap(pagelist, nr_pages, VM_MAP, PAGE_KERNEL);

851 if (!buf->base) {

852 kfree(pagelist);

853 goto err;

854 }

855

856 buf->nr_pages = nr_pages;

857 buf->length = nr_pages * PAGE_SIZE;

858 buf->pos = 0;

859

860 kfree(pagelist);

861 return buf;

862 err:

863 kfree(buf);

864 return NULL;

865 }

866

867 static void hisi_ptt_pmu_free_aux(void *aux)

868 {

869 struct hisi_ptt_pmu_buf *buf = aux;

870

871 vunmap(buf->base);

872 kfree(buf);

873 }

874

875 static void hisi_ptt_pmu_start(struct perf_event *event, int flags)

876 {

877 struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu);

878 struct perf_output_handle *handle = &hisi_ptt->trace_ctrl.handle;

879 struct hw_perf_event *hwc = &event->hw;

880 struct device *dev = event->pmu->dev;

881 struct hisi_ptt_pmu_buf *buf;

882 int cpu = event->cpu;

883 int ret;

884

885 hwc->state = 0;

886

887 /* Serialize the perf process if user specified several CPUs */

888 spin_lock(&hisi_ptt->pmu_lock);

889 if (hisi_ptt->trace_ctrl.started) {

890 dev_dbg(dev, "trace has already started\n");

891 goto stop;

892 }

893

894 /*

895 * Handle the interrupt on the same cpu which starts the trace to avoid

896 * context mismatch. Otherwise we'll trigger the WARN from the perf

897 * core in event_function_local(). If CPU passed is offline we'll fail

898 * here, just log it since we can do nothing here.

899 */

900 ret = irq_set_affinity(pci_irq_vector(hisi_ptt->pdev, HISI_PTT_TRACE_DMA_IRQ),

901 cpumask_of(cpu));

902 if (ret)

903 dev_warn(dev, "failed to set the affinity of trace interrupt\n");

904

905 hisi_ptt->trace_ctrl.on_cpu = cpu;

906

907 buf = perf_aux_output_begin(handle, event);

908 if (!buf) {

909 dev_dbg(dev, "aux output begin failed\n");

910 goto stop;

911 }

912

913 buf->pos = handle->head % buf->length;

914

915 hisi_ptt_pmu_init_configs(hisi_ptt, event);

916

917 ret = hisi_ptt_trace_start(hisi_ptt);

918 if (ret) {

919 dev_dbg(dev, "trace start failed, ret = %d\n", ret);

920 perf_aux_output_end(handle, 0);

921 goto stop;

922 }

923

924 spin_unlock(&hisi_ptt->pmu_lock);

925 return;

926 stop:

927 event->hw.state |= PERF_HES_STOPPED;

928 spin_unlock(&hisi_ptt->pmu_lock);

929 }

930

931 static void hisi_ptt_pmu_stop(struct perf_event *event, int flags)

932 {

933 struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu);

934 struct hw_perf_event *hwc = &event->hw;

935

936 if (hwc->state & PERF_HES_STOPPED)

937 return;

938

939 spin_lock(&hisi_ptt->pmu_lock);

940 if (hisi_ptt->trace_ctrl.started) {

941 hisi_ptt_trace_end(hisi_ptt);

942

943 if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt))

944 dev_warn(event->pmu->dev, "Device is still busy\n");

945

946 hisi_ptt_update_aux(hisi_ptt, hisi_ptt->trace_ctrl.buf_index, true);

947 }

948 spin_unlock(&hisi_ptt->pmu_lock);

949

950 hwc->state |= PERF_HES_STOPPED;

951 perf_event_update_userpage(event);

952 hwc->state |= PERF_HES_UPTODATE;

953 }

954

955 static int hisi_ptt_pmu_add(struct perf_event *event, int flags)

956 {

957 struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu);

958 struct hw_perf_event *hwc = &event->hw;

959 int cpu = event->cpu;

960

961 /* Only allow the cpus on the device's node to add the event */

962 if (!cpumask_test_cpu(cpu, cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev))))

963 return 0;

964

965 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;

966

967 if (flags & PERF_EF_START) {

968 hisi_ptt_pmu_start(event, PERF_EF_RELOAD);

969 if (hwc->state & PERF_HES_STOPPED)

970 return -EINVAL;

971 }

972

973 return 0;

974 }

975

976 static void hisi_ptt_pmu_del(struct perf_event *event, int flags)

977 {

978 hisi_ptt_pmu_stop(event, PERF_EF_UPDATE);

979 }

980

981 static void hisi_ptt_remove_cpuhp_instance(void *hotplug_node)

982 {

983 cpuhp_state_remove_instance_nocalls(hisi_ptt_pmu_online, hotplug_node);

984 }

985

986 static void hisi_ptt_unregister_pmu(void *pmu)

987 {

988 perf_pmu_unregister(pmu);

989 }

990

991 static int hisi_ptt_register_pmu(struct hisi_ptt *hisi_ptt)

992 {

993 u16 core_id, sicl_id;

994 char *pmu_name;

995 u32 reg;

996 int ret;

997

998 ret = cpuhp_state_add_instance_nocalls(hisi_ptt_pmu_online,

999 &hisi_ptt->hotplug_node);

1000 if (ret)

1001 return ret;

1002

1003 ret = devm_add_action_or_reset(&hisi_ptt->pdev->dev,

1004 hisi_ptt_remove_cpuhp_instance,

1005 &hisi_ptt->hotplug_node);

1006 if (ret)

1007 return ret;

1008

1009 mutex_init(&hisi_ptt->tune_lock);

1010 spin_lock_init(&hisi_ptt->pmu_lock);

1011

1012 hisi_ptt->hisi_ptt_pmu = (struct pmu) {

1013 .module = THIS_MODULE,

1014 .capabilities = PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_ITRACE,

1015 .task_ctx_nr = perf_sw_context,

1016 .attr_groups = hisi_ptt_pmu_groups,

1017 .event_init = hisi_ptt_pmu_event_init,

1018 .setup_aux = hisi_ptt_pmu_setup_aux,

1019 .free_aux = hisi_ptt_pmu_free_aux,

1020 .start = hisi_ptt_pmu_start,

1021 .stop = hisi_ptt_pmu_stop,

1022 .add = hisi_ptt_pmu_add,

1023 .del = hisi_ptt_pmu_del,

1024 };

1025

1026 reg = readl(hisi_ptt->iobase + HISI_PTT_LOCATION);

1027 core_id = FIELD_GET(HISI_PTT_CORE_ID, reg);

1028 sicl_id = FIELD_GET(HISI_PTT_SICL_ID, reg);

1029

1030 pmu_name = devm_kasprintf(&hisi_ptt->pdev->dev, GFP_KERNEL, "hisi_ptt%u_%u",

1031 sicl_id, core_id);

1032 if (!pmu_name)

1033 return -ENOMEM;

1034

1035 ret = perf_pmu_register(&hisi_ptt->hisi_ptt_pmu, pmu_name, -1);

1036 if (ret)

1037 return ret;

1038

1039 return devm_add_action_or_reset(&hisi_ptt->pdev->dev,

1040 hisi_ptt_unregister_pmu,

1041 &hisi_ptt->hisi_ptt_pmu);

1042 }

1043

1044 static void hisi_ptt_unregister_filter_update_notifier(void *data)

1045 {

1046 struct hisi_ptt *hisi_ptt = data;

1047

1048 bus_unregister_notifier(&pci_bus_type, &hisi_ptt->hisi_ptt_nb);

1049

1050 /* Cancel any work that has been queued */

1051 cancel_delayed_work_sync(&hisi_ptt->work);

1052 }

1053

1054 /* Register the bus notifier for dynamically updating the filter list */

1055 static int hisi_ptt_register_filter_update_notifier(struct hisi_ptt *hisi_ptt)

1056 {

1057 int ret;

1058

1059 hisi_ptt->hisi_ptt_nb.notifier_call = hisi_ptt_notifier_call;

1060 ret = bus_register_notifier(&pci_bus_type, &hisi_ptt->hisi_ptt_nb);

1061 if (ret)

1062 return ret;

1063

1064 return devm_add_action_or_reset(&hisi_ptt->pdev->dev,

1065 hisi_ptt_unregister_filter_update_notifier,

1066 hisi_ptt);

1067 }

1068

1069 /*

1070 * The DMA of PTT trace can only use direct mappings due to some

1071 * hardware restriction. Check whether there is no IOMMU or the

1072 * policy of the IOMMU domain is passthrough, otherwise the trace

1073 * cannot work.

1074 *

1075 * The PTT device is supposed to behind an ARM SMMUv3, which

1076 * should have passthrough the device by a quirk.

1077 */

1078 static int hisi_ptt_check_iommu_mapping(struct pci_dev *pdev)

1079 {

1080 struct iommu_domain *iommu_domain;

1081

1082 iommu_domain = iommu_get_domain_for_dev(&pdev->dev);

1083 if (!iommu_domain || iommu_domain->type == IOMMU_DOMAIN_IDENTITY)

1084 return 0;

1085

1086 return -EOPNOTSUPP;

1087 }

1088

1089 static int hisi_ptt_probe(struct pci_dev *pdev,

1090 const struct pci_device_id *id)

1091 {

1092 struct hisi_ptt *hisi_ptt;

1093 int ret;

1094

1095 ret = hisi_ptt_check_iommu_mapping(pdev);

1096 if (ret) {

1097 pci_err(pdev, "requires direct DMA mappings\n");

1098 return ret;

1099 }

1100

1101 hisi_ptt = devm_kzalloc(&pdev->dev, sizeof(*hisi_ptt), GFP_KERNEL);

1102 if (!hisi_ptt)

1103 return -ENOMEM;

1104

1105 hisi_ptt->pdev = pdev;

1106 pci_set_drvdata(pdev, hisi_ptt);

1107

1108 ret = pcim_enable_device(pdev);

1109 if (ret) {

1110 pci_err(pdev, "failed to enable device, ret = %d\n", ret);

1111 return ret;

1112 }

1113

1114 ret = pcim_iomap_regions(pdev, BIT(2), DRV_NAME);

1115 if (ret) {

1116 pci_err(pdev, "failed to remap io memory, ret = %d\n", ret);

1117 return ret;

1118 }

1119

1120 hisi_ptt->iobase = pcim_iomap_table(pdev)[2];

1121

1122 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));

1123 if (ret) {

1124 pci_err(pdev, "failed to set 64 bit dma mask, ret = %d\n", ret);

1125 return ret;

1126 }

1127

1128 pci_set_master(pdev);

1129

1130 ret = hisi_ptt_register_irq(hisi_ptt);

1131 if (ret)

1132 return ret;

1133

1134 ret = hisi_ptt_init_ctrls(hisi_ptt);

1135 if (ret) {

1136 pci_err(pdev, "failed to init controls, ret = %d\n", ret);

1137 return ret;

1138 }

1139

1140 ret = hisi_ptt_register_filter_update_notifier(hisi_ptt);

1141 if (ret)

1142 pci_warn(pdev, "failed to register filter update notifier, ret = %d", ret);

1143

1144 ret = hisi_ptt_register_pmu(hisi_ptt);

1145 if (ret) {

1146 pci_err(pdev, "failed to register PMU device, ret = %d", ret);

1147 return ret;

1148 }

1149

1150 return 0;

1151 }

1152

1153 static const struct pci_device_id hisi_ptt_id_tbl[] = {

1154 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa12e) },

1155 { }

1156 };

1157 MODULE_DEVICE_TABLE(pci, hisi_ptt_id_tbl);

1158

1159 static struct pci_driver hisi_ptt_driver = {

1160 .name = DRV_NAME,

1161 .id_table = hisi_ptt_id_tbl,

1162 .probe = hisi_ptt_probe,

1163 };

1164

1165 static int hisi_ptt_cpu_teardown(unsigned int cpu, struct hlist_node *node)

1166 {

1167 struct hisi_ptt *hisi_ptt;

1168 struct device *dev;

1169 int target, src;

1170

1171 hisi_ptt = hlist_entry_safe(node, struct hisi_ptt, hotplug_node);

1172 src = hisi_ptt->trace_ctrl.on_cpu;

1173 dev = hisi_ptt->hisi_ptt_pmu.dev;

1174

1175 if (!hisi_ptt->trace_ctrl.started || src != cpu)

1176 return 0;

1177

1178 target = cpumask_any_but(cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev)), cpu);

1179 if (target >= nr_cpu_ids) {

1180 dev_err(dev, "no available cpu for perf context migration\n");

1181 return 0;

1182 }

1183

1184 perf_pmu_migrate_context(&hisi_ptt->hisi_ptt_pmu, src, target);

1185

1186 /*

1187 * Also make sure the interrupt bind to the migrated CPU as well. Warn

1188 * the user on failure here.

1189 */

1190 if (irq_set_affinity(pci_irq_vector(hisi_ptt->pdev, HISI_PTT_TRACE_DMA_IRQ),

1191 cpumask_of(target)))

1192 dev_warn(dev, "failed to set the affinity of trace interrupt\n");

1193

1194 hisi_ptt->trace_ctrl.on_cpu = target;

1195 return 0;

1196 }

1197

1198 static int __init hisi_ptt_init(void)

1199 {

1200 int ret;

1201

1202 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRV_NAME, NULL,

1203 hisi_ptt_cpu_teardown);

1204 if (ret < 0)

1205 return ret;

1206 hisi_ptt_pmu_online = ret;

1207

1208 ret = pci_register_driver(&hisi_ptt_driver);

1209 if (ret)

1210 cpuhp_remove_multi_state(hisi_ptt_pmu_online);

1211

1212 return ret;

1213 }

1214 module_init(hisi_ptt_init);

1215

1216 static void __exit hisi_ptt_exit(void)

1217 {

1218 pci_unregister_driver(&hisi_ptt_driver);

1219 cpuhp_remove_multi_state(hisi_ptt_pmu_online);

1220 }

1221 module_exit(hisi_ptt_exit);

1222

1223 MODULE_LICENSE("GPL");

1224 MODULE_AUTHOR("Yicong Yang <yangyicong@hisilicon.com>");

1225 MODULE_DESCRIPTION("Driver for HiSilicon PCIe tune and trace device");