tcg_out_goto(s, lb->raddr);
return true;
}
-
-static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi,
- TCGType ext, TCGReg data_reg, TCGReg addr_reg,
- tcg_insn_unit *raddr, tcg_insn_unit *label_ptr)
+#else
+static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l)
{
- TCGLabelQemuLdst *label = new_ldst_label(s);
+ if (!reloc_pc19(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
+ return false;
+ }
+
+ tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_X1, l->addrlo_reg);
+ tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);
- label->is_ld = is_ld;
- label->oi = oi;
- label->type = ext;
- label->datalo_reg = data_reg;
- label->addrlo_reg = addr_reg;
- label->raddr = tcg_splitwx_to_rx(raddr);
- label->label_ptr[0] = label_ptr;
+ /* "Tail call" to the helper, with the return address back inline. */
+ tcg_out_adr(s, TCG_REG_LR, l->raddr);
+ tcg_out_goto_long(s, (const void *)(l->is_ld ? helper_unaligned_ld
+ : helper_unaligned_st));
+ return true;
}
-/* We expect to use a 7-bit scaled negative offset from ENV. */
-QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0);
-QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -512);
+static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
+{
+ return tcg_out_fail_alignment(s, l);
+}
-/* These offsets are built into the LDP below. */
-QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, mask) != 0);
-QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, table) != 8);
+static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
+{
+ return tcg_out_fail_alignment(s, l);
+}
+#endif /* CONFIG_SOFTMMU */
-/* Load and compare a TLB entry, emitting the conditional jump to the
- slow path for the failure case, which will be patched later when finalizing
- the slow path. Generated code returns the host addend in X1,
- clobbers X0,X2,X3,TMP. */
-static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, MemOp opc,
- tcg_insn_unit **label_ptr, int mem_index,
- bool is_read)
+/*
+ * For softmmu, perform the TLB load and compare.
+ * For useronly, perform any required alignment tests.
+ * In both cases, return a TCGLabelQemuLdst structure if the slow path
+ * is required and fill in @h with the host address for the fast path.
+ */
+static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h,
+ TCGReg addr_reg, MemOpIdx oi,
+ bool is_ld)
{
+ TCGType addr_type = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
+ TCGLabelQemuLdst *ldst = NULL;
+ MemOp opc = get_memop(oi);
unsigned a_bits = get_alignment_bits(opc);
- unsigned s_bits = opc & MO_SIZE;
unsigned a_mask = (1u << a_bits) - 1;
+
+#ifdef CONFIG_SOFTMMU
+ unsigned s_bits = opc & MO_SIZE;
unsigned s_mask = (1u << s_bits) - 1;
+ unsigned mem_index = get_mmuidx(oi);
TCGReg x3;
TCGType mask_type;
uint64_t compare_mask;
+ ldst = new_ldst_label(s);
+ ldst->is_ld = is_ld;
+ ldst->oi = oi;
+ ldst->addrlo_reg = addr_reg;
+
mask_type = (TARGET_PAGE_BITS + CPU_TLB_DYN_MAX_BITS > 32
? TCG_TYPE_I64 : TCG_TYPE_I32);
/* Load env_tlb(env)->f[mmu_idx].{mask,table} into {x0,x1}. */
+ QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0);
+ QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -512);
+ QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, mask) != 0);
+ QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, table) != 8);
tcg_out_insn(s, 3314, LDP, TCG_REG_X0, TCG_REG_X1, TCG_AREG0,
TLB_MASK_TABLE_OFS(mem_index), 1, 0);
tcg_out_insn(s, 3502, ADD, 1, TCG_REG_X1, TCG_REG_X1, TCG_REG_X0);
/* Load the tlb comparator into X0, and the fast path addend into X1. */
- tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_X0, TCG_REG_X1, is_read
- ? offsetof(CPUTLBEntry, addr_read)
- : offsetof(CPUTLBEntry, addr_write));
+ tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_X0, TCG_REG_X1,
+ is_ld ? offsetof(CPUTLBEntry, addr_read)
+ : offsetof(CPUTLBEntry, addr_write));
tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_X1, TCG_REG_X1,
offsetof(CPUTLBEntry, addend));
- /* For aligned accesses, we check the first byte and include the alignment
- bits within the address. For unaligned access, we check that we don't
- cross pages using the address of the last byte of the access. */
+ /*
+ * For aligned accesses, we check the first byte and include the alignment
+ * bits within the address. For unaligned access, we check that we don't
+ * cross pages using the address of the last byte of the access.
+ */
if (a_bits >= s_bits) {
x3 = addr_reg;
} else {
tcg_out_cmp(s, TARGET_LONG_BITS == 64, TCG_REG_X0, TCG_REG_X3, 0);
/* If not equal, we jump to the slow path. */
- *label_ptr = s->code_ptr;
+ ldst->label_ptr[0] = s->code_ptr;
tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0);
-}
+ *h = (HostAddress){
+ .base = TCG_REG_X1,
+ .index = addr_reg,
+ .index_ext = addr_type
+ };
#else
-static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addr_reg,
- unsigned a_bits)
-{
- unsigned a_mask = (1 << a_bits) - 1;
- TCGLabelQemuLdst *label = new_ldst_label(s);
-
- label->is_ld = is_ld;
- label->addrlo_reg = addr_reg;
+ if (a_mask) {
+ ldst = new_ldst_label(s);
- /* tst addr, #mask */
- tcg_out_logicali(s, I3404_ANDSI, 0, TCG_REG_XZR, addr_reg, a_mask);
+ ldst->is_ld = is_ld;
+ ldst->oi = oi;
+ ldst->addrlo_reg = addr_reg;
- label->label_ptr[0] = s->code_ptr;
-
- /* b.ne slow_path */
- tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0);
+ /* tst addr, #mask */
+ tcg_out_logicali(s, I3404_ANDSI, 0, TCG_REG_XZR, addr_reg, a_mask);
- label->raddr = tcg_splitwx_to_rx(s->code_ptr);
-}
-
-static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l)
-{
- if (!reloc_pc19(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
- return false;
+ /* b.ne slow_path */
+ ldst->label_ptr[0] = s->code_ptr;
+ tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0);
}
- tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_X1, l->addrlo_reg);
- tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);
-
- /* "Tail call" to the helper, with the return address back inline. */
- tcg_out_adr(s, TCG_REG_LR, l->raddr);
- tcg_out_goto_long(s, (const void *)(l->is_ld ? helper_unaligned_ld
- : helper_unaligned_st));
- return true;
-}
-
-static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
-{
- return tcg_out_fail_alignment(s, l);
-}
+ if (USE_GUEST_BASE) {
+ *h = (HostAddress){
+ .base = TCG_REG_GUEST_BASE,
+ .index = addr_reg,
+ .index_ext = addr_type
+ };
+ } else {
+ *h = (HostAddress){
+ .base = addr_reg,
+ .index = TCG_REG_XZR,
+ .index_ext = TCG_TYPE_I64
+ };
+ }
+#endif
-static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
-{
- return tcg_out_fail_alignment(s, l);
+ return ldst;
}
-#endif /* CONFIG_SOFTMMU */
static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp memop, TCGType ext,
TCGReg data_r, HostAddress h)
static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
MemOpIdx oi, TCGType data_type)
{
- MemOp memop = get_memop(oi);
- TCGType addr_type = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
+ TCGLabelQemuLdst *ldst;
HostAddress h;
- /* Byte swapping is left to middle-end expansion. */
- tcg_debug_assert((memop & MO_BSWAP) == 0);
+ ldst = prepare_host_addr(s, &h, addr_reg, oi, true);
+ tcg_out_qemu_ld_direct(s, get_memop(oi), data_type, data_reg, h);
-#ifdef CONFIG_SOFTMMU
- tcg_insn_unit *label_ptr;
-
- tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, get_mmuidx(oi), 1);
-
- h = (HostAddress){
- .base = TCG_REG_X1,
- .index = addr_reg,
- .index_ext = addr_type
- };
- tcg_out_qemu_ld_direct(s, memop, data_type, data_reg, h);
-
- add_qemu_ldst_label(s, true, oi, data_type, data_reg, addr_reg,
- s->code_ptr, label_ptr);
-#else /* !CONFIG_SOFTMMU */
- unsigned a_bits = get_alignment_bits(memop);
- if (a_bits) {
- tcg_out_test_alignment(s, true, addr_reg, a_bits);
- }
- if (USE_GUEST_BASE) {
- h = (HostAddress){
- .base = TCG_REG_GUEST_BASE,
- .index = addr_reg,
- .index_ext = addr_type
- };
- } else {
- h = (HostAddress){
- .base = addr_reg,
- .index = TCG_REG_XZR,
- .index_ext = TCG_TYPE_I64
- };
+ if (ldst) {
+ ldst->type = data_type;
+ ldst->datalo_reg = data_reg;
+ ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
}
- tcg_out_qemu_ld_direct(s, memop, data_type, data_reg, h);
-#endif /* CONFIG_SOFTMMU */
}
static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
MemOpIdx oi, TCGType data_type)
{
- MemOp memop = get_memop(oi);
- TCGType addr_type = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
+ TCGLabelQemuLdst *ldst;
HostAddress h;
- /* Byte swapping is left to middle-end expansion. */
- tcg_debug_assert((memop & MO_BSWAP) == 0);
-
-#ifdef CONFIG_SOFTMMU
- tcg_insn_unit *label_ptr;
-
- tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, get_mmuidx(oi), 0);
-
- h = (HostAddress){
- .base = TCG_REG_X1,
- .index = addr_reg,
- .index_ext = addr_type
- };
- tcg_out_qemu_st_direct(s, memop, data_reg, h);
+ ldst = prepare_host_addr(s, &h, addr_reg, oi, false);
+ tcg_out_qemu_st_direct(s, get_memop(oi), data_reg, h);
- add_qemu_ldst_label(s, false, oi, data_type, data_reg, addr_reg,
- s->code_ptr, label_ptr);
-#else /* !CONFIG_SOFTMMU */
- unsigned a_bits = get_alignment_bits(memop);
- if (a_bits) {
- tcg_out_test_alignment(s, false, addr_reg, a_bits);
+ if (ldst) {
+ ldst->type = data_type;
+ ldst->datalo_reg = data_reg;
+ ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
}
- if (USE_GUEST_BASE) {
- h = (HostAddress){
- .base = TCG_REG_GUEST_BASE,
- .index = addr_reg,
- .index_ext = addr_type
- };
- } else {
- h = (HostAddress){
- .base = addr_reg,
- .index = TCG_REG_XZR,
- .index_ext = TCG_TYPE_I64
- };
- }
- tcg_out_qemu_st_direct(s, memop, data_reg, h);
-#endif /* CONFIG_SOFTMMU */
}
static const tcg_insn_unit *tb_ret_addr;
projects / qemu.git / commitdiff