return MIN(split, mem_max - mem_off) + mem_off;
}
-#ifndef CONFIG_USER_ONLY
-/* These are normally defined only for CONFIG_USER_ONLY in <exec/cpu_ldst.h> */
-static inline void set_helper_retaddr(uintptr_t ra) { }
-static inline void clear_helper_retaddr(void) { }
-#endif
-
/*
* The result of tlb_vaddr_to_host for user-only is just g2h(x),
* which is always non-null. Elide the useless test.
return;
}
mem_off = reg_off >> diffsz;
- set_helper_retaddr(retaddr);
/*
* If the (remaining) load is entirely within a single page, then:
if (test_host_page(host)) {
mem_off = host_fn(vd, vg, host - mem_off, mem_off, mem_max);
tcg_debug_assert(mem_off == mem_max);
- clear_helper_retaddr();
/* After having taken any fault, zero leading inactive elements. */
swap_memzero(vd, reg_off);
return;
}
#endif
- clear_helper_retaddr();
memcpy(vd, &scratch, reg_max);
}
intptr_t i, oprsz = simd_oprsz(desc);
ARMVectorReg scratch[2] = { };
- set_helper_retaddr(ra);
for (i = 0; i < oprsz; ) {
uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));
do {
addr += 2 * size;
} while (i & 15);
}
- clear_helper_retaddr();
/* Wait until all exceptions have been raised to write back. */
memcpy(&env->vfp.zregs[rd], &scratch[0], oprsz);
intptr_t i, oprsz = simd_oprsz(desc);
ARMVectorReg scratch[3] = { };
- set_helper_retaddr(ra);
for (i = 0; i < oprsz; ) {
uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));
do {
addr += 3 * size;
} while (i & 15);
}
- clear_helper_retaddr();
/* Wait until all exceptions have been raised to write back. */
memcpy(&env->vfp.zregs[rd], &scratch[0], oprsz);
intptr_t i, oprsz = simd_oprsz(desc);
ARMVectorReg scratch[4] = { };
- set_helper_retaddr(ra);
for (i = 0; i < oprsz; ) {
uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));
do {
addr += 4 * size;
} while (i & 15);
}
- clear_helper_retaddr();
/* Wait until all exceptions have been raised to write back. */
memcpy(&env->vfp.zregs[rd], &scratch[0], oprsz);
return;
}
mem_off = reg_off >> diffsz;
- set_helper_retaddr(retaddr);
/*
* If the (remaining) load is entirely within a single page, then:
if (test_host_page(host)) {
mem_off = host_fn(vd, vg, host - mem_off, mem_off, mem_max);
tcg_debug_assert(mem_off == mem_max);
- clear_helper_retaddr();
/* After any fault, zero any leading inactive elements. */
swap_memzero(vd, reg_off);
return;
}
#endif
- clear_helper_retaddr();
record_fault(env, reg_off, reg_max);
}
intptr_t i, oprsz = simd_oprsz(desc);
void *vd = &env->vfp.zregs[rd];
- set_helper_retaddr(ra);
for (i = 0; i < oprsz; ) {
uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));
do {
addr += msize;
} while (i & 15);
}
- clear_helper_retaddr();
}
static void sve_st2_r(CPUARMState *env, void *vg, target_ulong addr,
void *d1 = &env->vfp.zregs[rd];
void *d2 = &env->vfp.zregs[(rd + 1) & 31];
- set_helper_retaddr(ra);
for (i = 0; i < oprsz; ) {
uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));
do {
addr += 2 * msize;
} while (i & 15);
}
- clear_helper_retaddr();
}
static void sve_st3_r(CPUARMState *env, void *vg, target_ulong addr,
void *d2 = &env->vfp.zregs[(rd + 1) & 31];
void *d3 = &env->vfp.zregs[(rd + 2) & 31];
- set_helper_retaddr(ra);
for (i = 0; i < oprsz; ) {
uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));
do {
addr += 3 * msize;
} while (i & 15);
}
- clear_helper_retaddr();
}
static void sve_st4_r(CPUARMState *env, void *vg, target_ulong addr,
void *d3 = &env->vfp.zregs[(rd + 2) & 31];
void *d4 = &env->vfp.zregs[(rd + 3) & 31];
- set_helper_retaddr(ra);
for (i = 0; i < oprsz; ) {
uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));
do {
addr += 4 * msize;
} while (i & 15);
}
- clear_helper_retaddr();
}
#define DO_STN_1(N, NAME, ESIZE) \
intptr_t i, oprsz = simd_oprsz(desc);
ARMVectorReg scratch = { };
- set_helper_retaddr(ra);
for (i = 0; i < oprsz; ) {
uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));
do {
i += 4, pg >>= 4;
} while (i & 15);
}
- clear_helper_retaddr();
/* Wait until all exceptions have been raised to write back. */
memcpy(vd, &scratch, oprsz);
intptr_t i, oprsz = simd_oprsz(desc) / 8;
ARMVectorReg scratch = { };
- set_helper_retaddr(ra);
for (i = 0; i < oprsz; i++) {
uint8_t pg = *(uint8_t *)(vg + H1(i));
if (likely(pg & 1)) {
tlb_fn(env, &scratch, i * 8, base + (off << scale), ra);
}
}
- clear_helper_retaddr();
/* Wait until all exceptions have been raised to write back. */
memcpy(vd, &scratch, oprsz * 8);
reg_off = find_next_active(vg, 0, reg_max, MO_32);
if (likely(reg_off < reg_max)) {
/* Perform one normal read, which will fault or not. */
- set_helper_retaddr(ra);
addr = off_fn(vm, reg_off);
addr = base + (addr << scale);
tlb_fn(env, vd, reg_off, addr, ra);
/* The rest of the reads will be non-faulting. */
- clear_helper_retaddr();
}
/* After any fault, zero the leading predicated false elements. */
reg_off = find_next_active(vg, 0, reg_max, MO_64);
if (likely(reg_off < reg_max)) {
/* Perform one normal read, which will fault or not. */
- set_helper_retaddr(ra);
addr = off_fn(vm, reg_off);
addr = base + (addr << scale);
tlb_fn(env, vd, reg_off, addr, ra);
/* The rest of the reads will be non-faulting. */
- clear_helper_retaddr();
}
/* After any fault, zero the leading predicated false elements. */
const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
intptr_t i, oprsz = simd_oprsz(desc);
- set_helper_retaddr(ra);
for (i = 0; i < oprsz; ) {
uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3));
do {
i += 4, pg >>= 4;
} while (i & 15);
}
- clear_helper_retaddr();
}
static void sve_st1_zd(CPUARMState *env, void *vd, void *vg, void *vm,
const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
intptr_t i, oprsz = simd_oprsz(desc) / 8;
- set_helper_retaddr(ra);
for (i = 0; i < oprsz; i++) {
uint8_t pg = *(uint8_t *)(vg + H1(i));
if (likely(pg & 1)) {
tlb_fn(env, vd, i * 8, base + (off << scale), ra);
}
}
- clear_helper_retaddr();
}
#define DO_ST1_ZPZ_S(MEM, OFS) \