diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 33f0f6fc2f809..8ae4dedff8f28 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -1459,6 +1459,10 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { ); } + fn vscale(&mut self, _: Self::Type) -> Self::Value { + unimplemented!("`rustc_codegen_gcc` doesn't support scalable vectors yet") + } + fn select( &mut self, cond: RValue<'gcc>, diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index acfec99438059..afb6985d21a95 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1226,6 +1226,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } + fn vscale(&mut self, ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVScale(self.llbuilder, ty) } + } + fn select( &mut self, cond: &'ll Value, diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index b14a6206ae682..dd527eafe8362 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2139,6 +2139,8 @@ unsafe extern "C" { IsVolatile: bool, ) -> &'a Value; + pub(crate) fn LLVMRustBuildVScale<'a>(B: &Builder<'a>, Ty: &'a Type) -> &'a Value; + pub(crate) fn LLVMRustTimeTraceProfilerInitialize(); pub(crate) fn LLVMRustTimeTraceProfilerFinishThread(); diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 363e47aa60fa1..d68549c6871f4 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -476,6 +476,11 @@ pub trait BuilderMethods<'a, 'tcx>: flags: MemFlags, ); + // Produce a value from calling the `vscale` intrinsic (containing the `vscale` multiplier that + // a scalable vector's element size and count can be multiplied by to get the real size of the + // vector) + fn vscale(&mut self, ty: Self::Type) -> Self::Value; + /// *Typed* copy for non-overlapping places. /// /// Has a default implementation in terms of `memcpy`, but specific backends @@ -513,6 +518,12 @@ pub trait BuilderMethods<'a, 'tcx>: temp.val.store_with_flags(self, dst.with_type(layout), flags); } else if !layout.is_zst() { let bytes = self.const_usize(layout.size.bytes()); + let bytes = if layout.peel_transparent_wrappers(self).ty.is_scalable_vector() { + let vscale = self.vscale(self.type_i64()); + self.mul(vscale, bytes) + } else { + bytes + }; self.memcpy(dst.llval, dst.align, src.llval, src.align, bytes, flags, None); } } diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index a50b06e5120b2..8b063af187a58 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1497,6 +1497,10 @@ LLVMRustBuildMemMove(LLVMBuilderRef B, LLVMValueRef Dst, unsigned DstAlign, unwrap(Size), IsVolatile)); } +extern "C" LLVMValueRef LLVMRustBuildVScale(LLVMBuilderRef B, LLVMTypeRef Ty) { + return wrap(unwrap(B)->CreateVScale(unwrap(Ty))); +} + extern "C" LLVMValueRef LLVMRustBuildMemSet(LLVMBuilderRef B, LLVMValueRef Dst, unsigned DstAlign, LLVMValueRef Val, LLVMValueRef Size, diff --git a/tests/codegen-llvm/scalable-vectors/memcpy.rs b/tests/codegen-llvm/scalable-vectors/memcpy.rs new file mode 100644 index 0000000000000..859f50b5b1178 --- /dev/null +++ b/tests/codegen-llvm/scalable-vectors/memcpy.rs @@ -0,0 +1,88 @@ +//@ compile-flags: -Copt-level=0 -Ctarget-feature=+sve,+sve2 +//@ edition: 2021 +//@ only-aarch64 + +#![crate_type = "lib"] +#![feature(simd_ffi)] +#![feature(stdarch_aarch64_sve)] + +// Test that `vscale * size` is generated for `memcpy` of scalable vector types + +use std::arch::aarch64::*; + +#[allow(improper_ctypes)] +unsafe extern "C" { + fn svcreate2_s16_wrapper(__dst: *mut svint16x2_t, x0: *const svint16_t, x1: *const svint16_t); + fn svcreate3_s16_wrapper( + __dst: *mut svint16x3_t, + x0: *const svint16_t, + x1: *const svint16_t, + x2: *const svint16_t, + ); + fn svcreate4_s16_wrapper( + __dst: *mut svint16x4_t, + x0: *const svint16_t, + x1: *const svint16_t, + x2: *const svint16_t, + x3: *const svint16_t, + ); +} + +pub fn foo(x0_val: svint16_t) { + unsafe { + let __pred = svptrue_b16(); + + let mut __c_return_value = std::mem::MaybeUninit::uninit(); + svcreate2_s16_wrapper(__c_return_value.as_mut_ptr(), &raw const x0_val, &raw const x0_val); + let __c_return_value = __c_return_value.assume_init(); + // CHECK: call void @svcreate2_s16_wrapper( + // CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64() + // CHECK-NEXT: [[VSCALE_SIZE:%.*]] = mul i64 [[VSCALE]], 32 + // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64({{.*}}, {{.*}}, i64 [[VSCALE_SIZE]] + + let eq = svcmpeq_s16( + __pred, + svget2_s16::<1>(__c_return_value), + svget2_s16::<1>(__c_return_value), + ); + + let mut __c_return_value = std::mem::MaybeUninit::uninit(); + svcreate3_s16_wrapper( + __c_return_value.as_mut_ptr(), + &raw const x0_val, + &raw const x0_val, + &raw const x0_val, + ); + let __c_return_value = __c_return_value.assume_init(); + // CHECK: call void @svcreate3_s16_wrapper( + // CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64() + // CHECK-NEXT: [[VSCALE_SIZE:%.*]] = mul i64 [[VSCALE]], 48 + // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64({{.*}}, {{.*}}, i64 [[VSCALE_SIZE]] + + let eq = svcmpeq_s16( + __pred, + svget3_s16::<2>(__c_return_value), + svget3_s16::<2>(__c_return_value), + ); + + let mut __c_return_value = std::mem::MaybeUninit::uninit(); + svcreate4_s16_wrapper( + __c_return_value.as_mut_ptr(), + &raw const x0_val, + &raw const x0_val, + &raw const x0_val, + &raw const x0_val, + ); + let __c_return_value = __c_return_value.assume_init(); + // CHECK: call void @svcreate4_s16_wrapper( + // CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64() + // CHECK-NEXT: [[VSCALE_SIZE:%.*]] = mul i64 [[VSCALE]], 64 + // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64({{.*}}, {{.*}}, i64 [[VSCALE_SIZE]] + + let eq = svcmpeq_s16( + __pred, + svget4_s16::<3>(__c_return_value), + svget4_s16::<3>(__c_return_value), + ); + } +}