codegen_ssa: multiply scalable vec size by vscale

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1459,6 +1459,10 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         );
     }
 
+    fn vscale(&mut self, _: Self::Type) -> Self::Value {
+        unimplemented!("`rustc_codegen_gcc` doesn't support scalable vectors yet")
+    }
+
     fn select(
         &mut self,
         cond: RValue<'gcc>,

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1226,6 +1226,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         }
     }
 
+    fn vscale(&mut self, ty: &'ll Type) -> &'ll Value {
+        unsafe { llvm::LLVMRustBuildVScale(self.llbuilder, ty) }
+    }
+
     fn select(
         &mut self,
         cond: &'ll Value,

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -2139,6 +2139,8 @@ unsafe extern "C" {
         IsVolatile: bool,
     ) -> &'a Value;
 
+    pub(crate) fn LLVMRustBuildVScale<'a>(B: &Builder<'a>, Ty: &'a Type) -> &'a Value;
+
     pub(crate) fn LLVMRustTimeTraceProfilerInitialize();
 
     pub(crate) fn LLVMRustTimeTraceProfilerFinishThread();

compiler/rustc_codegen_ssa/src/traits/builder.rs

@@ -476,6 +476,11 @@ pub trait BuilderMethods<'a, 'tcx>:
         flags: MemFlags,
     );
 
+    // Produce a value from calling the `vscale` intrinsic (containing the `vscale` multiplier that
+    // a scalable vector's element size and count can be multiplied by to get the real size of the
+    // vector)
+    fn vscale(&mut self, ty: Self::Type) -> Self::Value;
+
     /// *Typed* copy for non-overlapping places.
     ///
     /// Has a default implementation in terms of `memcpy`, but specific backends
@@ -513,6 +518,12 @@ pub trait BuilderMethods<'a, 'tcx>:
             temp.val.store_with_flags(self, dst.with_type(layout), flags);
         } else if !layout.is_zst() {
             let bytes = self.const_usize(layout.size.bytes());
+            let bytes = if layout.peel_transparent_wrappers(self).ty.is_scalable_vector() {
+                let vscale = self.vscale(self.type_i64());
+                self.mul(vscale, bytes)
+            } else {
+                bytes
+            };
             self.memcpy(dst.llval, dst.align, src.llval, src.align, bytes, flags, None);
         }
     }

compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

@@ -1497,6 +1497,10 @@ LLVMRustBuildMemMove(LLVMBuilderRef B, LLVMValueRef Dst, unsigned DstAlign,
                                        unwrap(Size), IsVolatile));
 }
 
+extern "C" LLVMValueRef LLVMRustBuildVScale(LLVMBuilderRef B, LLVMTypeRef Ty) {
+  return wrap(unwrap(B)->CreateVScale(unwrap(Ty)));
+}
+
 extern "C" LLVMValueRef LLVMRustBuildMemSet(LLVMBuilderRef B, LLVMValueRef Dst,
                                             unsigned DstAlign, LLVMValueRef Val,
                                             LLVMValueRef Size,

tests/codegen-llvm/scalable-vectors/memcpy.rs

@@ -0,0 +1,88 @@
+//@ compile-flags: -Copt-level=0 -Ctarget-feature=+sve,+sve2
+//@ edition: 2021
+//@ only-aarch64
+
+#![crate_type = "lib"]
+#![feature(simd_ffi)]
+#![feature(stdarch_aarch64_sve)]
+
+// Test that `vscale * size` is generated for `memcpy` of scalable vector types
+
+use std::arch::aarch64::*;
+
+#[allow(improper_ctypes)]
+unsafe extern "C" {
+    fn svcreate2_s16_wrapper(__dst: *mut svint16x2_t, x0: *const svint16_t, x1: *const svint16_t);
+    fn svcreate3_s16_wrapper(
+        __dst: *mut svint16x3_t,
+        x0: *const svint16_t,
+        x1: *const svint16_t,
+        x2: *const svint16_t,
+    );
+    fn svcreate4_s16_wrapper(
+        __dst: *mut svint16x4_t,
+        x0: *const svint16_t,
+        x1: *const svint16_t,
+        x2: *const svint16_t,
+        x3: *const svint16_t,
+    );
+}
+
+pub fn foo(x0_val: svint16_t) {
+    unsafe {
+        let __pred = svptrue_b16();
+
+        let mut __c_return_value = std::mem::MaybeUninit::uninit();
+        svcreate2_s16_wrapper(__c_return_value.as_mut_ptr(), &raw const x0_val, &raw const x0_val);
+        let __c_return_value = __c_return_value.assume_init();
+        // CHECK: call void @svcreate2_s16_wrapper(
+        // CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64()
+        // CHECK-NEXT: [[VSCALE_SIZE:%.*]] = mul i64 [[VSCALE]], 32
+        // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64({{.*}}, {{.*}}, i64 [[VSCALE_SIZE]]
+
+        let eq = svcmpeq_s16(
+            __pred,
+            svget2_s16::<1>(__c_return_value),
+            svget2_s16::<1>(__c_return_value),
+        );
+
+        let mut __c_return_value = std::mem::MaybeUninit::uninit();
+        svcreate3_s16_wrapper(
+            __c_return_value.as_mut_ptr(),
+            &raw const x0_val,
+            &raw const x0_val,
+            &raw const x0_val,
+        );
+        let __c_return_value = __c_return_value.assume_init();
+        // CHECK: call void @svcreate3_s16_wrapper(
+        // CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64()
+        // CHECK-NEXT: [[VSCALE_SIZE:%.*]] = mul i64 [[VSCALE]], 48
+        // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64({{.*}}, {{.*}}, i64 [[VSCALE_SIZE]]
+
+        let eq = svcmpeq_s16(
+            __pred,
+            svget3_s16::<2>(__c_return_value),
+            svget3_s16::<2>(__c_return_value),
+        );
+
+        let mut __c_return_value = std::mem::MaybeUninit::uninit();
+        svcreate4_s16_wrapper(
+            __c_return_value.as_mut_ptr(),
+            &raw const x0_val,
+            &raw const x0_val,
+            &raw const x0_val,
+            &raw const x0_val,
+        );
+        let __c_return_value = __c_return_value.assume_init();
+        // CHECK: call void @svcreate4_s16_wrapper(
+        // CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64()
+        // CHECK-NEXT: [[VSCALE_SIZE:%.*]] = mul i64 [[VSCALE]], 64
+        // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64({{.*}}, {{.*}}, i64 [[VSCALE_SIZE]]
+
+        let eq = svcmpeq_s16(
+            __pred,
+            svget4_s16::<3>(__c_return_value),
+            svget4_s16::<3>(__c_return_value),
+        );
+    }
+}