Rollup of 15 pull requests

compiler/rustc_abi/src/extern_abi.rs

@@ -343,10 +343,16 @@ impl ExternAbi {
                 // This ABI does not support calls at all (except via assembly).
                 false
             }
+            Self::RustCall => {
+                // Argument untupling requires additional support for tail calls to be possible,
+                // see <https://github.com/rust-lang/rust/pull/158248>. There is no real uses of
+                // tail calling `extern "rust-call"` functions
+                false
+            }
+
             Self::C { .. }
             | Self::System { .. }
             | Self::Rust
-            | Self::RustCall
             | Self::RustCold
             | Self::RustInvalid
             | Self::Unadjusted

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1459,6 +1459,10 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         );
     }
 
+    fn vscale(&mut self, _: Self::Type) -> Self::Value {
+        unimplemented!("`rustc_codegen_gcc` doesn't support scalable vectors yet")
+    }
+
     fn select(
         &mut self,
         cond: RValue<'gcc>,

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1226,6 +1226,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         }
     }
 
+    fn vscale(&mut self, ty: &'ll Type) -> &'ll Value {
+        unsafe { llvm::LLVMRustBuildVScale(self.llbuilder, ty) }
+    }
+
     fn select(
         &mut self,
         cond: &'ll Value,

compiler/rustc_codegen_llvm/src/builder/autodiff.rs

@@ -3,6 +3,9 @@ use std::ptr;
 use rustc_ast::expand::autodiff_attrs::{DiffActivity, DiffMode};
 use rustc_ast::expand::typetree::FncTree;
 use rustc_codegen_ssa::common::TypeKind;
+use rustc_codegen_ssa::mir::IntrinsicResult;
+use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
+use rustc_codegen_ssa::mir::place::PlaceValue;
 use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods};
 use rustc_data_structures::thin_vec::ThinVec;
 use rustc_hir::attrs::RustcAutodiff;
@@ -11,7 +14,7 @@ use rustc_middle::{bug, ty};
 use rustc_target::callconv::PassMode;
 use tracing::debug;
 
-use crate::builder::{Builder, PlaceRef, UNNAMED};
+use crate::builder::{Builder, UNNAMED};
 use crate::context::SimpleCx;
 use crate::declare::declare_simple_fn;
 use crate::llvm::{self, TRUE, Type, Value};
@@ -296,9 +299,10 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>(
     ret_ty: &'ll Type,
     fn_args: &[&'ll Value],
     attrs: &RustcAutodiff,
-    dest: PlaceRef<'tcx, &'ll Value>,
+    dest_layout: ty::layout::TyAndLayout<'tcx>,
+    dest_place: Option<PlaceValue<&'ll Value>>,
     fnc_tree: FncTree,
-) {
+) -> IntrinsicResult<'tcx, &'ll Value> {
     // We have to pick the name depending on whether we want forward or reverse mode autodiff.
     let mut ad_name: String = match attrs.mode {
         DiffMode::Forward => "__enzyme_fwddiff",
@@ -381,11 +385,18 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>(
     let call = builder.call(enzyme_ty, None, None, ad_fn, &args, None, None);
 
     let fn_ret_ty = builder.cx.val_ty(call);
-    if fn_ret_ty != builder.cx.type_void() && fn_ret_ty != builder.cx.type_struct(&[], false) {
+    if fn_ret_ty == builder.cx.type_void() || fn_ret_ty == builder.cx.type_struct(&[], false) {
         // If we return void or an empty struct, then our caller (due to how we generated it)
         // does not expect a return value. As such, we have no pointer (or place) into which
         // we could store our value, and would store into an undef, which would cause UB.
         // As such, we just ignore the return value in those cases.
-        builder.store_to_place(call, dest.val);
+        IntrinsicResult::Operand(OperandValue::ZeroSized)
+    } else if let Some(dest_place) = dest_place {
+        builder.store_to_place(call, dest_place);
+        IntrinsicResult::WroteIntoPlace
+    } else {
+        IntrinsicResult::Operand(
+            OperandRef::from_immediate_or_packed_pair(builder, call, dest_layout).val,
+        )
     }
 }

compiler/rustc_codegen_llvm/src/context.rs

@@ -978,10 +978,7 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     }
 
     fn intrinsic_call_expects_place_always(&self, name: Symbol) -> bool {
-        matches!(
-            name,
-            sym::autodiff | sym::volatile_load | sym::unaligned_volatile_load | sym::black_box
-        )
+        matches!(name, sym::volatile_load | sym::unaligned_volatile_load | sym::black_box)
     }
 }
 

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -43,6 +43,7 @@ use crate::errors::{
     AutoDiffWithoutEnable, AutoDiffWithoutLto, IntrinsicSignatureMismatch, IntrinsicWrongArch,
     OffloadWithoutEnable, OffloadWithoutFatLTO, UnknownIntrinsic,
 };
+use crate::intrinsic::ty::typetree::fnc_typetrees;
 use crate::llvm::{self, Type, Value};
 use crate::type_of::LayoutLlvmExt;
 use crate::va_arg::emit_va_arg;
@@ -223,12 +224,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 )
             }
             sym::autodiff => {
-                let result = PlaceRef {
-                    val: result_place.unwrap(),
-                    layout: result_layout,
-                };
-                codegen_autodiff(self, tcx, instance, args, result);
-                return IntrinsicResult::WroteIntoPlace;
+                return codegen_autodiff(self, tcx, instance, args, result_layout, result_place);
             }
             sym::offload => {
                 if tcx.sess.opts.unstable_opts.offload.is_empty() {
@@ -1728,8 +1724,9 @@ fn codegen_autodiff<'ll, 'tcx>(
     tcx: TyCtxt<'tcx>,
     instance: ty::Instance<'tcx>,
     args: &[OperandRef<'tcx, &'ll Value>],
-    result: PlaceRef<'tcx, &'ll Value>,
-) {
+    result_layout: ty::layout::TyAndLayout<'tcx>,
+    result_place: Option<PlaceValue<&'ll Value>>,
+) -> IntrinsicResult<'tcx, &'ll Value> {
     if !tcx.sess.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable) {
         let _ = tcx.dcx().emit_almost_fatal(AutoDiffWithoutEnable);
     }
@@ -1769,9 +1766,9 @@ fn codegen_autodiff<'ll, 'tcx>(
             diff_id,
             diff_args
         ),
-        Err(_) => {
+        Err(err) => {
             // An error has already been emitted
-            return;
+            return IntrinsicResult::Err(err);
         }
     };
 
@@ -1791,7 +1788,7 @@ fn codegen_autodiff<'ll, 'tcx>(
         &mut diff_attrs.input_activity,
     );
 
-    let fnc_tree = rustc_middle::ty::fnc_typetrees(tcx, source_fn_ptr_ty);
+    let fnc_tree = fnc_typetrees(tcx, source_fn_ptr_ty);
 
     // Build body
     generate_enzyme_call(
@@ -1802,9 +1799,10 @@ fn codegen_autodiff<'ll, 'tcx>(
         llret_ty,
         &val_arr,
         &diff_attrs,
-        result,
+        result_layout,
+        result_place,
         fnc_tree,
-    );
+    )
 }
 
 // Generates the LLVM code to offload a Rust function to a target device (e.g., GPU).

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -2139,6 +2139,8 @@ unsafe extern "C" {
         IsVolatile: bool,
     ) -> &'a Value;
 
+    pub(crate) fn LLVMRustBuildVScale<'a>(B: &Builder<'a>, Ty: &'a Type) -> &'a Value;
+
     pub(crate) fn LLVMRustTimeTraceProfilerInitialize();
 
     pub(crate) fn LLVMRustTimeTraceProfilerFinishThread();

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -47,7 +47,7 @@ use rustc_session::{Session, filesearch};
 use rustc_span::Symbol;
 use rustc_target::spec::crt_objects::CrtObjects;
 use rustc_target::spec::{
-    BinaryFormat, Cc, CfgAbi, Env, LinkOutputKind, LinkSelfContainedComponents,
+    Arch, BinaryFormat, Cc, CfgAbi, Env, LinkOutputKind, LinkSelfContainedComponents,
     LinkSelfContainedDefault, LinkerFeatures, LinkerFlavor, LinkerFlavorCli, Lld, Os, RelocModel,
     RelroLevel, SanitizerSet, SplitDebuginfo,
 };
@@ -2413,6 +2413,69 @@ fn add_rpath_args(
     }
 }
 
+fn strip_numeric_suffix<'a>(base: &'a str, suffix: impl AsRef<str>, fallback: &'a str) -> &'a str {
+    if suffix.as_ref().parse::<u32>().is_ok() { base } else { fallback }
+}
+
+fn undecorate_c_symbol<'a>(
+    name: &'a str,
+    sess: &Session,
+    kind: SymbolExportKind,
+) -> Option<&'a str> {
+    match sess.target.binary_format {
+        BinaryFormat::MachO => {
+            // Mach-O: strip the leading underscore that all external symbols have.
+            // The Darwin linker's export_symbols will add it back.
+            name.strip_prefix('_')
+        }
+        BinaryFormat::Coff => {
+            // MSVC C++ mangled names start with '?' and use a completely different
+            // decorating scheme that includes '@@' as structural delimiters.
+            // They must not be subjected to C calling-convention undecoration.
+            if name.starts_with('?') {
+                return Some(name);
+            }
+            Some(match sess.target.arch {
+                Arch::X86 => {
+                    // COFF 32-bit: strip calling-convention decorations.
+                    if let Some(rest) = name.strip_prefix('@') {
+                        // fastcall: @foo@N -> foo
+                        rest.rsplit_once('@')
+                            .map(|(base, suffix)| strip_numeric_suffix(base, suffix, name))
+                            .unwrap_or(name)
+                    } else if let Some(stripped) = name.strip_prefix('_') {
+                        if let Some((base, suffix)) = stripped.rsplit_once('@') {
+                            // stdcall: _foo@N -> foo
+                            strip_numeric_suffix(base, suffix, stripped)
+                        } else {
+                            // cdecl: _foo -> foo
+                            stripped
+                        }
+                    } else {
+                        // vectorcall: foo@@N -> foo
+                        name.rsplit_once("@@")
+                            .map(|(base, suffix)| strip_numeric_suffix(base, suffix, name))
+                            .unwrap_or(name)
+                    }
+                }
+                Arch::X86_64 => {
+                    // COFF 64-bit: vectorcall mangling (foo@@N -> foo) also applies on x86_64.
+                    name.rsplit_once("@@")
+                        .map(|(base, suffix)| strip_numeric_suffix(base, suffix, name))
+                        .unwrap_or(name)
+                }
+                Arch::Arm64EC if kind == SymbolExportKind::Text => {
+                    // Arm64EC: `#` prefix distinguishes ARM64EC text symbols from x64 thunks.
+                    name.strip_prefix('#').unwrap_or(name)
+                }
+                _ => name,
+            })
+        }
+        // ELF: no decoration
+        _ => Some(name),
+    }
+}
+
 fn add_c_staticlib_symbols(
     sess: &Session,
     lib: &NativeLib,
@@ -2454,7 +2517,14 @@ fn add_c_staticlib_symbols(
         }
 
         for symbol in object.symbols() {
-            if symbol.scope() != object::SymbolScope::Dynamic {
+            // The `object` crate returns `Dynamic` for ELF/Mach-O global symbols,
+            // but always returns `Linkage` for COFF external symbols.
+            // Accept both for COFF (Windows and UEFI).
+            let scope = symbol.scope();
+            if scope != object::SymbolScope::Dynamic
+                && !(sess.target.binary_format == BinaryFormat::Coff
+                    && scope == object::SymbolScope::Linkage)
+            {
                 continue;
             }
 
@@ -2469,9 +2539,10 @@ fn add_c_staticlib_symbols(
                 _ => continue,
             };
 
-            // FIXME:The symbol mangle rules are slightly different in Windows(32-bit) and Apple.
-            // Need to be resolved.
-            out.push((name.to_string(), export_kind));
+            let Some(undecorated) = undecorate_c_symbol(name, sess, export_kind) else {
+                continue;
+            };
+            out.push((undecorated.to_string(), export_kind));
         }
     }
 

compiler/rustc_codegen_ssa/src/traits/builder.rs

@@ -476,6 +476,11 @@ pub trait BuilderMethods<'a, 'tcx>:
         flags: MemFlags,
     );
 
+    // Produce a value from calling the `vscale` intrinsic (containing the `vscale` multiplier that
+    // a scalable vector's element size and count can be multiplied by to get the real size of the
+    // vector)
+    fn vscale(&mut self, ty: Self::Type) -> Self::Value;
+
     /// *Typed* copy for non-overlapping places.
     ///
     /// Has a default implementation in terms of `memcpy`, but specific backends
@@ -513,6 +518,12 @@ pub trait BuilderMethods<'a, 'tcx>:
             temp.val.store_with_flags(self, dst.with_type(layout), flags);
         } else if !layout.is_zst() {
             let bytes = self.const_usize(layout.size.bytes());
+            let bytes = if layout.peel_transparent_wrappers(self).ty.is_scalable_vector() {
+                let vscale = self.vscale(self.type_i64());
+                self.mul(vscale, bytes)
+            } else {
+                bytes
+            };
             self.memcpy(dst.llval, dst.align, src.llval, src.align, bytes, flags, None);
         }
     }

compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

@@ -1497,6 +1497,10 @@ LLVMRustBuildMemMove(LLVMBuilderRef B, LLVMValueRef Dst, unsigned DstAlign,
                                        unwrap(Size), IsVolatile));
 }
 
+extern "C" LLVMValueRef LLVMRustBuildVScale(LLVMBuilderRef B, LLVMTypeRef Ty) {
+  return wrap(unwrap(B)->CreateVScale(unwrap(Ty)));
+}
+
 extern "C" LLVMValueRef LLVMRustBuildMemSet(LLVMBuilderRef B, LLVMValueRef Dst,
                                             unsigned DstAlign, LLVMValueRef Val,
                                             LLVMValueRef Size,