aarch64: Implement TLSDESC for TLS GD accesses (#7201)

* aarch64: Implement TLSDESC for TLS GD accesses

* aarch64: Box ExternalName to avoid growing the `Inst` enum

cranelift/codegen/src/binemit/mod.rs

@@ -64,15 +64,21 @@ pub enum Reloc {
     /// Offset within page of TLVP slot.
     MachOAarch64TlsAdrPageOff12,
 
-    /// Aarch64 TLS GD
+    /// Aarch64 TLSDESC Adr Page21
-    /// Set an ADRP immediate field to the top 21 bits of the final address. Checks for overflow.
+    /// This is equivalent to `R_AARCH64_TLSDESC_ADR_PAGE21` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
-    /// This is equivalent to `R_AARCH64_TLSGD_ADR_PAGE21` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#relocations-for-thread-local-storage)
+    Aarch64TlsDescAdrPage21,
-    Aarch64TlsGdAdrPage21,
 
-    /// Aarch64 TLS GD
+    /// Aarch64 TLSDESC Ld64 Lo12
-    /// Set the add immediate field to the low 12 bits of the final address. Does not check for overflow.
+    /// This is equivalent to `R_AARCH64_TLSDESC_LD64_LO12` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
-    /// This is equivalent to `R_AARCH64_TLSGD_ADD_LO12_NC` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#relocations-for-thread-local-storage)
+    Aarch64TlsDescLd64Lo12,
-    Aarch64TlsGdAddLo12Nc,
+
+    /// Aarch64 TLSDESC Add Lo12
+    /// This is equivalent to `R_AARCH64_TLSGD_ADD_LO12` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
+    Aarch64TlsDescAddLo12,
+
+    /// Aarch64 TLSDESC Call
+    /// This is equivalent to `R_AARCH64_TLSDESC_CALL` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#57105thread-local-storage-descriptors)
+    Aarch64TlsDescCall,
 
     /// AArch64 GOT Page
     /// Set the immediate value of an ADRP to bits 32:12 of X; check that –232 <= X < 232
@@ -141,8 +147,10 @@ impl fmt::Display for Reloc {
             Self::MachOX86_64Tlv => write!(f, "MachOX86_64Tlv"),
             Self::MachOAarch64TlsAdrPage21 => write!(f, "MachOAarch64TlsAdrPage21"),
             Self::MachOAarch64TlsAdrPageOff12 => write!(f, "MachOAarch64TlsAdrPageOff12"),
-            Self::Aarch64TlsGdAdrPage21 => write!(f, "Aarch64TlsGdAdrPage21"),
+            Self::Aarch64TlsDescAdrPage21 => write!(f, "Aarch64TlsDescAdrPage21"),
-            Self::Aarch64TlsGdAddLo12Nc => write!(f, "Aarch64TlsGdAddLo12Nc"),
+            Self::Aarch64TlsDescLd64Lo12 => write!(f, "Aarch64TlsDescLd64Lo12"),
+            Self::Aarch64TlsDescAddLo12 => write!(f, "Aarch64TlsDescAddLo12"),
+            Self::Aarch64TlsDescCall => write!(f, "Aarch64TlsDescCall"),
             Self::Aarch64AdrGotPage21 => write!(f, "Aarch64AdrGotPage21"),
             Self::Aarch64Ld64GotLo12Nc => write!(f, "Aarch64AdrGotLo12Nc"),
             Self::S390xTlsGd64 => write!(f, "TlsGd64"),

cranelift/codegen/src/isa/aarch64/inst.isle

@@ -947,8 +947,9 @@
 
        ;; A call to the `ElfTlsGetAddr` libcall. Returns address of TLS symbol in x0.
        (ElfTlsGetAddr
-        (symbol ExternalName)
+        (symbol BoxExternalName)
-        (rd WritableReg))
+        (rd WritableReg)
+        (tmp WritableReg))
 
        (MachOTlsGetAddr
         (symbol ExternalName)
@@ -3764,7 +3765,8 @@
 (decl elf_tls_get_addr (ExternalName) Reg)
 (rule (elf_tls_get_addr name)
       (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.ElfTlsGetAddr name dst))))
+            (tmp WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.ElfTlsGetAddr (box_external_name name) dst tmp))))
         dst))
 
 (decl macho_tls_get_addr (ExternalName) Reg)

cranelift/codegen/src/isa/aarch64/inst/emit.rs

@@ -4,7 +4,7 @@ use cranelift_control::ControlPlane;
 use regalloc2::Allocation;
 
 use crate::binemit::{Reloc, StackMap};
-use crate::ir::{self, types::*, LibCall, MemFlags, RelSourceLoc, TrapCode};
+use crate::ir::{self, types::*, MemFlags, RelSourceLoc, TrapCode};
 use crate::isa::aarch64::inst::*;
 use crate::machinst::{ty_bits, Reg, RegClass, Writable};
 use crate::trace;
@@ -3537,32 +3537,81 @@ impl MachInstEmit for Inst {
                 }
             }
 
-            &Inst::ElfTlsGetAddr { ref symbol, rd } => {
+            &Inst::ElfTlsGetAddr {
+                ref symbol,
+                rd,
+                tmp,
+            } => {
                 let rd = allocs.next_writable(rd);
+                let tmp = allocs.next_writable(tmp);
                 assert_eq!(xreg(0), rd.to_reg());
 
+                // See the original proposal for TLSDESC.
+                // http://www.fsfla.org/~lxoliva/writeups/TLS/paper-lk2006.pdf
+                //
+                // Implement the TLSDESC instruction sequence:
+                //   adrp x0, :tlsdesc:tlsvar
+                //   ldr  tmp, [x0, :tlsdesc_lo12:tlsvar]
+                //   add  x0, x0, :tlsdesc_lo12:tlsvar
+                //   blr  tmp
+                //   mrs  tmp, tpidr_el0
+                //   add  x0, x0, tmp
+                //
                 // This is the instruction sequence that GCC emits for ELF GD TLS Relocations in aarch64
-                // See: https://gcc.godbolt.org/z/KhMh5Gvra
+                // See: https://gcc.godbolt.org/z/e4j7MdErh
 
-                // adrp x0, <label>
+                // adrp x0, :tlsdesc:tlsvar
-                sink.add_reloc(Reloc::Aarch64TlsGdAdrPage21, symbol, 0);
+                sink.add_reloc(Reloc::Aarch64TlsDescAdrPage21, &**symbol, 0);
-                let inst = Inst::Adrp { rd, off: 0 };
+                Inst::Adrp { rd, off: 0 }.emit(&[], sink, emit_info, state);
-                inst.emit(&[], sink, emit_info, state);
 
-                // add x0, x0, <label>
+                // ldr  tmp, [x0, :tlsdesc_lo12:tlsvar]
-                sink.add_reloc(Reloc::Aarch64TlsGdAddLo12Nc, symbol, 0);
+                sink.add_reloc(Reloc::Aarch64TlsDescLd64Lo12, &**symbol, 0);
-                sink.put4(0x91000000);
+                Inst::ULoad64 {
+                    rd: tmp,
+                    mem: AMode::reg(rd.to_reg()),
+                    flags: MemFlags::trusted(),
+                }
+                .emit(&[], sink, emit_info, state);
 
-                // bl __tls_get_addr
+                // add x0, x0, :tlsdesc_lo12:tlsvar
-                sink.add_reloc(
+                sink.add_reloc(Reloc::Aarch64TlsDescAddLo12, &**symbol, 0);
-                    Reloc::Arm64Call,
+                Inst::AluRRImm12 {
-                    &ExternalName::LibCall(LibCall::ElfTlsGetAddr),
+                    alu_op: ALUOp::Add,
-                    0,
+                    size: OperandSize::Size64,
-                );
+                    rd,
-                sink.put4(0x94000000);
+                    rn: rd.to_reg(),
+                    imm12: Imm12::maybe_from_u64(0).unwrap(),
+                }
+                .emit(&[], sink, emit_info, state);
 
-                // nop
+                // blr tmp
-                sink.put4(0xd503201f);
+                sink.add_reloc(Reloc::Aarch64TlsDescCall, &**symbol, 0);
+                Inst::CallInd {
+                    info: crate::isa::Box::new(CallIndInfo {
+                        rn: tmp.to_reg(),
+                        uses: smallvec![],
+                        defs: smallvec![],
+                        clobbers: PRegSet::empty(),
+                        opcode: Opcode::CallIndirect,
+                        caller_callconv: CallConv::SystemV,
+                        callee_callconv: CallConv::SystemV,
+                        callee_pop_size: 0,
+                    }),
+                }
+                .emit(&[], sink, emit_info, state);
+
+                // mrs tmp, tpidr_el0
+                sink.put4(0xd53bd040 | machreg_to_gpr(tmp.to_reg()));
+
+                // add x0, x0, tmp
+                Inst::AluRRR {
+                    alu_op: ALUOp::Add,
+                    size: OperandSize::Size64,
+                    rd,
+                    rn: rd.to_reg(),
+                    rm: tmp.to_reg(),
+                }
+                .emit(&[], sink, emit_info, state);
             }
 
             &Inst::MachOTlsGetAddr { ref symbol, rd } => {

cranelift/codegen/src/isa/aarch64/inst/mod.rs

@@ -939,11 +939,15 @@ fn aarch64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut Operan
         &Inst::Bti { .. } => {}
         &Inst::VirtualSPOffsetAdj { .. } => {}
 
-        &Inst::ElfTlsGetAddr { rd, .. } => {
+        &Inst::ElfTlsGetAddr { rd, tmp, .. } => {
+            // TLSDESC has a very neat calling convention. It is required to preserve
+            // all registers except x0 and x30. X30 is non allocatable in cranelift since
+            // its the link register.
+            //
+            // Additionally we need a second register as a temporary register for the
+            // TLSDESC sequence. This register can be any register other than x0 (and x30).
             collector.reg_fixed_def(rd, regs::xreg(0));
-            let mut clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(CallConv::SystemV);
+            collector.reg_early_def(tmp);
-            clobbers.remove(regs::xreg_preg(0));
-            collector.reg_clobbers(clobbers);
         }
         &Inst::MachOTlsGetAddr { rd, .. } => {
             collector.reg_fixed_def(rd, regs::xreg(0));
@@ -2842,9 +2846,14 @@ impl Inst {
             }
             &Inst::EmitIsland { needed_space } => format!("emit_island {}", needed_space),
 
-            &Inst::ElfTlsGetAddr { ref symbol, rd } => {
+            &Inst::ElfTlsGetAddr {
+                ref symbol,
+                rd,
+                tmp,
+            } => {
                 let rd = pretty_print_reg(rd.to_reg(), allocs);
-                format!("elf_tls_get_addr {}, {}", rd, symbol.display(None))
+                let tmp = pretty_print_reg(tmp.to_reg(), allocs);
+                format!("elf_tls_get_addr {}, {}, {}", rd, tmp, symbol.display(None))
             }
             &Inst::MachOTlsGetAddr { ref symbol, rd } => {
                 let rd = pretty_print_reg(rd.to_reg(), allocs);

cranelift/filetests/filetests/isa/aarch64/tls-elf-gd.clif

@@ -13,21 +13,11 @@ block0(v0: i32):
 ; VCode:
 ;   stp fp, lr, [sp, #-16]!
 ;   mov fp, sp
-;   str x24, [sp, #-16]!
-;   stp d14, d15, [sp, #-16]!
-;   stp d12, d13, [sp, #-16]!
-;   stp d10, d11, [sp, #-16]!
-;   stp d8, d9, [sp, #-16]!
 ; block0:
-;   mov x24, x0
+;   mov x5, x0
-;   elf_tls_get_addr x0, userextname0
+;   elf_tls_get_addr x0, x3, userextname0
 ;   mov x1, x0
-;   mov x0, x24
+;   mov x0, x5
-;   ldp d8, d9, [sp], #16
-;   ldp d10, d11, [sp], #16
-;   ldp d12, d13, [sp], #16
-;   ldp d14, d15, [sp], #16
-;   ldr x24, [sp], #16
 ;   ldp fp, lr, [sp], #16
 ;   ret
 ;
@@ -35,24 +25,16 @@ block0(v0: i32):
 ; block0: ; offset 0x0
 ;   stp x29, x30, [sp, #-0x10]!
 ;   mov x29, sp
-;   str x24, [sp, #-0x10]!
+; block1: ; offset 0x8
-;   stp d14, d15, [sp, #-0x10]!
+;   mov x5, x0
-;   stp d12, d13, [sp, #-0x10]!
+;   adrp x0, #0 ; reloc_external Aarch64TlsDescAdrPage21 u1:0 0
-;   stp d10, d11, [sp, #-0x10]!
+;   ldr x3, [x0] ; reloc_external Aarch64TlsDescLd64Lo12 u1:0 0
-;   stp d8, d9, [sp, #-0x10]!
+;   add x0, x0, #0 ; reloc_external Aarch64TlsDescAddLo12 u1:0 0
-; block1: ; offset 0x1c
+;   blr x3 ; reloc_external Aarch64TlsDescCall u1:0 0
-;   mov x24, x0
+;   mrs x3, tpidr_el0
-;   adrp x0, #0 ; reloc_external Aarch64TlsGdAdrPage21 u1:0 0
+;   add x0, x0, x3
-;   add x0, x0, #0 ; reloc_external Aarch64TlsGdAddLo12Nc u1:0 0
-;   bl #0x28 ; reloc_external Call %ElfTlsGetAddr 0
-;   nop
 ;   mov x1, x0
-;   mov x0, x24
+;   mov x0, x5
-;   ldp d8, d9, [sp], #0x10
-;   ldp d10, d11, [sp], #0x10
-;   ldp d12, d13, [sp], #0x10
-;   ldp d14, d15, [sp], #0x10
-;   ldr x24, [sp], #0x10
 ;   ldp x29, x30, [sp], #0x10
 ;   ret
 

cranelift/object/src/backend.rs

@@ -718,30 +718,55 @@ impl ObjectModule {
                     12,
                 )
             }
-            Reloc::Aarch64TlsGdAdrPage21 => {
+            Reloc::Aarch64TlsDescAdrPage21 => {
                 assert_eq!(
                     self.object.format(),
                     object::BinaryFormat::Elf,
-                    "Aarch64TlsGdAdrPrel21 is not supported for this file format"
+                    "Aarch64TlsDescAdrPage21 is not supported for this file format"
                 );
                 (
-                    RelocationKind::Elf(object::elf::R_AARCH64_TLSGD_ADR_PAGE21),
+                    RelocationKind::Elf(object::elf::R_AARCH64_TLSDESC_ADR_PAGE21),
                     RelocationEncoding::Generic,
                     21,
                 )
             }
-            Reloc::Aarch64TlsGdAddLo12Nc => {
+            Reloc::Aarch64TlsDescLd64Lo12 => {
                 assert_eq!(
                     self.object.format(),
                     object::BinaryFormat::Elf,
-                    "Aarch64TlsGdAddLo12Nc is not supported for this file format"
+                    "Aarch64TlsDescLd64Lo12 is not supported for this file format"
                 );
                 (
-                    RelocationKind::Elf(object::elf::R_AARCH64_TLSGD_ADD_LO12_NC),
+                    RelocationKind::Elf(object::elf::R_AARCH64_TLSDESC_LD64_LO12),
                     RelocationEncoding::Generic,
                     12,
                 )
             }
+            Reloc::Aarch64TlsDescAddLo12 => {
+                assert_eq!(
+                    self.object.format(),
+                    object::BinaryFormat::Elf,
+                    "Aarch64TlsDescAddLo12 is not supported for this file format"
+                );
+                (
+                    RelocationKind::Elf(object::elf::R_AARCH64_TLSDESC_ADD_LO12),
+                    RelocationEncoding::Generic,
+                    12,
+                )
+            }
+            Reloc::Aarch64TlsDescCall => {
+                assert_eq!(
+                    self.object.format(),
+                    object::BinaryFormat::Elf,
+                    "Aarch64TlsDescCall is not supported for this file format"
+                );
+                (
+                    RelocationKind::Elf(object::elf::R_AARCH64_TLSDESC_CALL),
+                    RelocationEncoding::Generic,
+                    0,
+                )
+            }
+
             Reloc::Aarch64AdrGotPage21 => match self.object.format() {
                 object::BinaryFormat::Elf => (
                     RelocationKind::Elf(object::elf::R_AARCH64_ADR_GOT_PAGE),