s390x: Fix TLS GD relocation order (#7978)

When emitting a call to __tls_get_offset, the instruction needs to
carry two relocations, a R_390_PLT32DBL targeting __tls_get_offset
and a R_390_TLS_GDCALL targeting the TLS symbol.  Specifically, the
system linker expects to see these two relocation in that order.

However, the cranelift backend currently emits the relocations in
reverse order - this unfortunately causes the linker to corrupt
the instruction sequence when performing a TLS relaxation.

To fix this in the backend, I need support in machinst common code
to emit a relocation at some offset to the instruction about to be
emitted (e.g. the relocation should target two bytes into the 6-byte
instruction that will be emitted next).  I've added a new routine
add_reloc_at_offset to that effect.  This also allowed to simplify
some existing code in the backend.

Also, change the disassembler to print multiple relocations on
a single instruction, if present.

cranelift/codegen/src/isa/s390x/inst/emit.rs

@@ -218,15 +218,8 @@ pub fn mem_emit(
         &MemArg::Symbol {
             ref name, offset, ..
         } => {
-            let reloc = Reloc::S390xPCRel32Dbl;
-            put_with_reloc(
-                sink,
-                &enc_ril_b(opcode_ril.unwrap(), rd, 0),
-                2,
-                reloc,
-                name,
-                offset.into(),
-            );
+            sink.add_reloc_at_offset(2, Reloc::S390xPCRel32Dbl, &**name, (offset + 2).into());
+            put(sink, &enc_ril_b(opcode_ril.unwrap(), rd, 0));
         }
         _ => unreachable!(),
     }
@@ -1320,25 +1313,6 @@ fn put_with_trap(sink: &mut MachBuffer<Inst>, enc: &[u8], trap_code: TrapCode) {
     sink.put1(enc[len - 1]);
 }
 
-/// Emit encoding to sink, adding a relocation at byte offset.
-fn put_with_reloc(
-    sink: &mut MachBuffer<Inst>,
-    enc: &[u8],
-    offset: usize,
-    ri2_reloc: Reloc,
-    ri2_name: &ExternalName,
-    ri2_offset: i64,
-) {
-    let len = enc.len();
-    for i in 0..offset {
-        sink.put1(enc[i]);
-    }
-    sink.add_reloc(ri2_reloc, ri2_name, ri2_offset + offset as i64);
-    for i in offset..len {
-        sink.put1(enc[i]);
-    }
-}
-
 /// State carried between emissions of a sequence of instructions.
 #[derive(Default, Clone, Debug)]
 pub struct EmitState {
@@ -3505,6 +3479,13 @@ impl Inst {
             &Inst::Call { link, ref info } => {
                 debug_assert_eq!(link.to_reg(), gpr(14));
 
+                let opcode = 0xc05; // BRASL
+
+                // Add relocation for target function.  This has to be done *before*
+                // the S390xTlsGdCall relocation if any, to ensure linker relaxation
+                // works correctly.
+                sink.add_reloc_at_offset(2, Reloc::S390xPLTRel32Dbl, &info.dest, 2);
+
                 // Add relocation for TLS libcalls to enable linker optimizations.
                 match &info.tls_symbol {
                     None => {}
@@ -3514,19 +3495,10 @@ impl Inst {
                     _ => unreachable!(),
                 }
 
-                let opcode = 0xc05; // BRASL
-                let reloc = Reloc::S390xPLTRel32Dbl;
                 if let Some(s) = state.take_stack_map() {
                     sink.add_stack_map(StackMapExtent::UpcomingBytes(6), s);
                 }
-                put_with_reloc(
-                    sink,
-                    &enc_ril_b(opcode, link.to_reg(), 0),
-                    2,
-                    reloc,
-                    &info.dest,
-                    0,
-                );
+                put(sink, &enc_ril_b(opcode, link.to_reg(), 0));
                 if info.opcode.is_call() {
                     sink.add_call_site(info.opcode);
                 }

cranelift/codegen/src/machinst/buffer.rs

@@ -1549,9 +1549,10 @@ impl<I: VCodeInst> MachBuffer<I> {
         }
     }
 
-    /// Add an external relocation at the current offset.
-    pub fn add_reloc<T: Into<RelocTarget> + Clone>(
+    /// Add an external relocation at the given offset from current offset.
+    pub fn add_reloc_at_offset<T: Into<RelocTarget> + Clone>(
         &mut self,
+        offset: CodeOffset,
         kind: Reloc,
         target: &T,
         addend: Addend,
@@ -1591,13 +1592,23 @@ impl<I: VCodeInst> MachBuffer<I> {
         // when a relocation can't otherwise be resolved later, so it shouldn't
         // actually result in any memory unsafety or anything like that.
         self.relocs.push(MachReloc {
-            offset: self.data.len() as CodeOffset,
+            offset: self.data.len() as CodeOffset + offset,
             kind,
             target,
             addend,
         });
     }
 
+    /// Add an external relocation at the current offset.
+    pub fn add_reloc<T: Into<RelocTarget> + Clone>(
+        &mut self,
+        kind: Reloc,
+        target: &T,
+        addend: Addend,
+    ) {
+        self.add_reloc_at_offset(0, kind, target, addend);
+    }
+
     /// Add a trap record at the current offset.
     pub fn add_trap(&mut self, code: TrapCode) {
         self.traps.push(MachTrap {

cranelift/codegen/src/machinst/mod.rs

@@ -430,7 +430,7 @@ impl<T: CompilePhase> CompiledCodeBase<T> {
                 let end = i.address() + i.bytes().len() as u64;
                 let contains = |off| i.address() <= off && off < end;
 
-                if let Some(reloc) = relocs.iter().find(|reloc| contains(reloc.offset as u64)) {
+                for reloc in relocs.iter().filter(|reloc| contains(reloc.offset as u64)) {
                     write!(
                         buf,
                         " ; reloc_external {} {} {}",

cranelift/filetests/filetests/isa/s390x/tls_elf.clif

@@ -37,7 +37,7 @@ block0(v0: i32):
 ;   .byte 0x00, 0x00
 ;   .byte 0x00, 0x00
 ;   lg %r2, 0(%r1)
-;   brasl %r14, 0x22 ; reloc_external TlsGdCall u1:0 0
+;   brasl %r14, 0x22 ; reloc_external PLTRel32Dbl %ElfTlsGetOffset 2 ; reloc_external TlsGdCall u1:0 0
 ;   ear %r3, %a0
 ;   sllg %r5, %r3, 0x20
 ;   ear %r5, %a1