feat: Add support for linker plugins (#1411)

Fixes #1

Author: davidlattimore

BENCHMARKING.md

@@ -8,8 +8,6 @@ particular:
 
 * Wild defaults to `--gc-sections`, so for a fair comparison, that should be passed to all the linkers.
 * Wild defaults to `-z now`, so best to pass that to all linkers.
-* Wild doesn't support linker plugins and various other flags. Generally use of these flags will
-  result in a warning.
 
 ## How to benchmark
 

Cargo.lock

@@ -58,6 +58,7 @@ itertools = "0.14.0"
 jobserver = "0.1.30"
 leb128 = "0.2.5"
 libc = "0.2.171"
+libloading = "0.8.6"
 memchr = "2.6.0"
 memmap2 = "0.9.0"
 mimalloc = { version = "0.1", default-features = false }

Cargo.toml

@@ -129,6 +129,7 @@ The following is working with the caveat that there may be bugs:
 * GNU jobserver support
 * Very basic linker script support (section mapping, keeping sections, alignment, defining start /
   stop symbols).
+* Linker plugin LTO (requires building Wild with `--features=plugins`)
 
 ### What isn't yet supported?
 
@@ -141,7 +142,6 @@ priority:
 * More complex linker scripts
 * Mac support
 * Windows support
-* LTO
 
 ### How can I verify that Wild was used to link a binary?
 

README.md

@@ -32,6 +32,7 @@ itertools = { workspace = true }
 jobserver = { workspace = true }
 leb128 = {workspace = true}
 libc = { workspace = true }
+libloading = { workspace = true, optional = true }
 linker-layout = { path = "../linker-layout", version = "0.8.0" }
 linker-trace = { path = "../linker-trace", version = "0.8.0" }
 linker-utils = { path = "../linker-utils", version = "0.8.0" }
@@ -64,6 +65,9 @@ tempfile = { workspace = true }
 # Support for running the linker as a subprocess.
 fork = []
 
+# Support for linker-plugins.
+plugins = ["libloading"]
+
 # Enable support for emitting a perfetto trace.
 perfetto = ["perfetto-recorder/enable", "tracing/release_max_level_info"]
 

libwild/Cargo.toml

@@ -33,6 +33,7 @@ use object::elf::GNU_PROPERTY_X86_ISA_1_V2;
 use object::elf::GNU_PROPERTY_X86_ISA_1_V3;
 use object::elf::GNU_PROPERTY_X86_ISA_1_V4;
 use rayon::ThreadPoolBuilder;
+use std::ffi::CString;
 use std::fmt::Display;
 use std::mem::take;
 use std::num::NonZero;
@@ -108,6 +109,8 @@ pub struct Args {
     pub(crate) export_list_path: Option<PathBuf>,
     pub(crate) auxiliary: Vec<String>,
     pub(crate) enable_new_dtags: bool,
+    pub(crate) plugin_path: Option<String>,
+    pub(crate) plugin_args: Vec<CString>,
 
     /// Symbol definitions from `--defsym` options. Each entry is (symbol_name, value_or_symbol).
     pub(crate) defsym: Vec<(String, DefsymValue)>,
@@ -292,6 +295,12 @@ pub struct Modifiers {
 
     /// Whether archive semantics should be applied even for regular objects.
     pub(crate) archive_semantics: bool,
+
+    /// Whether the file is known to be a temporary file that will be deleted when the linker
+    /// exits, e.g. an output file from a linker plugin. This doesn't affect linking, but is
+    /// stored in the layout file if written so that linker-diff knows not to error if the file
+    /// is missing.
+    pub(crate) temporary: bool,
 }
 
 #[derive(Debug, Eq, PartialEq)]
@@ -487,6 +496,8 @@ impl Default for Args {
             auxiliary: Vec::new(),
             numeric_experiments: Vec::new(),
             rpath_set: Default::default(),
+            plugin_path: None,
+            plugin_args: Vec::new(),
         }
     }
 }
@@ -722,6 +733,7 @@ impl Default for Modifiers {
             allow_shared: true,
             whole_archive: false,
             archive_semantics: false,
+            temporary: false,
         }
     }
 }
@@ -2408,8 +2420,9 @@ fn setup_argument_parser() -> ArgumentParser {
         .declare_with_param()
         .long("plugin-opt")
         .help("Pass options to the plugin")
-        .execute(|_args, _modifier_stack, _value| {
-            // TODO: Implement support for linker plugins.
+        .execute(|args, _modifier_stack, value| {
+            args.plugin_args
+                .push(CString::new(value).context("Invalid --plugin-opt argument")?);
             Ok(())
         });
 
@@ -2426,8 +2439,8 @@ fn setup_argument_parser() -> ArgumentParser {
         .declare_with_param()
         .long("plugin")
         .help("Load plugin")
-        .execute(|_args, _modifier_stack, value| {
-            warn_unsupported(&format!("--plugin {value}"))?;
+        .execute(|args, _modifier_stack, value| {
+            args.plugin_path = Some(value.to_owned());
             Ok(())
         });
 

libwild/src/args.rs

@@ -43,6 +43,8 @@ impl<'data> SymbolInfoPrinter<'data> {
                     ResolvedFile::Dynamic(obj) => Some(obj.common.file_id),
                     ResolvedFile::LinkerScript(obj) => Some(obj.file_id),
                     ResolvedFile::SyntheticSymbols(obj) => Some(obj.file_id),
+                    #[cfg(feature = "plugins")]
+                    ResolvedFile::LtoInput(obj) => Some(obj.file_id),
                 })
             })
             .collect();
@@ -136,6 +138,11 @@ impl<'data> SymbolInfoPrinter<'data> {
                         input = "  <synthetic>".to_owned();
                         sym_debug = "Synthetic symbol".to_owned();
                     }
+                    #[cfg(feature = "plugins")]
+                    SequencedInput::LtoInput(o) => {
+                        input = o.to_string();
+                        sym_debug = o.symbol_properties_display(symbol_id).to_string();
+                    }
                 }
 
                 // Versions can be either literally within the symbol name or in separate version

libwild/src/diagnostics.rs

@@ -4,6 +4,8 @@ use crate::bail;
 use crate::elf;
 use crate::error::Result;
 use object::LittleEndian;
+use object::read::elf::FileHeader;
+use object::read::elf::SectionHeader;
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 pub(crate) enum FileKind {
@@ -12,6 +14,8 @@ pub(crate) enum FileKind {
     Archive,
     ThinArchive,
     Text,
+    LlvmIr,
+    GccIr,
 }
 
 impl FileKind {
@@ -34,16 +38,68 @@ impl FileKind {
             }
 
             match header.e_type.get(LittleEndian) {
-                object::elf::ET_REL => Ok(FileKind::ElfObject),
+                object::elf::ET_REL => {
+                    if is_gcc_bitcode(bytes, header).unwrap_or(false) {
+                        Ok(FileKind::GccIr)
+                    } else {
+                        Ok(FileKind::ElfObject)
+                    }
+                }
                 object::elf::ET_DYN => Ok(FileKind::ElfDynamic),
                 t => bail!("Unsupported ELF kind {t}"),
             }
         } else if bytes.is_ascii() {
             Ok(FileKind::Text)
         } else if bytes.starts_with(b"BC") {
-            bail!("LLVM IR (LTO mode) is not supported yet");
+            Ok(FileKind::LlvmIr)
         } else {
             bail!("Couldn't identify file type");
         }
     }
+
+    pub(crate) fn is_compiler_ir(self) -> bool {
+        matches!(self, FileKind::LlvmIr | FileKind::GccIr)
+    }
+}
+
+/// Returns whether the supplied file contents is GCC IR. Scanning the entire section table would be
+/// expensive. Instead, we assume that we'll find a GCC LTO section within the first few sections,
+/// so just scan part of the section header strings table. It's unfortunate that GCC didn't tag
+/// these objects in some fast-to-check way.
+fn is_gcc_bitcode(data: &[u8], header: &crate::elf::FileHeader) -> Option<bool> {
+    // If we don't have plugin support, then we skip checking if the file contains GCC IR. If it is,
+    // then we'll figure that out later on and report an error. We do this because this code has a
+    // measurable performance impact.
+    if !cfg!(feature = "plugins") {
+        return Some(false);
+    }
+    let e = LittleEndian;
+    let section_headers = header.section_headers(e, data).ok()?;
+    let sh_str_index = header.shstrndx(e, data).ok()?;
+    let strings_section_header = section_headers.get(sh_str_index as usize)?;
+    let start_offset = strings_section_header.sh_offset(e) as usize;
+    let len = strings_section_header.sh_size(e) as usize;
+    // In observed GCC IR files, the LTO section names start at offset 44 and end at 454. We want to
+    // scan roughly the middle of this range.
+    const START: usize = 100;
+    // The longest GCC LTO section name is 47 bytes. We scan a bit more in case the first LTO
+    // section started later than START.
+    const MAX_SCAN: usize = 200;
+    let strings = data.get(start_offset + START..start_offset + (START + MAX_SCAN).min(len))?;
+    Some(memchr::memmem::find(strings, b"\0.gnu.lto_.").is_some())
+}
+
+impl std::fmt::Display for FileKind {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let s = match self {
+            FileKind::ElfObject => "ELF object",
+            FileKind::ElfDynamic => "ELF dynamic",
+            FileKind::Archive => "archive",
+            FileKind::ThinArchive => "thin archive",
+            FileKind::Text => "text",
+            FileKind::LlvmIr => "LLVM-IR",
+            FileKind::GccIr => "GCC-IR",
+        };
+        std::fmt::Display::fmt(s, f)
+    }
 }

libwild/src/file_kind.rs

@@ -22,6 +22,8 @@ pub(crate) enum Group<'data> {
     Objects(&'data [SequencedInputObject<'data>]),
     LinkerScripts(Vec<SequencedLinkerScript<'data>>),
     SyntheticSymbols(SyntheticSymbols),
+    #[cfg(feature = "plugins")]
+    LtoInputs(Vec<crate::linker_plugins::LtoInput<'data>>),
 }
 
 #[derive(Debug)]
@@ -44,6 +46,8 @@ pub(crate) enum SequencedInput<'data> {
     Object(&'data SequencedInputObject<'data>),
     LinkerScript(&'data SequencedLinkerScript<'data>),
     SyntheticSymbols(&'data SyntheticSymbols),
+    #[cfg(feature = "plugins")]
+    LtoInput(&'data crate::linker_plugins::LtoInput<'data>),
 }
 
 impl Group<'_> {
@@ -55,6 +59,8 @@ impl Group<'_> {
             Group::Objects(objects) => objects[0].file_id.group(),
             Group::LinkerScripts(scripts) => scripts[0].file_id.group(),
             Group::SyntheticSymbols(s) => s.file_id.group(),
+            #[cfg(feature = "plugins")]
+            Group::LtoInputs(s) => s[0].file_id.group(),
         }
     }
 
@@ -80,6 +86,11 @@ impl Group<'_> {
                 }
             }
             Group::SyntheticSymbols(o) => o.symbol_id_range,
+            #[cfg(feature = "plugins")]
+            Group::LtoInputs(objects) => SymbolIdRange::covering(
+                objects[0].symbol_id_range,
+                objects[objects.len() - 1].symbol_id_range,
+            ),
         }
     }
 
@@ -314,6 +325,8 @@ impl SequencedInput<'_> {
             SequencedInput::Object(o) => o.symbol_id_range,
             SequencedInput::LinkerScript(o) => o.symbol_id_range,
             SequencedInput::SyntheticSymbols(o) => o.symbol_id_range,
+            #[cfg(feature = "plugins")]
+            SequencedInput::LtoInput(o) => o.symbol_id_range,
         }
     }
 
@@ -359,6 +372,8 @@ impl Display for Group<'_> {
             }
             Group::LinkerScripts(scripts) => write!(f, "{} linker script(s)", scripts.len()),
             Group::SyntheticSymbols(_) => write!(f, "<epilogue>"),
+            #[cfg(feature = "plugins")]
+            Group::LtoInputs(lto_inputs) => write!(f, "<{} lto inputs>", lto_inputs.len()),
         }
     }
 }
@@ -370,6 +385,8 @@ impl std::fmt::Display for SequencedInput<'_> {
             SequencedInput::Object(o) => std::fmt::Display::fmt(o, f),
             SequencedInput::LinkerScript(o) => std::fmt::Display::fmt(&o.parsed, f),
             SequencedInput::SyntheticSymbols(_) => std::fmt::Display::fmt("<epilogue>", f),
+            #[cfg(feature = "plugins")]
+            SequencedInput::LtoInput(o) => std::fmt::Display::fmt(o, f),
         }
     }
 }