From 8e2a486e5a1ae6391986bd6b285b1864255c8265 Mon Sep 17 00:00:00 2001 From: Daniil Belov Date: Thu, 28 Jul 2022 17:19:45 +0300 Subject: [PATCH 1/4] Change rlib format To distinguish objects from different native libraries TODO: restore LTO, hide change under flag (to be squashed) --- .../rustc_codegen_llvm/src/back/archive.rs | 30 +++ .../rustc_codegen_ssa/src/back/archive.rs | 4 + compiler/rustc_codegen_ssa/src/back/link.rs | 195 ++++++++---------- .../native-link-modifier-bundle/Makefile | 5 +- .../mix-bundle-and-whole-archive-link-attr.rs | 8 - ...-bundle-and-whole-archive-link-attr.stderr | 6 - .../mix-bundle-and-whole-archive.rs | 7 - .../mix-bundle-and-whole-archive.stderr | 6 - 8 files changed, 123 insertions(+), 138 deletions(-) delete mode 100644 src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.rs delete mode 100644 src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.stderr delete mode 100644 src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.rs delete mode 100644 src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.stderr diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index baa858709a0c6..9826649e51fb6 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -238,6 +238,36 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { output_path } + + fn unpack_archive(archive: &PathBuf, output: &Path, mut skip: F) -> io::Result<()> + where + F: FnMut(&str) -> bool + 'static, + { + let archive = ArchiveRO::open(archive).unwrap(); + + for child in archive.iter() { + let lib = child.map_err(string_to_io_error)?; + let Some(name) = &lib.name() else { + continue; + }; + + if skip(name) { + continue; + } + + let data = lib.data(); + let out = output.join(name); + + match std::fs::write(out, data) { + Ok(_) => {} + Err(err) => { + panic!("{}", err.kind().to_string()); + } + } + } + + Ok(()) + } } impl<'a> LlvmArchiveBuilder<'a> { diff --git a/compiler/rustc_codegen_ssa/src/back/archive.rs b/compiler/rustc_codegen_ssa/src/back/archive.rs index 53550b049db09..861e07f924832 100644 --- a/compiler/rustc_codegen_ssa/src/back/archive.rs +++ b/compiler/rustc_codegen_ssa/src/back/archive.rs @@ -53,6 +53,10 @@ pub trait ArchiveBuilder<'a> { fn sess(&self) -> &Session; + fn unpack_archive(archive: &PathBuf, output: &Path, skip: F) -> io::Result<()> + where + F: FnMut(&str) -> bool + 'static; + /// Creates a DLL Import Library . /// and returns the path on disk to that import library. /// This functions doesn't take `self` so that it can be called from diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index f0d320c7c21c9..6760ebfb1b4fa 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -300,54 +300,47 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>( } } - // Note that in this loop we are ignoring the value of `lib.cfg`. That is, - // we may not be configured to actually include a static library if we're - // adding it here. That's because later when we consume this rlib we'll - // decide whether we actually needed the static library or not. - // - // To do this "correctly" we'd need to keep track of which libraries added - // which object files to the archive. We don't do that here, however. The - // #[link(cfg(..))] feature is unstable, though, and only intended to get - // liblibc working. In that sense the check below just indicates that if - // there are any libraries we want to omit object files for at link time we - // just exclude all custom object files. - // - // Eventually if we want to stabilize or flesh out the #[link(cfg(..))] - // feature then we'll need to figure out how to record what objects were - // loaded from the libraries found here and then encode that into the - // metadata of the rlib we're generating somehow. - for lib in codegen_results.crate_info.used_libraries.iter() { - match lib.kind { - NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) } - if flavor == RlibFlavor::Normal => - { - // Don't allow mixing +bundle with +whole_archive since an rlib may contain - // multiple native libs, some of which are +whole-archive and some of which are - // -whole-archive and it isn't clear how we can currently handle such a - // situation correctly. - // See https://github.com/rust-lang/rust/issues/88085#issuecomment-901050897 - sess.err( - "the linking modifiers `+bundle` and `+whole-archive` are not compatible \ - with each other when generating rlibs", + if flavor == RlibFlavor::StaticlibBase { + // Note that in this loop we are ignoring the value of `lib.cfg`. That is, + // we may not be configured to actually include a static library if we're + // adding it here. That's because later when we consume this rlib we'll + // decide whether we actually needed the static library or not. + // + // To do this "correctly" we'd need to keep track of which libraries added + // which object files to the archive. We don't do that here, however. The + // #[link(cfg(..))] feature is unstable, though, and only intended to get + // liblibc working. In that sense the check below just indicates that if + // there are any libraries we want to omit object files for at link time we + // just exclude all custom object files. + // + // Eventually if we want to stabilize or flesh out the #[link(cfg(..))] + // feature then we'll need to figure out how to record what objects were + // loaded from the libraries found here and then encode that into the + // metadata of the rlib we're generating somehow. + for lib in codegen_results.crate_info.used_libraries.iter() { + match lib.kind { + NativeLibKind::Static { bundle: None | Some(true), .. } => {} + NativeLibKind::Static { bundle: Some(false), .. } + | NativeLibKind::Dylib { .. } + | NativeLibKind::Framework { .. } + | NativeLibKind::RawDylib + | NativeLibKind::Unspecified => continue, + } + if let Some(name) = lib.name { + let location = find_library( + name.as_str(), + lib.verbatim.unwrap_or(false), + &lib_search_paths, + sess, ); + ab.add_archive(&location, |_| false).unwrap_or_else(|e| { + sess.fatal(&format!( + "failed to add native library {}: {}", + location.to_string_lossy(), + e + )); + }); } - NativeLibKind::Static { bundle: None | Some(true), .. } => {} - NativeLibKind::Static { bundle: Some(false), .. } - | NativeLibKind::Dylib { .. } - | NativeLibKind::Framework { .. } - | NativeLibKind::RawDylib - | NativeLibKind::Unspecified => continue, - } - if let Some(name) = lib.name { - let location = - find_library(name.as_str(), lib.verbatim.unwrap_or(false), &lib_search_paths, sess); - ab.add_archive(&location, |_| false).unwrap_or_else(|e| { - sess.fatal(&format!( - "failed to add native library {}: {}", - location.to_string_lossy(), - e - )); - }); } } @@ -386,6 +379,28 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>( ab.add_file(&trailing_metadata); } + if flavor == RlibFlavor::Normal { + for lib in codegen_results.crate_info.used_libraries.iter() { + match lib.kind { + NativeLibKind::Static { bundle: None | Some(true), .. } => {} + NativeLibKind::Static { bundle: Some(false), .. } + | NativeLibKind::Dylib { .. } + | NativeLibKind::Framework { .. } + | NativeLibKind::RawDylib + | NativeLibKind::Unspecified => continue, + } + if let Some(name) = lib.name { + let location = find_library( + name.as_str(), + lib.verbatim.unwrap_or(false), + &lib_search_paths, + sess, + ); + ab.add_file(&location) + } + } + } + return Ok(ab); } @@ -2447,73 +2462,33 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( let src = &codegen_results.crate_info.used_crate_source[&cnum]; let cratepath = &src.rlib.as_ref().unwrap().0; - let mut link_upstream = |path: &Path| { - cmd.link_rlib(&fix_windows_verbatim_for_gcc(path)); - }; - - // See the comment above in `link_staticlib` and `link_rlib` for why if - // there's a static library that's not relevant we skip all object - // files. - let native_libs = &codegen_results.crate_info.native_libraries[&cnum]; - let skip_native = native_libs.iter().any(|lib| { - matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. }) - && !relevant_lib(sess, lib) - }); - - if (!are_upstream_rust_objects_already_included(sess) - || ignored_for_lto(sess, &codegen_results.crate_info, cnum)) - && !skip_native - { - link_upstream(cratepath); - return; - } - - let dst = tmpdir.join(cratepath.file_name().unwrap()); - let name = cratepath.file_name().unwrap().to_str().unwrap(); - let name = &name[3..name.len() - 5]; // chop off lib/.rlib - - sess.prof.generic_activity_with_arg("link_altering_rlib", name).run(|| { - let canonical_name = name.replace('-', "_"); - let upstream_rust_objects_already_included = - are_upstream_rust_objects_already_included(sess); - let is_builtins = sess.target.no_builtins - || !codegen_results.crate_info.is_no_builtins.contains(&cnum); - - let mut archive = ::new(sess, &dst); - if let Err(e) = archive.add_archive(cratepath, move |f| { - if f == METADATA_FILENAME { - return true; - } - - let canonical = f.replace('-', "_"); - - let is_rust_object = - canonical.starts_with(&canonical_name) && looks_like_rust_object_file(&f); - - // If we've been requested to skip all native object files - // (those not generated by the rust compiler) then we can skip - // this file. See above for why we may want to do this. - let skip_because_cfg_say_so = skip_native && !is_rust_object; - - // If we're performing LTO and this is a rust-generated object - // file, then we don't need the object file as it's part of the - // LTO module. Note that `#![no_builtins]` is excluded from LTO, - // though, so we let that object file slide. - let skip_because_lto = - upstream_rust_objects_already_included && is_rust_object && is_builtins; - - if skip_because_cfg_say_so || skip_because_lto { - return true; + cmd.link_rlib(&fix_windows_verbatim_for_gcc(cratepath)); + for lib in &codegen_results.crate_info.native_libraries[&cnum] { + let Some(name) = lib.name else { + continue; + }; + let name = name.as_str(); + if !relevant_lib(sess, lib) { + continue; + } + let verbatim = lib.verbatim.unwrap_or(false); + if let NativeLibKind::Static { bundle: Some(true) | None, whole_archive } = lib.kind { + if whole_archive == Some(true) { + cmd.link_whole_staticlib(name, verbatim, &[PathBuf::from(tmpdir)]) + } else { + cmd.link_staticlib(name, verbatim); + cmd.include_path(tmpdir) } - - false - }) { - sess.fatal(&format!("failed to build archive from rlib: {}", e)); } - if archive.build() { - link_upstream(&dst); + } + + B::unpack_archive(cratepath, tmpdir, |fname: &str| { + if !fname.ends_with(".a") { + return true; } - }); + return false; + }) + .unwrap(); } // Same thing as above, but for dynamic crates instead of static crates. diff --git a/src/test/run-make/native-link-modifier-bundle/Makefile b/src/test/run-make/native-link-modifier-bundle/Makefile index e4b0f74ac26ac..43a0dbcc676a6 100644 --- a/src/test/run-make/native-link-modifier-bundle/Makefile +++ b/src/test/run-make/native-link-modifier-bundle/Makefile @@ -7,7 +7,10 @@ # with the LLVM bitcode generated by rustc. NM = "$(LLVM_BIN_DIR)"/llvm-nm -all: $(call NATIVE_STATICLIB,native-staticlib) +all: + # skip for now + +all_: # Build a staticlib and a rlib, the `native_func` symbol will be bundled into them $(RUSTC) bundled.rs --crate-type=staticlib --crate-type=rlib $(NM) $(TMPDIR)/libbundled.a | $(CGREP) -e "T _*native_func" diff --git a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.rs b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.rs deleted file mode 100644 index 066048795c8a2..0000000000000 --- a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.rs +++ /dev/null @@ -1,8 +0,0 @@ -// compile-flags: -Zunstable-options --crate-type rlib -// build-fail -// error-pattern: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs - -#[link(name = "mylib", kind = "static", modifiers = "+bundle,+whole-archive")] -extern "C" { } - -fn main() { } diff --git a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.stderr b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.stderr deleted file mode 100644 index 246efb8d627cb..0000000000000 --- a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.stderr +++ /dev/null @@ -1,6 +0,0 @@ -error: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs - -error: could not find native static library `mylib`, perhaps an -L flag is missing? - -error: aborting due to 2 previous errors - diff --git a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.rs b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.rs deleted file mode 100644 index 1d0768d99cffd..0000000000000 --- a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.rs +++ /dev/null @@ -1,7 +0,0 @@ -// Mixing +bundle and +whole-archive is not allowed - -// compile-flags: -l static:+bundle,+whole-archive=mylib -Zunstable-options --crate-type rlib -// build-fail -// error-pattern: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs - -fn main() { } diff --git a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.stderr b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.stderr deleted file mode 100644 index 246efb8d627cb..0000000000000 --- a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.stderr +++ /dev/null @@ -1,6 +0,0 @@ -error: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs - -error: could not find native static library `mylib`, perhaps an -L flag is missing? - -error: aborting due to 2 previous errors - From 8a842f048ee8b9c99032a720620ec10a02baf834 Mon Sep 17 00:00:00 2001 From: Daniil Belov Date: Mon, 1 Aug 2022 16:25:40 +0300 Subject: [PATCH 2/4] lto recovered, test added --- compiler/rustc_codegen_ssa/src/back/link.rs | 135 +++++++++++++++--- compiler/rustc_interface/src/tests.rs | 1 + compiler/rustc_session/src/options.rs | 2 + src/test/run-make/issue_99429/Makefile | 35 +++++ src/test/run-make/issue_99429/main.rs | 5 + src/test/run-make/issue_99429/native_dep_1.c | 3 + src/test/run-make/issue_99429/native_dep_2.c | 3 + src/test/run-make/issue_99429/native_dep_3.c | 3 + .../run-make/issue_99429/rust_dep_local.rs | 13 ++ src/test/run-make/issue_99429/rust_dep_up.rs | 13 ++ .../native-link-modifier-bundle/Makefile | 7 +- src/test/rustdoc-ui/z-help.stdout | 1 + .../mix-bundle-and-whole-archive-link-attr.rs | 8 ++ ...-bundle-and-whole-archive-link-attr.stderr | 5 + .../mix-bundle-and-whole-archive.rs | 7 + .../mix-bundle-and-whole-archive.stderr | 5 + 16 files changed, 224 insertions(+), 22 deletions(-) create mode 100644 src/test/run-make/issue_99429/Makefile create mode 100644 src/test/run-make/issue_99429/main.rs create mode 100644 src/test/run-make/issue_99429/native_dep_1.c create mode 100644 src/test/run-make/issue_99429/native_dep_2.c create mode 100644 src/test/run-make/issue_99429/native_dep_3.c create mode 100644 src/test/run-make/issue_99429/rust_dep_local.rs create mode 100644 src/test/run-make/issue_99429/rust_dep_up.rs create mode 100644 src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.rs create mode 100644 src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.stderr create mode 100644 src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.rs create mode 100644 src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.stderr diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index 6760ebfb1b4fa..83605b6815f23 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -300,7 +300,9 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>( } } - if flavor == RlibFlavor::StaticlibBase { + if flavor == RlibFlavor::StaticlibBase + || !sess.opts.unstable_opts.separate_native_rlib_dependencies + { // Note that in this loop we are ignoring the value of `lib.cfg`. That is, // we may not be configured to actually include a static library if we're // adding it here. That's because later when we consume this rlib we'll @@ -319,6 +321,13 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>( // metadata of the rlib we're generating somehow. for lib in codegen_results.crate_info.used_libraries.iter() { match lib.kind { + NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) } => { + if flavor == RlibFlavor::Normal { + sess.err( + "the linking modifiers `+bundle` and `+whole-archive` are not compatible \ + with each other when generating rlibs"); + } + } NativeLibKind::Static { bundle: None | Some(true), .. } => {} NativeLibKind::Static { bundle: Some(false), .. } | NativeLibKind::Dylib { .. } @@ -379,7 +388,9 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>( ab.add_file(&trailing_metadata); } - if flavor == RlibFlavor::Normal { + // add all native rlib dependencies + // archieves added to the end of .rlib archieve + if flavor == RlibFlavor::Normal && sess.opts.unstable_opts.separate_native_rlib_dependencies { for lib in codegen_results.crate_info.used_libraries.iter() { match lib.kind { NativeLibKind::Static { bundle: None | Some(true), .. } => {} @@ -2461,18 +2472,41 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( ) { let src = &codegen_results.crate_info.used_crate_source[&cnum]; let cratepath = &src.rlib.as_ref().unwrap().0; + let native_libs = &codegen_results.crate_info.native_libraries[&cnum]; + let crate_name = cratepath.file_name().unwrap().to_str().unwrap(); - cmd.link_rlib(&fix_windows_verbatim_for_gcc(cratepath)); - for lib in &codegen_results.crate_info.native_libraries[&cnum] { - let Some(name) = lib.name else { + // New rlib format + if sess.opts.unstable_opts.separate_native_rlib_dependencies { + cmd.link_rlib(&fix_windows_verbatim_for_gcc(cratepath)); + if are_upstream_rust_objects_already_included(sess) { + return; + } + for lib in native_libs { + let Some(name) = lib.name else { continue; }; - let name = name.as_str(); - if !relevant_lib(sess, lib) { - continue; - } - let verbatim = lib.verbatim.unwrap_or(false); - if let NativeLibKind::Static { bundle: Some(true) | None, whole_archive } = lib.kind { + let name = name.as_str(); + let NativeLibKind::Static { bundle: Some(true) | None, whole_archive } = lib.kind else { + continue; + }; + if !ignored_for_lto(sess, &codegen_results.crate_info, cnum) { + if name == METADATA_FILENAME { + continue; + } + let relevant = relevant_lib(sess, lib); + let crate_name = &crate_name[3..crate_name.len() - 5]; // chop off lib/.rlib + let canonical_name = crate_name.replace('-', "_"); + let is_rust_object = name.replace('-', "_").starts_with(&canonical_name) + && looks_like_rust_object_file(&name); + let is_builtins = sess.target.no_builtins + || !codegen_results.crate_info.is_no_builtins.contains(&cnum); + let skip_because_cfg_say_so = !relevant && !is_rust_object; + let skip_because_lto = is_rust_object && is_builtins; + if skip_because_cfg_say_so || skip_because_lto { + continue; + } + } + let verbatim = lib.verbatim.unwrap_or(false); if whole_archive == Some(true) { cmd.link_whole_staticlib(name, verbatim, &[PathBuf::from(tmpdir)]) } else { @@ -2480,15 +2514,82 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( cmd.include_path(tmpdir) } } + + B::unpack_archive(cratepath, tmpdir, |fname: &str| { + if !fname.ends_with(".a") { + return true; + } + return false; + }) + .unwrap(); + return; } - B::unpack_archive(cratepath, tmpdir, |fname: &str| { - if !fname.ends_with(".a") { - return true; + // Old rlib format + let mut link_upstream = |path: &Path| { + cmd.link_rlib(&fix_windows_verbatim_for_gcc(path)); + }; + // See the comment above in `link_staticlib` and `link_rlib` for why if + // there's a static library that's not relevant we skip all object + // files. + let skip_native = native_libs.iter().any(|lib| { + matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. }) + && !relevant_lib(sess, lib) + }); + + if (!are_upstream_rust_objects_already_included(sess) + || ignored_for_lto(sess, &codegen_results.crate_info, cnum)) + && !skip_native + { + link_upstream(cratepath); + return; + } + + let dst = tmpdir.join(cratepath.file_name().unwrap()); + + let crate_name = &crate_name[3..crate_name.len() - 5]; // chop off lib/.rlib + let canonical_name = crate_name.replace('-', "_"); + sess.prof.generic_activity_with_arg("link_altering_rlib", crate_name).run(|| { + let upstream_rust_objects_already_included = + are_upstream_rust_objects_already_included(sess); + let is_builtins = sess.target.no_builtins + || !codegen_results.crate_info.is_no_builtins.contains(&cnum); + + let mut archive = ::new(sess, &dst); + if let Err(e) = archive.add_archive(cratepath, move |f| { + if f == METADATA_FILENAME { + return true; + } + + let canonical = f.replace('-', "_"); + + let is_rust_object = + canonical.starts_with(&canonical_name) && looks_like_rust_object_file(&f); + + // If we've been requested to skip all native object files + // (those not generated by the rust compiler) then we can skip + // this file. See above for why we may want to do this. + let skip_because_cfg_say_so = skip_native && !is_rust_object; + + // If we're performing LTO and this is a rust-generated object + // file, then we don't need the object file as it's part of the + // LTO module. Note that `#![no_builtins]` is excluded from LTO, + // though, so we let that object file slide. + let skip_because_lto = + upstream_rust_objects_already_included && is_rust_object && is_builtins; + + if skip_because_cfg_say_so || skip_because_lto { + return true; + } + + false + }) { + sess.fatal(&format!("failed to build archive from rlib: {}", e)); } - return false; - }) - .unwrap(); + if archive.build() { + link_upstream(&dst); + } + }); } // Same thing as above, but for dynamic crates instead of static crates. diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index 21d9eaccf67e7..8d22d9c3d0e97 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -784,6 +784,7 @@ fn test_unstable_options_tracking_hash() { tracked!(sanitizer_memory_track_origins, 2); tracked!(sanitizer_recover, SanitizerSet::ADDRESS); tracked!(saturating_float_casts, Some(true)); + tracked!(separate_native_rlib_dependencies, true); tracked!(share_generics, Some(true)); tracked!(show_span, Some(String::from("abc"))); tracked!(simulate_remapped_rust_src_base, Some(PathBuf::from("/rustc/abc"))); diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 501997679f4bf..fbfdd7616d19f 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1510,6 +1510,8 @@ options! { `instructions:u` (retired instructions, userspace-only) `instructions-minus-irqs:u` (subtracting hardware interrupt counts for extra accuracy)" ), + separate_native_rlib_dependencies: bool = (false, parse_bool, [TRACKED], + "change rlib format to store native libraries as archieves"), share_generics: Option = (None, parse_opt_bool, [TRACKED], "make the current crate share its generic instantiations"), show_span: Option = (None, parse_opt_string, [TRACKED], diff --git a/src/test/run-make/issue_99429/Makefile b/src/test/run-make/issue_99429/Makefile new file mode 100644 index 0000000000000..8dffec365c0ee --- /dev/null +++ b/src/test/run-make/issue_99429/Makefile @@ -0,0 +1,35 @@ +-include ../../run-make-fulldeps/tools.mk + +# We're using the llvm-nm instead of the system nm to ensure it is compatible +# with the LLVM bitcode generated by rustc. +NM = "$(LLVM_BIN_DIR)"/llvm-nm + +all: $(call NATIVE_STATICLIB,native_dep_1) $(call NATIVE_STATICLIB,native_dep_2) $(call NATIVE_STATICLIB,native_dep_3) + # Build native libs + # $(RUSTC) native_dep_1.rs --crate-type=staticlib + # $(RUSTC) native_dep_2.rs --crate-type=staticlib + # $(RUSTC) native_dep_3.rs --crate-type=staticlib + $(NM) $(TMPDIR)/libnative_dep_1.a | $(CGREP) "T native_f1" + $(NM) $(TMPDIR)/libnative_dep_2.a | $(CGREP) "T native_f2" + $(NM) $(TMPDIR)/libnative_dep_3.a | $(CGREP) "T native_f3" + + # Build new rlibs + $(RUSTC) rust_dep_up.rs --crate-type=rlib -Zseparate_native_rlib_dependencies + $(NM) $(TMPDIR)/librust_dep_up.rlib | $(CGREP) "U native_f2" + $(NM) $(TMPDIR)/librust_dep_up.rlib | $(CGREP) "U native_f3" + $(NM) $(TMPDIR)/librust_dep_up.rlib | $(CGREP) -e "T.*rust_dep_up" + ar t $(TMPDIR)/librust_dep_up.rlib | $(CGREP) "libnative_dep_2.a" # FIXME: crossplatform + ar t $(TMPDIR)/librust_dep_up.rlib | $(CGREP) "libnative_dep_3.a" # FIXME: crossplatform + $(RUSTC) rust_dep_local.rs --extern rlib=$(TMPDIR)/librust_dep_up.rlib -Zseparate_native_rlib_dependencies --crate-type=rlib -Z unstable-options + $(NM) $(TMPDIR)/librust_dep_local.rlib | $(CGREP) "U native_f1" + $(NM) $(TMPDIR)/librust_dep_local.rlib | $(CGREP) -e "T.*rust_dep_local" + ar t $(TMPDIR)/librust_dep_local.rlib | $(CGREP) "libnative_dep_1.a" # FIXME: crossplatform + + # Build bin + $(RUSTC) main.rs --extern lib=$(TMPDIR)/librust_dep_local.rlib --crate-type=bin -Zseparate_native_rlib_dependencies --print link-args | $(CGREP) -e 'native_dep_1.*native_dep_2.*native_dep_3' + $(NM) $(TMPDIR)/main | $(CGREP) "T native_f1" + $(NM) $(TMPDIR)/main | $(CGREP) "T native_f2" + $(NM) $(TMPDIR)/main | $(CGREP) "T native_f3" + $(NM) $(TMPDIR)/main | $(CGREP) -e "T.*rust_dep_local" + $(NM) $(TMPDIR)/main | $(CGREP) -e "T.*rust_dep_up" + diff --git a/src/test/run-make/issue_99429/main.rs b/src/test/run-make/issue_99429/main.rs new file mode 100644 index 0000000000000..21b7066dfb676 --- /dev/null +++ b/src/test/run-make/issue_99429/main.rs @@ -0,0 +1,5 @@ +extern crate rust_dep_local; + +pub fn main() { + rust_dep_local::rust_dep_local(); +} diff --git a/src/test/run-make/issue_99429/native_dep_1.c b/src/test/run-make/issue_99429/native_dep_1.c new file mode 100644 index 0000000000000..8df82a26bda80 --- /dev/null +++ b/src/test/run-make/issue_99429/native_dep_1.c @@ -0,0 +1,3 @@ +int native_f1() { + return 1; +} diff --git a/src/test/run-make/issue_99429/native_dep_2.c b/src/test/run-make/issue_99429/native_dep_2.c new file mode 100644 index 0000000000000..929cb1054adc3 --- /dev/null +++ b/src/test/run-make/issue_99429/native_dep_2.c @@ -0,0 +1,3 @@ +int native_f2() { + return 2; +} diff --git a/src/test/run-make/issue_99429/native_dep_3.c b/src/test/run-make/issue_99429/native_dep_3.c new file mode 100644 index 0000000000000..dc7e61bbd5a70 --- /dev/null +++ b/src/test/run-make/issue_99429/native_dep_3.c @@ -0,0 +1,3 @@ +int native_f3() { + return 3; +} diff --git a/src/test/run-make/issue_99429/rust_dep_local.rs b/src/test/run-make/issue_99429/rust_dep_local.rs new file mode 100644 index 0000000000000..8280c7d6c5156 --- /dev/null +++ b/src/test/run-make/issue_99429/rust_dep_local.rs @@ -0,0 +1,13 @@ +#[link(name = "native_dep_1", kind = "static")] +extern "C" { + fn native_f1() -> i32; +} + +extern crate rust_dep_up; + +pub fn rust_dep_local() { + unsafe { + assert!(native_f1() == 1); + } + rust_dep_up::rust_dep_up(); +} diff --git a/src/test/run-make/issue_99429/rust_dep_up.rs b/src/test/run-make/issue_99429/rust_dep_up.rs new file mode 100644 index 0000000000000..edcd7c5212984 --- /dev/null +++ b/src/test/run-make/issue_99429/rust_dep_up.rs @@ -0,0 +1,13 @@ +#[link(name = "native_dep_2", kind = "static")] +#[link(name = "native_dep_3", kind = "static")] +extern "C" { + fn native_f2() -> i32; + fn native_f3() -> i32; +} + +pub fn rust_dep_up() { + unsafe { + assert!(native_f2() == 2); + assert!(native_f3() == 3); + } +} diff --git a/src/test/run-make/native-link-modifier-bundle/Makefile b/src/test/run-make/native-link-modifier-bundle/Makefile index 43a0dbcc676a6..3fe35c693bfc2 100644 --- a/src/test/run-make/native-link-modifier-bundle/Makefile +++ b/src/test/run-make/native-link-modifier-bundle/Makefile @@ -7,10 +7,7 @@ # with the LLVM bitcode generated by rustc. NM = "$(LLVM_BIN_DIR)"/llvm-nm -all: - # skip for now - -all_: +all: $(call NATIVE_STATICLIB,native-staticlib) # Build a staticlib and a rlib, the `native_func` symbol will be bundled into them $(RUSTC) bundled.rs --crate-type=staticlib --crate-type=rlib $(NM) $(TMPDIR)/libbundled.a | $(CGREP) -e "T _*native_func" @@ -33,4 +30,4 @@ all_: # Build a cdylib, `native-staticlib` will appear on the linker line because it was not bundled previously # The cdylib will contain the `native_func` symbol in the end $(RUSTC) cdylib-non-bundled.rs --crate-type=cdylib --print link-args | $(CGREP) -e '-l[" ]*native-staticlib' - $(NM) $(call DYLIB,cdylib_non_bundled) | $(CGREP) -e "[Tt] _*native_func" + $(NM) $(call DYLIB,cdylib_non_bundled) | $(CGREP) -e "[Tt] _*native_func" \ No newline at end of file diff --git a/src/test/rustdoc-ui/z-help.stdout b/src/test/rustdoc-ui/z-help.stdout index 86e42440bd0bc..5708f77cfd063 100644 --- a/src/test/rustdoc-ui/z-help.stdout +++ b/src/test/rustdoc-ui/z-help.stdout @@ -140,6 +140,7 @@ `wall-time` (monotonic clock, i.e. `std::time::Instant`) `instructions:u` (retired instructions, userspace-only) `instructions-minus-irqs:u` (subtracting hardware interrupt counts for extra accuracy) + -Z separate-native-rlib-dependencies=val -- change rlib format to store native libraries as archieves -Z share-generics=val -- make the current crate share its generic instantiations -Z show-span=val -- show spans for compiler debugging (expr|pat|ty) -Z span-debug=val -- forward proc_macro::Span's `Debug` impl to `Span` diff --git a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.rs b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.rs new file mode 100644 index 0000000000000..4812ef7b6d999 --- /dev/null +++ b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.rs @@ -0,0 +1,8 @@ +// compile-flags: -Zunstable-options --crate-type rlib +// build-fail +// error-pattern: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs + +#[link(name = "mylib", kind = "static", modifiers = "+bundle,+whole-archive")] +extern "C" {} + +fn main() {} diff --git a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.stderr b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.stderr new file mode 100644 index 0000000000000..b425422ce56b8 --- /dev/null +++ b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.stderr @@ -0,0 +1,5 @@ +error: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs + +error: could not find native static library `mylib`, perhaps an -L flag is missing? + +error: aborting due to 2 previous errors diff --git a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.rs b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.rs new file mode 100644 index 0000000000000..287a38d267c75 --- /dev/null +++ b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.rs @@ -0,0 +1,7 @@ +// Mixing +bundle and +whole-archive is not allowed + +// compile-flags: -l static:+bundle,+whole-archive=mylib -Zunstable-options --crate-type rlib +// build-fail +// error-pattern: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs + +fn main() {} diff --git a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.stderr b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.stderr new file mode 100644 index 0000000000000..b425422ce56b8 --- /dev/null +++ b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.stderr @@ -0,0 +1,5 @@ +error: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs + +error: could not find native static library `mylib`, perhaps an -L flag is missing? + +error: aborting due to 2 previous errors From efe09c0f6bc379edcffaf14de5549f1175f7943c Mon Sep 17 00:00:00 2001 From: Daniil Belov Date: Mon, 1 Aug 2022 16:31:35 +0300 Subject: [PATCH 3/4] fix --- src/test/run-make/native-link-modifier-bundle/Makefile | 2 +- .../mix-bundle-and-whole-archive-link-attr.rs | 4 ++-- .../mix-bundle-and-whole-archive-link-attr.stderr | 1 + .../native-library-link-flags/mix-bundle-and-whole-archive.rs | 2 +- .../mix-bundle-and-whole-archive.stderr | 1 + 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/test/run-make/native-link-modifier-bundle/Makefile b/src/test/run-make/native-link-modifier-bundle/Makefile index 3fe35c693bfc2..e4b0f74ac26ac 100644 --- a/src/test/run-make/native-link-modifier-bundle/Makefile +++ b/src/test/run-make/native-link-modifier-bundle/Makefile @@ -30,4 +30,4 @@ all: $(call NATIVE_STATICLIB,native-staticlib) # Build a cdylib, `native-staticlib` will appear on the linker line because it was not bundled previously # The cdylib will contain the `native_func` symbol in the end $(RUSTC) cdylib-non-bundled.rs --crate-type=cdylib --print link-args | $(CGREP) -e '-l[" ]*native-staticlib' - $(NM) $(call DYLIB,cdylib_non_bundled) | $(CGREP) -e "[Tt] _*native_func" \ No newline at end of file + $(NM) $(call DYLIB,cdylib_non_bundled) | $(CGREP) -e "[Tt] _*native_func" diff --git a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.rs b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.rs index 4812ef7b6d999..066048795c8a2 100644 --- a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.rs +++ b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.rs @@ -3,6 +3,6 @@ // error-pattern: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs #[link(name = "mylib", kind = "static", modifiers = "+bundle,+whole-archive")] -extern "C" {} +extern "C" { } -fn main() {} +fn main() { } diff --git a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.stderr b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.stderr index b425422ce56b8..246efb8d627cb 100644 --- a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.stderr +++ b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.stderr @@ -3,3 +3,4 @@ error: the linking modifiers `+bundle` and `+whole-archive` are not compatible w error: could not find native static library `mylib`, perhaps an -L flag is missing? error: aborting due to 2 previous errors + diff --git a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.rs b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.rs index 287a38d267c75..1d0768d99cffd 100644 --- a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.rs +++ b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.rs @@ -4,4 +4,4 @@ // build-fail // error-pattern: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs -fn main() {} +fn main() { } diff --git a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.stderr b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.stderr index b425422ce56b8..246efb8d627cb 100644 --- a/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.stderr +++ b/src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.stderr @@ -3,3 +3,4 @@ error: the linking modifiers `+bundle` and `+whole-archive` are not compatible w error: could not find native static library `mylib`, perhaps an -L flag is missing? error: aborting due to 2 previous errors + From 417635e6ee38bfeaec73fd6a7129aa924df95744 Mon Sep 17 00:00:00 2001 From: Daniil Belov Date: Tue, 2 Aug 2022 15:31:29 +0300 Subject: [PATCH 4/4] fix add_upstream_rust_crates --- compiler/rustc_codegen_ssa/src/back/link.rs | 117 ++++++++++---------- 1 file changed, 60 insertions(+), 57 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index 83605b6815f23..e61d3b9142d31 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -2390,6 +2390,12 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( // will provide them to the linker itself. if sess.opts.unstable_opts.link_native_libraries { let mut last = (None, NativeLibKind::Unspecified, None); + if sess.opts.unstable_opts.separate_native_rlib_dependencies { + B::unpack_archive(&src.rlib.as_ref().unwrap().0, tmpdir, |fname: &str| { + !fname.ends_with(".a") + }) + .unwrap(); + } for lib in &codegen_results.crate_info.native_libraries[&cnum] { let Some(name) = lib.name else { continue; @@ -2405,7 +2411,29 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( } else { (lib.name, lib.kind, lib.verbatim) }; - + /* + match lib.kind { + NativeLibKind::Static { bundle: Some(false), whole_archive } + | NativeLibKind::Static { bundle: Some(true) | None, whole_archive } + if sess.opts.unstable_opts.separate_native_rlib_dependencies => + { + let verbatim = lib.verbatim.unwrap_or(false); + let tmp_path = &[PathBuf::from(tmpdir)]; + let path: &[PathBuf] = + if sess.opts.unstable_opts.separate_native_rlib_dependencies { + tmp_path + } else { + search_path.get_or_init(|| archive_search_paths(sess)) + }; + if whole_archive == Some(true) { + cmd.link_whole_staticlib(name, verbatim, path); + } else { + cmd.link_staticlib(name, verbatim); + } + } + _ => {} + }; + */ if let NativeLibKind::Static { bundle: Some(false), whole_archive } = lib.kind { @@ -2420,6 +2448,25 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( cmd.link_staticlib(name, verbatim); } } + if sess.opts.unstable_opts.separate_native_rlib_dependencies { + if let NativeLibKind::Static { + bundle: Some(true) | None, + whole_archive, + } = lib.kind + { + let verbatim = lib.verbatim.unwrap_or(false); + if whole_archive == Some(true) { + cmd.link_whole_staticlib( + name, + verbatim, + &[PathBuf::from(tmpdir)], + ); + } else { + cmd.link_staticlib(name, verbatim); + } + } + } + // } } } @@ -2472,66 +2519,15 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( ) { let src = &codegen_results.crate_info.used_crate_source[&cnum]; let cratepath = &src.rlib.as_ref().unwrap().0; - let native_libs = &codegen_results.crate_info.native_libraries[&cnum]; - let crate_name = cratepath.file_name().unwrap().to_str().unwrap(); - - // New rlib format - if sess.opts.unstable_opts.separate_native_rlib_dependencies { - cmd.link_rlib(&fix_windows_verbatim_for_gcc(cratepath)); - if are_upstream_rust_objects_already_included(sess) { - return; - } - for lib in native_libs { - let Some(name) = lib.name else { - continue; - }; - let name = name.as_str(); - let NativeLibKind::Static { bundle: Some(true) | None, whole_archive } = lib.kind else { - continue; - }; - if !ignored_for_lto(sess, &codegen_results.crate_info, cnum) { - if name == METADATA_FILENAME { - continue; - } - let relevant = relevant_lib(sess, lib); - let crate_name = &crate_name[3..crate_name.len() - 5]; // chop off lib/.rlib - let canonical_name = crate_name.replace('-', "_"); - let is_rust_object = name.replace('-', "_").starts_with(&canonical_name) - && looks_like_rust_object_file(&name); - let is_builtins = sess.target.no_builtins - || !codegen_results.crate_info.is_no_builtins.contains(&cnum); - let skip_because_cfg_say_so = !relevant && !is_rust_object; - let skip_because_lto = is_rust_object && is_builtins; - if skip_because_cfg_say_so || skip_because_lto { - continue; - } - } - let verbatim = lib.verbatim.unwrap_or(false); - if whole_archive == Some(true) { - cmd.link_whole_staticlib(name, verbatim, &[PathBuf::from(tmpdir)]) - } else { - cmd.link_staticlib(name, verbatim); - cmd.include_path(tmpdir) - } - } - B::unpack_archive(cratepath, tmpdir, |fname: &str| { - if !fname.ends_with(".a") { - return true; - } - return false; - }) - .unwrap(); - return; - } - - // Old rlib format let mut link_upstream = |path: &Path| { cmd.link_rlib(&fix_windows_verbatim_for_gcc(path)); }; + // See the comment above in `link_staticlib` and `link_rlib` for why if // there's a static library that's not relevant we skip all object // files. + let native_libs = &codegen_results.crate_info.native_libraries[&cnum]; let skip_native = native_libs.iter().any(|lib| { matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. }) && !relevant_lib(sess, lib) @@ -2546,10 +2542,11 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( } let dst = tmpdir.join(cratepath.file_name().unwrap()); + let name = cratepath.file_name().unwrap().to_str().unwrap(); + let name = &name[3..name.len() - 5]; // chop off lib/.rlib - let crate_name = &crate_name[3..crate_name.len() - 5]; // chop off lib/.rlib - let canonical_name = crate_name.replace('-', "_"); - sess.prof.generic_activity_with_arg("link_altering_rlib", crate_name).run(|| { + sess.prof.generic_activity_with_arg("link_altering_rlib", name).run(|| { + let canonical_name = name.replace('-', "_"); let upstream_rust_objects_already_included = are_upstream_rust_objects_already_included(sess); let is_builtins = sess.target.no_builtins @@ -2578,6 +2575,12 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( let skip_because_lto = upstream_rust_objects_already_included && is_rust_object && is_builtins; + if + // sess.opts.unstable_opts.separate_native_rlib_dependencies && + f.ends_with(".a") { + return true; + } + if skip_because_cfg_say_so || skip_because_lto { return true; }