Merge pull request #482 from antoyo/master

Sync from fork

Author: antoyo

.github/workflows/ci.yml

@@ -80,7 +80,14 @@ jobs:
         ./y.sh prepare --only-libcore
         ./y.sh build
         cargo test
-        ./y.sh clean all
+
+    - name: Run y.sh cargo build
+      run: |
+        ./y.sh cargo build --manifest-path tests/hello-world/Cargo.toml
+
+    - name: Clean
+      run: |
+        ./y.sh clean all      
 
     - name: Prepare dependencies
       run: |
@@ -96,7 +103,10 @@ jobs:
         ./y.sh test --release --clean --build-sysroot ${{ matrix.commands }}
 
     - name: Check formatting
-      run: cargo fmt -- --check
+      run: |
+        cargo fmt -- --check
+        cd build_system
+        cargo fmt -- --check
 
     - name: clippy
       run: |

.github/workflows/m68k.yml

@@ -54,13 +54,7 @@ jobs:
       run: curl -LO https://github.com/cross-cg-gcc-tools/cross-gcc/releases/latest/download/gcc-m68k-13.deb
 
     - name: Download VM artifact
-      uses: dawidd6/action-download-artifact@v2
-      with:
-          workflow: m68k.yml
-          name: debian-m68k
-          repo: cross-cg-gcc-tools/vms
-          branch: master
-          event: push
+      run: curl -LO https://github.com/cross-cg-gcc-tools/vms/releases/latest/download/debian-m68k.img
 
     - name: Setup path to libgccjit
       run: |
@@ -88,6 +82,13 @@ jobs:
         sudo mount debian-m68k.img vm
         sudo cp $(which qemu-m68k-static) vm/usr/bin/
 
+    - name: Build sample project with target defined as JSON spec
+      run: |
+        ./y.sh prepare --only-libcore --cross
+        ./y.sh build --target-triple m68k-unknown-linux-gnu --target ${{ github.workspace }}/target_specs/m68k-unknown-linux-gnu.json
+        ./y.sh cargo build --manifest-path=./tests/hello-world/Cargo.toml --target ${{ github.workspace }}/target_specs/m68k-unknown-linux-gnu.json
+        ./y.sh clean all
+
     - name: Build
       run: |
         ./y.sh prepare --only-libcore --cross

.github/workflows/release.yml

@@ -62,12 +62,12 @@ jobs:
         git config --global user.email "[email protected]"
         git config --global user.name "User"
         ./y.sh prepare
-        # FIXME(antoyo): we cannot enable LTO for stdarch tests currently because of some failing LTO tests using proc-macros.
-        echo -n 'lto = "fat"' >> build_sysroot/Cargo.toml
 
     - name: Add more failing tests because of undefined symbol errors (FIXME)
       run: cat tests/failing-lto-tests.txt >> tests/failing-ui-tests.txt
 
     - name: Run tests
       run: |
+        # FIXME(antoyo): we cannot enable LTO for stdarch tests currently because of some failing LTO tests using proc-macros.
+        echo -n 'lto = "fat"' >> build_system/build_sysroot/Cargo.toml
         EMBED_LTO_BITCODE=1 ./y.sh test --release --clean --release-sysroot --build-sysroot ${{ matrix.commands }}

.github/workflows/stdarch.yml

@@ -89,12 +89,10 @@ jobs:
     - name: Run stdarch tests
       if: ${{ !matrix.cargo_runner }}
       run: |
-        cd build_sysroot/sysroot_src/library/stdarch/
-        CHANNEL=release TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ../../../../y.sh cargo test
+        CHANNEL=release TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ./y.sh cargo test --manifest-path build/build_sysroot/sysroot_src/library/stdarch/Cargo.toml
 
     - name: Run stdarch tests
       if: ${{ matrix.cargo_runner }}
       run: |
-        cd build_sysroot/sysroot_src/library/stdarch/
         # FIXME: these tests fail when the sysroot is compiled with LTO because of a missing symbol in proc-macro.
-        STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ../../../../y.sh cargo test -- --skip rtm --skip tbm --skip sse4a
+        STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ./y.sh cargo test --manifest-path build/build_sysroot/sysroot_src/library/stdarch/Cargo.toml -- --skip rtm --skip tbm --skip sse4a

.gitignore

@@ -6,10 +6,6 @@ perf.data
 perf.data.old
 *.events
 *.string*
-/build_sysroot/sysroot
-/build_sysroot/sysroot_src
-/build_sysroot/Cargo.lock
-/build_sysroot/test_target/Cargo.lock
 gimple*
 *asm
 res

build_sysroot/Cargo.toml renamed to build_system/build_sysroot/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-authors = ["bjorn3 <[email protected]>"]
+authors = ["rustc_codegen_gcc devs"]
 name = "sysroot"
 version = "0.0.0"
 resolver = "2"
@@ -18,5 +18,5 @@ rustc-std-workspace-alloc = { path = "./sysroot_src/library/rustc-std-workspace-
 rustc-std-workspace-std = { path = "./sysroot_src/library/rustc-std-workspace-std" }
 
 [profile.release]
-debug = true
+debug = "limited"
 #lto = "fat" # TODO(antoyo): re-enable when the failing LTO tests regarding proc-macros are fixed.

build_sysroot/src/lib.rs renamed to build_system/build_sysroot/lib.rs

@@ -1,5 +1,7 @@
 use crate::config::{Channel, ConfigInfo};
-use crate::utils::{run_command, run_command_with_output_and_env, walk_dir};
+use crate::utils::{
+    copy_file, create_dir, get_sysroot_dir, run_command, run_command_with_output_and_env, walk_dir,
+};
 use std::collections::HashMap;
 use std::ffi::OsStr;
 use std::fs;
@@ -55,8 +57,7 @@ impl BuildArg {
     }
 }
 
-pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Result<(), String> {
-    let start_dir = Path::new("build_sysroot");
+fn cleanup_sysroot_previous_build(start_dir: &Path) {
     // Cleanup for previous run
     // Clean target dir except for build scripts and incremental cache
     let _ = walk_dir(
@@ -100,6 +101,26 @@ pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Resu
     let _ = fs::remove_file(start_dir.join("Cargo.lock"));
     let _ = fs::remove_file(start_dir.join("test_target/Cargo.lock"));
     let _ = fs::remove_dir_all(start_dir.join("sysroot"));
+}
+
+pub fn create_build_sysroot_content(start_dir: &Path) -> Result<(), String> {
+    if !start_dir.is_dir() {
+        create_dir(start_dir)?;
+    }
+    copy_file("build_system/build_sysroot/Cargo.toml", &start_dir.join("Cargo.toml"))?;
+
+    let src_dir = start_dir.join("src");
+    if !src_dir.is_dir() {
+        create_dir(&src_dir)?;
+    }
+    copy_file("build_system/build_sysroot/lib.rs", &start_dir.join("src/lib.rs"))
+}
+
+pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Result<(), String> {
+    let start_dir = get_sysroot_dir();
+
+    cleanup_sysroot_previous_build(&start_dir);
+    create_build_sysroot_content(&start_dir)?;
 
     // Builds libs
     let mut rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default();
@@ -110,7 +131,6 @@ pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Resu
     if config.no_default_features {
         rustflags.push_str(" -Csymbol-mangling-version=v0");
     }
-    let mut env = env.clone();
 
     let mut args: Vec<&dyn AsRef<OsStr>> = vec![&"cargo", &"build", &"--target", &config.target];
 
@@ -127,18 +147,13 @@ pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Resu
         "debug"
     };
 
+    let mut env = env.clone();
     env.insert("RUSTFLAGS".to_string(), rustflags);
-    run_command_with_output_and_env(&args, Some(start_dir), Some(&env))?;
+    run_command_with_output_and_env(&args, Some(&start_dir), Some(&env))?;
 
     // Copy files to sysroot
     let sysroot_path = start_dir.join(format!("sysroot/lib/rustlib/{}/lib/", config.target_triple));
-    fs::create_dir_all(&sysroot_path).map_err(|error| {
-        format!(
-            "Failed to create directory `{}`: {:?}",
-            sysroot_path.display(),
-            error
-        )
-    })?;
+    create_dir(&sysroot_path)?;
     let copier = |dir_to_copy: &Path| {
         // FIXME: should not use shell command!
         run_command(&[&"cp", &"-r", &dir_to_copy, &sysroot_path], None).map(|_| ())
@@ -151,43 +166,20 @@ pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Resu
 
     // Copy the source files to the sysroot (Rust for Linux needs this).
     let sysroot_src_path = start_dir.join("sysroot/lib/rustlib/src/rust");
-    fs::create_dir_all(&sysroot_src_path).map_err(|error| {
-        format!(
-            "Failed to create directory `{}`: {:?}",
-            sysroot_src_path.display(),
-            error
-        )
-    })?;
-    run_command(
-        &[
-            &"cp",
-            &"-r",
-            &start_dir.join("sysroot_src/library/"),
-            &sysroot_src_path,
-        ],
-        None,
-    )?;
+    create_dir(&sysroot_src_path)?;
+    run_command(&[&"cp", &"-r", &start_dir.join("sysroot_src/library/"), &sysroot_src_path], None)?;
 
     Ok(())
 }
 
 fn build_codegen(args: &mut BuildArg) -> Result<(), String> {
     let mut env = HashMap::new();
 
-    env.insert(
-        "LD_LIBRARY_PATH".to_string(),
-        args.config_info.gcc_path.clone(),
-    );
-    env.insert(
-        "LIBRARY_PATH".to_string(),
-        args.config_info.gcc_path.clone(),
-    );
+    env.insert("LD_LIBRARY_PATH".to_string(), args.config_info.gcc_path.clone());
+    env.insert("LIBRARY_PATH".to_string(), args.config_info.gcc_path.clone());
 
     if args.config_info.no_default_features {
-        env.insert(
-            "RUSTFLAGS".to_string(),
-            "-Csymbol-mangling-version=v0".to_string(),
-        );
+        env.insert("RUSTFLAGS".to_string(), "-Csymbol-mangling-version=v0".to_string());
     }
 
     let mut command: Vec<&dyn AsRef<OsStr>> = vec![&"cargo", &"rustc"];
@@ -212,12 +204,7 @@ fn build_codegen(args: &mut BuildArg) -> Result<(), String> {
     // We voluntarily ignore the error.
     let _ = fs::remove_dir_all("target/out");
     let gccjit_target = "target/out/gccjit";
-    fs::create_dir_all(gccjit_target).map_err(|error| {
-        format!(
-            "Failed to create directory `{}`: {:?}",
-            gccjit_target, error
-        )
-    })?;
+    create_dir(gccjit_target)?;
 
     println!("[BUILD] sysroot");
     build_sysroot(&env, &args.config_info)?;

build_system/src/build.rs

@@ -16,9 +16,7 @@ fn args() -> Result<Option<Vec<String>>, String> {
     }
     let args = std::env::args().skip(2).collect::<Vec<_>>();
     if args.is_empty() {
-        return Err(
-            "Expected at least one argument for `cargo` subcommand, found none".to_string(),
-        );
+        return Err("Expected at least one argument for `cargo` subcommand, found none".to_string());
     }
     Ok(Some(args))
 }
@@ -48,17 +46,10 @@ pub fn run() -> Result<(), String> {
     let current_exe = std::env::current_exe()
         .and_then(|path| path.canonicalize())
         .map_err(|error| format!("Failed to get current exe path: {:?}", error))?;
-    let mut parent_dir = current_exe
-        .components()
-        .map(|comp| comp.as_os_str())
-        .collect::<Vec<_>>();
+    let mut parent_dir = current_exe.components().map(|comp| comp.as_os_str()).collect::<Vec<_>>();
     // We run this script from "build_system/target/release/y", so we need to remove these elements.
     for to_remove in &["y", "release", "target", "build_system"] {
-        if parent_dir
-            .last()
-            .map(|part| part == to_remove)
-            .unwrap_or(false)
-        {
+        if parent_dir.last().map(|part| part == to_remove).unwrap_or(false) {
             parent_dir.pop();
         } else {
             return Err(format!(
@@ -69,11 +60,7 @@ pub fn run() -> Result<(), String> {
     }
     let parent_dir = PathBuf::from(parent_dir.join(&OsStr::new("/")));
     std::env::set_current_dir(&parent_dir).map_err(|error| {
-        format!(
-            "Failed to go to `{}` folder: {:?}",
-            parent_dir.display(),
-            error
-        )
+        format!("Failed to go to `{}` folder: {:?}", parent_dir.display(), error)
     })?;
 
     let mut env: HashMap<String, String> = std::env::vars().collect();
@@ -92,11 +79,7 @@ pub fn run() -> Result<(), String> {
 
     // We go back to the original folder since we now have set up everything we needed.
     std::env::set_current_dir(&current_dir).map_err(|error| {
-        format!(
-            "Failed to go back to `{}` folder: {:?}",
-            current_dir.display(),
-            error
-        )
+        format!("Failed to go back to `{}` folder: {:?}", current_dir.display(), error)
     })?;
 
     let rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default();

build_system/src/cargo.rs

@@ -1,4 +1,4 @@
-use crate::utils::{remove_file, run_command};
+use crate::utils::{get_sysroot_dir, remove_file, run_command};
 
 use std::fs::remove_dir_all;
 use std::path::Path;
@@ -42,11 +42,12 @@ fn usage() {
 }
 
 fn clean_all() -> Result<(), String> {
+    let build_sysroot = get_sysroot_dir();
     let dirs_to_remove = [
-        "target",
-        "build_sysroot/sysroot",
-        "build_sysroot/sysroot_src",
-        "build_sysroot/target",
+        "target".into(),
+        build_sysroot.join("sysroot"),
+        build_sysroot.join("sysroot_src"),
+        build_sysroot.join("target"),
     ];
     for dir in dirs_to_remove {
         let _ = remove_dir_all(dir);
@@ -56,10 +57,11 @@ fn clean_all() -> Result<(), String> {
         let _ = remove_dir_all(Path::new(crate::BUILD_DIR).join(dir));
     }
 
-    let files_to_remove = ["build_sysroot/Cargo.lock", "perf.data", "perf.data.old"];
+    let files_to_remove =
+        [build_sysroot.join("Cargo.lock"), "perf.data".into(), "perf.data.old".into()];
 
     for file in files_to_remove {
-        let _ = remove_file(file);
+        let _ = remove_file(&file);
     }
 
     println!("Successfully ran `clean all`");