// SPDX-License-Identifier: MPL-2.0 #include "vybuild.hpp" #include #include #include #include #include #include #include #include #include #include #include #include "elfio.hpp" #include "common/versions.h" using namespace std; extern int get_deepness(); queue> vyld_compilation_tasks; mutex vyld_compilation_tasks_mutex; void print_deepness_vyld(string text) { cout<<"[VyBuild] "; for (int i=0;i split_args_shell_vyld(const string& input) { wordexp_t result; vector args; if (wordexp(input.c_str(),&result,0)==0) { for (size_t i=0;i argv; auto args=split_args_shell_vyld(command); args[0]=find_executable_vyld(args[0]).c_str(); for (auto &a:args) argv.push_back(a.c_str()); argv.push_back(nullptr); execvp(argv[0],const_cast(argv.data())); cout<0) { buffer< headers_from_dfile_vyld(string dfile_path) { if (!fs::exists(dfile_path)) throw std::runtime_error(dfile_path+" doesn't exist."); ifstream dfile(dfile_path); vector deps; string line,all; while (getline(dfile,line)) { if (!line.empty() && line.back()=='\\') { line.pop_back(); all+=line+" "; } else { all+=line+ " "; } } auto pos=all.find(':'); if (pos==string::npos) return deps; string rest=all.substr(pos+1); istringstream iss(rest); string token; while (iss>>token) { deps.push_back(token); } return deps; } struct vyx_addresses { uint64_t text_offset; uint64_t stack_base; }; uint64_t rand_in_range(uint64_t min,uint64_t max) { static random_device rd; static mt19937_64 gen(rd()); std::uniform_int_distribution dis(min, max); return dis(gen); } uint64_t random_canonical(uint64_t start,uint64_t end) { uint64_t addr=rand_in_range(start/4096,end/4096)*4096; return addr; } vyx_addresses generate_vyx_addresses() { vyx_addresses addr; addr.text_offset=random_canonical(0xFFFF800000000000ULL,0xFFFF900000000000ULL-1); addr.stack_base=random_canonical(0xFFFFF00000000000ULL,0xFFFFFF8000000000ULL-0x100000ULL); return addr; } size_t pad_size(size_t value,size_t multiple) { if (multiple==0) return value; size_t remainder=value%multiple; if (remainder==0) return value; return value+multiple-remainder; } #pragma pack(1) struct payload_header { uint8_t sig[8]={'V','P','A','Y','L','O','A','D'}; uint32_t payload_size_bytes=0; uint32_t payload_pages_count=0; uint8_t map_flags=0; uint64_t load_address=0; }; #pragma pack() #define EXECUTABLE (1<<0) #define WRITABLE (1<<1) static bool char_to_value(char c,uint8_t &out) { if (c>='0' && c<='9') { out=c-'0'; return true; } if (c>='a' && c<='f') { out=c-'a'+10; return true; } if (c>='A' && c<='F') { out=c-'A'+10; return true; } return false; } uint64_t string_hex_to_u64(const std::string &str){ if (str.size()<3) return false; if (!(str[0]=='0' && (str[1]=='x'))) return false; uint64_t value=0; for (size_t i=2;i generate_payloads(const json& payloads) { vector out={}; if (payloads.size()==0) return out; for (auto &p:payloads) { if (!p.is_object()) throw std::runtime_error("Payload must be stored in a dictionnary."); vybuild::actions::check_arg_string(p,"source_file"); if (!p.contains("is_executable")) throw std::runtime_error("Is executable isn't provided."); if (!p["is_executable"].is_boolean()) throw std::runtime_error("Is executable isn't a boolean."); if (!p.contains("is_writable")) throw std::runtime_error("Is writable isn't provided."); if (!p["is_writable"].is_boolean()) throw std::runtime_error("Is writbale isn't a boolean."); if (!p.contains("expected_va")) throw std::runtime_error("Expected VA isn't provided."); if (!p["expected_va"].is_string()) throw std::runtime_error("Expected VA isn't a string."); fs::path source_file=p["source_file"].get(); bool source_file_variables=p["source_file_variables"].get(); if (source_file_variables) source_file=vybuild::parser::string_vars(source_file); if (!fs::exists(source_file)) throw std::runtime_error("Source file doesn't exist: "+source_file.string()); if (!fs::is_regular_file(source_file)) throw std::runtime_error("Source file isn't a regular file: "+source_file.string()); size_t payload_size=fs::file_size(source_file); size_t payload_page_count=pad_size(payload_size,4096)/4096; uint64_t va=string_hex_to_u64(p["expected_va"].get()); if (va==0 || va%4096!=0) throw std::runtime_error("Invalid expected VA: expected VA must be non null and 4KB aligned."); payload_header header; header.payload_size_bytes=payload_size; header.payload_pages_count=payload_page_count; header.load_address=va; if (p["is_executable"].get()) header.map_flags|=EXECUTABLE; if (p["is_writable"].get()) header.map_flags|=WRITABLE; ifstream payload_bin(source_file,ios::binary); if (!payload_bin) throw std::runtime_error("VyldCompilationAction: Cannot open payload source file: "+source_file.string()); vector payload_content(payload_size); payload_bin.read(reinterpret_cast(payload_content.data()),payload_size); if (!payload_bin) throw std::runtime_error("VyldCompilationAction: Cannot read payload source file: "+source_file.string()); vector final_payload(sizeof(payload_header)); memcpy(final_payload.data(),(uint8_t*)&header,sizeof(payload_header)); final_payload.insert(final_payload.end(),payload_content.begin(),payload_content.end()); out.insert(out.end(),final_payload.begin(),final_payload.end()); } return out; } vybuild::actions::ActionStatus parallel_vyld_compilation(string source,string compiler_path,string compiler_hash,vector compiler_args,string dfile_path,string header_command,string output_file) { if (compiler_hash!="") { string c_content=""; auto vars_list=vybuild::vars::get_global_variables_list(); if (header_command!="") { string dep_command=header_command; regex regvar1("%dfile%"); dep_command=regex_replace(dep_command,regvar1,dfile_path); regex regvar2("%sourcefile%"); dep_command=regex_replace(dep_command,regvar2,source); int res=std::system(dep_command.c_str()); if (res!=0) { print_deepness_vyld("ParallelVyldCompilation: Cannot get headers list for file \""+source+"\" with command: "+dep_command); return vybuild::actions::ActionStatus::Fail; } auto headers=headers_from_dfile_vyld(dfile_path); for (auto &h:headers) { ifstream header(h); vector header_data(fs::file_size(fs::absolute(h))); if (!header) { print_deepness_vyld("ParallelVyldCompilation: Couldn't open source/header file: "+h); return vybuild::actions::ActionStatus::SystemError; } header.read(header_data.data(),header_data.size()); if (!header) { print_deepness_vyld("ParallelVyldCompilation: Couldn't read source/header file: "+h); return vybuild::actions::ActionStatus::SystemError; } c_content+=string(header_data.begin(),header_data.end()); header.close(); } fs::remove(dfile_path); } c_content+="\n"+compiler_path; c_content+="\n"+compiler_hash; for (auto& a:compiler_args) c_content+="\n"+a; string key=vybuild::cache::xxhash_to_string(XXH3_128bits(c_content.data(),c_content.size())); if (vybuild::cache::cache_contains(key)) { vybuild::cache::extract_file(key,output_file); return vybuild::actions::ActionStatus::Success; } int pipefd[2]; if (pipe(pipefd)==-1) { print_deepness_vyld("ParallelVyldCompilation: An error happened while creating pipe."); return vybuild::actions::ActionStatus::SystemError; } pid_t pid=fork(); if (pid<0) { print_deepness_vyld("ParallelVyldCompilation: An error happened while forking."); return vybuild::actions::ActionStatus::SystemError; } if (pid==0) { close(pipefd[0]); dup2(pipefd[1],STDOUT_FILENO); dup2(pipefd[1],STDERR_FILENO); close(pipefd[1]); std::vector argv={compiler_path.c_str()}; argv.push_back("-c"); argv.push_back(source.c_str()); argv.push_back("-o"); argv.push_back(output_file.c_str()); for (int i=0;i(argv.data())); print_deepness_vyld("ParallelVyldCompilation: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer< argv={compiler_path.c_str()}; argv.push_back("-c"); argv.push_back(source.c_str()); argv.push_back("-o"); argv.push_back(output_file.c_str()); for (int i=0;i(argv.data())); print_deepness_vyld("ParallelVyldCompilation: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer< task; { std::lock_guard lock(vyld_compilation_tasks_mutex); if (vyld_compilation_tasks.empty()) break; task=std::move(vyld_compilation_tasks.front()); vyld_compilation_tasks.pop(); } auto status=task(); if (status!=vybuild::actions::ActionStatus::Success) { return status; } } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::VyldCompilationAction::run_action() { if (this->source_files_variables) this->source_files=vybuild::parser::vector_string_vars(this->source_files); if (this->source_files_path_completion) this->source_files=vybuild::parser::vector_string_path_completions(this->source_files,".c"); if (this->source_asm_variables) this->source_asm=vybuild::parser::vector_string_vars(this->source_asm); if (this->source_asm_path_completion) this->source_asm=vybuild::parser::vector_string_path_completions(this->source_asm,".asm"); if (this->compiler_variables) this->compiler=vybuild::parser::string_vars(this->compiler); if (this->assembler_variables) this->assembler=vybuild::parser::string_vars(this->assembler); if (this->linker_variables) this->linker=vybuild::parser::string_vars(this->linker); if (this->compiler_flags_variables) this->compiler_flags=vybuild::parser::string_vars(this->compiler_flags); if (this->output_file_variables) this->output_file=vybuild::parser::string_vars(this->output_file); if (this->headers_command_variables) this->headers_command=vybuild::parser::string_vars(this->headers_command); string compiler_path=find_executable_vyld(this->compiler); string compiler_hash=""; auto vars_list=vybuild::vars::get_global_variables_list(); if (this->object_files_cache_authorized) { ifstream compiler_bin(compiler_path,ios::binary); if (!compiler_bin) { print_deepness_vyld("VyldCompilationAction: Cannot open compiler binary: "+compiler_path); return vybuild::actions::ActionStatus::SystemError; } size_t compiler_size=fs::file_size(compiler_path); vector compiler_data(compiler_size); compiler_bin.read(reinterpret_cast(compiler_data.data()),compiler_size); if (!compiler_bin) { print_deepness_vyld("VyldCompilationAction: Cannot read compiler binary: "+compiler_path); return vybuild::actions::ActionStatus::SystemError; } compiler_hash=vybuild::cache::xxhash_to_string(XXH3_128bits(compiler_data.data(),compiler_data.size())); for (auto &v:vars_list) { if (v.starts_with("\\$__VYBUILD_COMPILER_HASH_")) { compiler_hash+="\n"+get_command_output_vyld(vybuild::vars::get_global_variable(v)); } } } char buf[PATH_MAX]; ssize_t len=readlink("/proc/self/exe",buf,sizeof(buf)-1); fs::path exe_path; if (len!=-1) { buf[len]='\0'; exe_path=fs::path(buf); } else { print_deepness_vyld("VyldCompilationAction: Couldn't obtain vybuild executable path."); return vybuild::actions::ActionStatus::SystemError; } exe_path=exe_path.parent_path(); if (!fs::exists(exe_path/"_vyx_start.c")) { print_deepness_vyld("VyldCompilationAction: _vyx_start.c isn't accessible."); return vybuild::actions::ActionStatus::SystemError; } this->source_files.insert(this->source_files.begin(),string(exe_path/"_vyx_start.c")); size_t nb_thread=thread::hardware_concurrency(); size_t original_quantity=this->source_files.size(); fs::path build_path=".vybuild_build"; fs::create_directory(build_path); auto compiler_args=split_args_shell_vyld(this->compiler_flags+" -fno-pic -m64 -mcmodel=large"); auto compiler_args_entry=split_args_shell_vyld("-ffreestanding -nostdlib -nostartfiles -Wall -Wextra -fno-stack-protector -Wno-missing-prototypes -fno-builtin -fno-pic -m64 -mcmodel=large"); vector outputs; for (int i=0;isource_files[i]); string filename=fs::path(source_file).filename().string(); string output_file=build_path/(filename.substr(0,filename.size()-2)+".o"); string dfile_path=build_path/(filename.substr(0,filename.size()-2)+".d"); string headers_cmd=this->headers_command; outputs.push_back(output_file); if (source_file!=string(exe_path/"_vyx_start.c")) { vyld_compilation_tasks.push([source_file=std::move(source_file),compiler_path,compiler_hash,compiler_args,dfile_path=std::move(dfile_path),headers_cmd,output_file=std::move(output_file)]() mutable -> vybuild::actions::ActionStatus { return parallel_vyld_compilation(source_file,compiler_path,compiler_hash,compiler_args,dfile_path,headers_cmd,output_file); }); } else { vyld_compilation_tasks.push([source_file=std::move(source_file),compiler_path,compiler_hash,compiler_args_entry,dfile_path=std::move(dfile_path),headers_cmd,output_file=std::move(output_file)]() mutable -> vybuild::actions::ActionStatus { return parallel_vyld_compilation(source_file,compiler_path,compiler_hash,compiler_args_entry,dfile_path,headers_cmd,output_file); }); } } vector threads; vector results; mutex results_mutex; size_t real_threads=0; if (original_quantity>nb_thread) { real_threads=nb_thread; } else { real_threads=original_quantity; } results.resize(real_threads,vybuild::actions::ActionStatus::Success); for (size_t i=0;i lock(results_mutex); results[i]=status; }); } for (auto &t:threads) { t.join(); } for (auto status:results) { if (status!=vybuild::actions::ActionStatus::Success) { return status; } } string assembler_path=find_executable_vyld(this->assembler); for (auto &f:this->source_asm) { string source_file=std::move(f); string filename=fs::path(source_file).filename().string(); string output_file=build_path/(filename.substr(0,filename.size()-4)+".o"); outputs.push_back(output_file); int pipefd[2]; if (pipe(pipefd)==-1) { print_deepness_vyld("VyldCompilationAction: An error happened while creating pipe."); return vybuild::actions::ActionStatus::SystemError; } pid_t pid=fork(); if (pid<0) { print_deepness_vyld("VyldCompilationAction: An error happened while forking."); return vybuild::actions::ActionStatus::SystemError; } if (pid==0) { close(pipefd[0]); dup2(pipefd[1],STDOUT_FILENO); dup2(pipefd[1],STDERR_FILENO); close(pipefd[1]); vector all_args; all_args.push_back(assembler_path); all_args.push_back("-f"); all_args.push_back("elf64"); all_args.push_back(source_file); all_args.push_back("-o"); all_args.push_back(output_file); std::vector argv; for (auto &s:all_args) argv.push_back(s.c_str()); argv.push_back(nullptr); execvp(assembler_path.c_str(),const_cast(argv.data())); print_deepness_vyld("VyldCompilationAction: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer< all_args; all_args.push_back(compiler_path); all_args.push_back("-r"); all_args.insert(all_args.end(),outputs.begin(),outputs.end()); all_args.push_back("-o"); all_args.push_back((build_path/"combined.o").string()); std::vector argv; for (auto &s:all_args) argv.push_back(s.c_str()); argv.push_back(nullptr); execvp(compiler_path.c_str(),const_cast(argv.data())); print_deepness_vyld("VyldCompilationAction: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer<get_size()==0) { print_deepness_vyld("VyldCompilationAction: .text is empty."); return vybuild::actions::ActionStatus::Fail; } size_t text_size=text_sec->get_size(); size_t text_size_padded=pad_size(text_size,4096); size_t data_size=0; size_t data_size_padded=0; size_t rodata_size=0; size_t rodata_size_padded=0; size_t bss_size=0; size_t bss_size_padded=0; if (data_sec) { data_size=data_sec->get_size(); data_size_padded=pad_size(data_size,4096); } if (rodata_sec) { rodata_size=rodata_sec->get_size(); for (const auto &s:reader.sections) { if (s->get_name().size()>7 && s->get_name().substr(0,7)==".rodata" && s->get_name()!=".rodata") { rodata_size+=s->get_size(); } } rodata_size_padded=pad_size(rodata_size,4096); } if (bss_sec) { bss_size=bss_sec->get_size(); bss_size_padded=pad_size(bss_size,4096); } vyx_addresses va=generate_vyx_addresses(); size_t text_offset=va.text_offset; size_t data_offset=text_offset+text_size_padded; size_t rodata_offset=data_offset+data_size_padded; size_t bss_offset=rodata_offset+rodata_size_padded; ofstream ld(build_path/"script.ld"); ld<<"ENTRY(_vyx_start)\n"; ld<<"MEMORY\n{\n"; ld<<" TEXT (rx) : ORIGIN = 0x"< TEXT\n\n"; ld<<" .data :\n"; ld<<" {\n"; ld<<" . = ORIGIN(DATA);\n"; ld<<" *(.data)\n"; ld<<" *(.data.*)\n"; ld<<" . = ALIGN(0x1000);\n"; ld<<" } > DATA\n\n"; ld<<" .rodata :\n"; ld<<" {\n"; ld<<" . = ORIGIN(RODATA);\n"; ld<<" *(.rodata)\n"; ld<<" *(.rodata.*)\n"; ld<<" . = ALIGN(0x1000);\n"; ld<<" } > RODATA\n\n"; ld<<" .bss :\n"; ld<<" {\n"; ld<<" . = ORIGIN(BSS);\n"; ld<<" *(.bss)\n"; ld<<" *(.bss.*)\n"; ld<<" . = ALIGN(0x1000);\n"; ld<<" } > BSS\n\n"; ld<<" /DISCARD/ :\n"; ld<<" {\n"; ld<<" *(.note.*)\n"; ld<<" *(.comment)\n"; ld<<" *(.eh_frame)\n"; ld<<" *(.eh_frame_hdr)\n"; ld<<" }\n}"; ld.close(); pipefd[0]=0; pipefd[1]=0; if (pipe(pipefd)==-1) { print_deepness_vyld("VyldCompilationAction: An error happened while creating pipe."); return vybuild::actions::ActionStatus::SystemError; } pid=fork(); if (pid<0) { print_deepness_vyld("VyldCompilationAction: An error happened while forking."); return vybuild::actions::ActionStatus::SystemError; } if (pid==0) { close(pipefd[0]); dup2(pipefd[1],STDOUT_FILENO); dup2(pipefd[1],STDERR_FILENO); close(pipefd[1]); vector all_args; all_args.push_back(this->linker); all_args.push_back("-T"); all_args.push_back((build_path/"script.ld").string()); all_args.push_back("-o"); all_args.push_back((build_path/"compiled.elf").string()); all_args.push_back((build_path/"combined.o").string()); std::vector argv; for (auto &s:all_args) argv.push_back(s.c_str()); argv.push_back(nullptr); execvp(this->linker.c_str(),const_cast(argv.data())); print_deepness_vyld("VyldCompilationAction: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer<get_size()==0) { print_deepness_vyld("VyldCompilationAction: .text is empty."); return vybuild::actions::ActionStatus::Fail; } vector elf_text_bin(text_size_padded,0); vector elf_data_bin(data_size_padded,0); vector elf_rodata_bin(rodata_size_padded,0); memcpy(elf_text_bin.data(),elf_text_sec->get_data(),elf_text_sec->get_size()); if (elf_data_sec->get_size()!=0) { memcpy(elf_data_bin.data(),elf_data_sec->get_data(),elf_data_sec->get_size()); } if (elf_rodata_sec->get_size()!=0) { memcpy(elf_rodata_bin.data(),elf_rodata_sec->get_data(),elf_rodata_sec->get_size()); } uint8_t vyx_sig[3]={'V','y','X'}; uint16_t vyx_ver=VY_COMMON_VERSIONS_VYX; string outfile=this->output_file; size_t payload_count=this->payloads.size(); auto payloads=generate_payloads(this->payloads); ofstream out(outfile,ios::binary); out.write(reinterpret_cast(vyx_sig),3); out.write(reinterpret_cast(&vyx_ver),2); out.write(reinterpret_cast(&va.text_offset),8); out.write(reinterpret_cast(&va.stack_base),8); out.write(reinterpret_cast(&text_size_padded),8); out.write(reinterpret_cast(&data_size_padded),8); out.write(reinterpret_cast(&rodata_size_padded),8); uint64_t bss_size_elf=elf_bss_sec->get_size(); out.write(reinterpret_cast(&bss_size_elf),8); out.write(reinterpret_cast(&payload_count),8); out.write(reinterpret_cast(elf_text_bin.data()),elf_text_bin.size()); if (data_size_padded!=0) out.write(reinterpret_cast(elf_data_bin.data()),elf_data_bin.size()); if (rodata_size_padded!=0) out.write(reinterpret_cast(elf_rodata_bin.data()),elf_rodata_bin.size()); if (payload_count!=0) { out.write(reinterpret_cast(payloads.data()),payloads.size()); } out.close(); fs::remove_all(build_path); return vybuild::actions::ActionStatus::Success; }