// SPDX-License-Identifier: MPL-2.0 #include "vybuild.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace std; int deepness=0; map assembler_path_hash; queue> tasks; mutex tasks_mutex; mutex print_mutex; void print_deepness(string text) { cout<<"[VyBuild] "; for (int i=0;i split_args_shell(const string& input) { wordexp_t result; vector args; if (wordexp(input.c_str(),&result,0)==0) { for (size_t i=0;i argv; auto args=split_args_shell(command); args[0]=find_executable(args[0]).c_str(); for (auto &a:args) argv.push_back(a.c_str()); argv.push_back(nullptr); execvp(argv[0],const_cast(argv.data())); cout<0) { buffer< headers_from_dfile(string dfile_path) { if (!fs::exists(dfile_path)) throw std::runtime_error(dfile_path+" doesn't exist."); ifstream dfile(dfile_path); vector deps; string line,all; while (getline(dfile,line)) { if (!line.empty() && line.back()=='\\') { line.pop_back(); all+=line+" "; } else { all+=line+ " "; } } auto pos=all.find(':'); if (pos==string::npos) return deps; string rest=all.substr(pos+1); istringstream iss(rest); string token; while (iss>>token) { deps.push_back(token); } return deps; } int get_deepness() { return deepness; } void vybuild::actions::check_arg_string(const json& args,std::string field_name) { if (!args.contains(field_name+"_variables")) throw std::runtime_error(field_name+"_variables isn't provided."); if (!args[field_name+"_variables"].is_boolean()) throw std::runtime_error(field_name+"_variables isn't a boolean"); if (!args.contains(field_name)) throw std::runtime_error(field_name+" isn't provided."); if (!args[field_name].is_string()) throw std::runtime_error(field_name+" isn't a string."); return; } void vybuild::actions::check_arg_vector_string(const json& args,std::string field_name) { if (!args.contains(field_name+"_variables")) throw std::runtime_error(field_name+"_variables isn't provided."); if (!args[field_name+"_variables"].is_boolean()) throw std::runtime_error(field_name+"_variables isn't a boolean"); if (!args.contains(field_name)) throw std::runtime_error(field_name+" isn't provided."); if (!args[field_name].is_array()) throw std::runtime_error(field_name+" isn't an array."); for (auto& a:args[field_name]) { if (!a.is_string()) throw std::runtime_error("One of the element of "+field_name+" isn't a string."); } return; } vybuild::actions::ParallelAction vybuild::actions::return_pararrel_action(const json& args,string action_name) { if (action_name=="run_command_wait") { return vybuild::actions::ParallelAction(vybuild::actions::RunCommandWaitAction(args)); } else if (action_name=="save_command_content") { return vybuild::actions::ParallelAction(vybuild::actions::SaveCommandContentAction(args)); } else if (action_name=="compile_one_asm") { return vybuild::actions::ParallelAction(vybuild::actions::CompileOneAsmAction(args)); } else if (action_name=="ensure_folder_existence") { return vybuild::actions::ParallelAction(vybuild::actions::EnsureFolderExistenceAction(args)); } else if (action_name=="move_file") { return vybuild::actions::ParallelAction(vybuild::actions::MoveFileAction(args)); } else if (action_name=="compile_one_cpp") { return vybuild::actions::ParallelAction(vybuild::actions::CompileOneCppAction(args)); } else if (action_name=="run_command_wait_dir") { return vybuild::actions::ParallelAction(vybuild::actions::RunCommandWaitDirAction(args)); } else if (action_name=="copy_folder") { return vybuild::actions::ParallelAction(vybuild::actions::CopyFolderAction(args)); } else if (action_name=="extract_from_cache") { return vybuild::actions::ParallelAction(vybuild::actions::ExtractFromCacheAction(args)); } else if (action_name=="store_in_cache") { return vybuild::actions::ParallelAction(vybuild::actions::StoreInCacheAction(args)); } else if (action_name=="ensure_folder_reset") { return vybuild::actions::ParallelAction(vybuild::actions::EnsureFolderResetAction(args)); } else if (action_name=="run_actions_if") { char path[PATH_MAX]; getcwd(path,PATH_MAX); return vybuild::actions::ParallelAction(vybuild::actions::RunActionsIfAction(args,false,string(path))); } else if (action_name=="run_command_str_wait") { return vybuild::actions::ParallelAction(vybuild::actions::RunCommandStrAction(args)); } else if (action_name=="run_actions_if_else") { char path[PATH_MAX]; getcwd(path,PATH_MAX); return vybuild::actions::ParallelAction(vybuild::actions::RunActionsIfElseAction(args,false,string(path))); } else if (action_name=="compute_element_hash") { return vybuild::actions::ParallelAction(vybuild::actions::ComputeElementHashAction(args)); } else if (action_name=="copy_file") { return vybuild::actions::ParallelAction(vybuild::actions::CopyFileAction(args)); } else if (action_name=="create_fat32") { return vybuild::actions::ParallelAction(vybuild::actions::CreateFat32Action(args)); } else if (action_name=="create_folder_fat32") { return vybuild::actions::ParallelAction(vybuild::actions::CreateFolderFat32Action(args)); } else if (action_name=="create_file_fat32") { return vybuild::actions::ParallelAction(vybuild::actions::CreateFileFat32Action(args)); } else if (action_name=="export_fat32") { return vybuild::actions::ParallelAction(vybuild::actions::ExportFat32Action(args)); } else if (action_name=="create_disk") { return vybuild::actions::ParallelAction(vybuild::actions::CreateDiskAction(args)); } else if (action_name=="add_partition_disk") { return vybuild::actions::ParallelAction(vybuild::actions::AddPartitionDiskAction(args)); } else if (action_name=="export_disk") { return vybuild::actions::ParallelAction(vybuild::actions::ExportDiskAction(args)); } else if (action_name=="vftm_load_crypto_lib") { return vybuild::actions::ParallelAction(vybuild::actions::VFTMLoadCryptoLibAction(args)); } else if (action_name=="vftm_run_keys_generator") { return vybuild::actions::ParallelAction(vybuild::actions::VFTMRunKeysGeneratorAction(args)); } else { throw std::runtime_error("Action named \""+action_name+"\" isn't supported for parallelism execution."); } } vybuild::actions::Action vybuild::actions::return_action(const json &args,string action_name,bool is_threading_allowed,string root_folder) { if (find(threaded_actions.begin(),threaded_actions.end(),action_name)!=threaded_actions.end() && !is_threading_allowed) throw std::runtime_error("Multithreaded actions aren't allowed in this submodule, as it's executed from RunSubmoduleParellelAction."); if (action_name=="run_submodule") { return vybuild::actions::Action(vybuild::actions::RunSubmoduleAction(args,root_folder)); } else if (action_name=="run_command_wait") { return vybuild::actions::Action(vybuild::actions::RunCommandWaitAction(args)); } else if (action_name=="run_actions_parallel") { return vybuild::actions::Action(vybuild::actions::RunActionsParallelAction(args)); } else if (action_name=="save_command_content") { return vybuild::actions::Action(vybuild::actions::SaveCommandContentAction(args)); } else if (action_name=="compile_one_asm") { return vybuild::actions::Action(vybuild::actions::CompileOneAsmAction(args)); } else if (action_name=="ensure_folder_existence") { return vybuild::actions::Action(vybuild::actions::EnsureFolderExistenceAction(args)); } else if (action_name=="move_file") { return vybuild::actions::Action(vybuild::actions::MoveFileAction(args)); } else if (action_name=="compile_one_cpp") { return vybuild::actions::Action(vybuild::actions::CompileOneCppAction(args)); } else if (action_name=="vyld_compilation") { return vybuild::actions::Action(vybuild::actions::VyldCompilationAction(args)); } else if (action_name=="run_submodule_if") { return vybuild::actions::Action(vybuild::actions::RunSubmoduleIfAction(args,root_folder)); } else if (action_name=="run_command_wait_dir") { return vybuild::actions::Action(vybuild::actions::RunCommandWaitDirAction(args)); } else if (action_name=="copy_folder") { return vybuild::actions::Action(vybuild::actions::CopyFolderAction(args)); } else if (action_name=="run_submodule_parallel") { return vybuild::actions::Action(vybuild::actions::RunSubmoduleParallelAction(args)); } else if (action_name=="extract_from_cache") { return vybuild::actions::Action(vybuild::actions::ExtractFromCacheAction(args)); } else if (action_name=="store_in_cache") { return vybuild::actions::Action(vybuild::actions::StoreInCacheAction(args)); } else if (action_name=="ensure_folder_reset") { return vybuild::actions::Action(vybuild::actions::EnsureFolderResetAction(args)); } else if (action_name=="run_actions_if") { return vybuild::actions::Action(vybuild::actions::RunActionsIfAction(args,is_threading_allowed,root_folder)); } else if (action_name=="run_command_str_wait") { return vybuild::actions::Action(vybuild::actions::RunCommandStrAction(args)); } else if (action_name=="run_actions_if_else") { return vybuild::actions::Action(vybuild::actions::RunActionsIfElseAction(args,is_threading_allowed,root_folder)); } else if (action_name=="compute_element_hash") { return vybuild::actions::Action(vybuild::actions::ComputeElementHashAction(args)); } else if (action_name=="compile_multiple_cpp") { return vybuild::actions::Action(vybuild::actions::CompileMultipleCppAction(args)); } else if (action_name=="copy_file") { return vybuild::actions::Action(vybuild::actions::CopyFileAction(args)); } else if (action_name=="create_fat32") { return vybuild::actions::Action(vybuild::actions::CreateFat32Action(args)); } else if (action_name=="create_folder_fat32") { return vybuild::actions::Action(vybuild::actions::CreateFolderFat32Action(args)); } else if (action_name=="create_file_fat32") { return vybuild::actions::Action(vybuild::actions::CreateFileFat32Action(args)); } else if (action_name=="export_fat32") { return vybuild::actions::Action(vybuild::actions::ExportFat32Action(args)); } else if (action_name=="create_disk") { return vybuild::actions::Action(vybuild::actions::CreateDiskAction(args)); } else if (action_name=="add_partition_disk") { return vybuild::actions::Action(vybuild::actions::AddPartitionDiskAction(args)); } else if (action_name=="export_disk") { return vybuild::actions::Action(vybuild::actions::ExportDiskAction(args)); } else if (action_name=="vftm_load_crypto_lib") { return vybuild::actions::Action(vybuild::actions::VFTMLoadCryptoLibAction(args)); } else if (action_name=="vftm_run_key_generator") { return vybuild::actions::Action(vybuild::actions::VFTMRunKeysGeneratorAction(args)); } else { throw std::runtime_error("Action named \""+action_name+"\" isn't a valid action."); } } vybuild::actions::ActionStatus vybuild::actions::RunSubmoduleAction::run_action() { if (this->file_variables) { this->file=vybuild::parser::string_vars(file); } if (this->wait==true) { deepness++; vybuild::module::Module submodule(this->file,true); print_deepness("Running submodule on main process: "+submodule.name); auto status=submodule.run_module(false,false); if (status==vybuild::actions::ActionStatus::Success) { deepness--; return status; } else { if (status==vybuild::actions::ActionStatus::Fail) { print_deepness("RunSubmoduleAction: Submodule failed to execute."); } else if (status==vybuild::actions::ActionStatus::SystemError) { print_deepness("RunSubmoduleAction: A system error happened in submodule."); } else if (status==vybuild::actions::ActionStatus::ExitFail) { print_deepness("RunSubmoduleAction: The submodule exited by itself, raising an error."); } if (this->block_if_fail) { deepness--; return status; } else { deepness--; return vybuild::actions::ActionStatus::Success; } } } else { deepness++; vybuild::module::Module submodule(this->file,true); print_deepness("Running submodule on secondary process: "+submodule.name); pid_t pid=fork(); if (pid<0) { print_deepness("RunSubmoduleAction: An error happened while forking."); if (this->block_if_fail) { deepness--; return vybuild::actions::ActionStatus::SystemError; } else { deepness--; return vybuild::actions::ActionStatus::Success; } } if (pid==0) { auto status=submodule.run_module(false,false); exit(0); } else { return vybuild::actions::ActionStatus::Success; } } } vybuild::actions::ActionStatus vybuild::actions::RunCommandWaitAction::run_action() { if (command_variables) { this->command=vybuild::parser::vector_string_vars(this->command); } char path[PATH_MAX]; getcwd(path,PATH_MAX); if (this->show_output=="silent") { int pipefd[2]; if (pipe(pipefd)==-1) { print_deepness("RunCommandWaitAction: An error happened while creating pipe."); return vybuild::actions::ActionStatus::SystemError; } pid_t pid=fork(); if (pid<0) { print_deepness("RunCommandWaitAction: An error happened while forking."); return vybuild::actions::ActionStatus::SystemError; } if (pid==0) { close(pipefd[0]); dup2(pipefd[1],STDOUT_FILENO); dup2(pipefd[1],STDERR_FILENO); close(pipefd[1]); std::vector argv; for (int i=0;icommand.size();i++) { argv.push_back(command[i].c_str()); } argv.push_back(nullptr); execvp(command[0].c_str(),const_cast(argv.data())); print_deepness("RunCommandWaitAction: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer<ignore_success_status) { if (find(this->success_status.begin(),this->success_status.end(),WEXITSTATUS(status))==this->success_status.end()) { print_deepness("RunCommandWaitAction: Command exited with error (status "+to_string(WEXITSTATUS(status))+"). Stopping."); return vybuild::actions::ActionStatus::Fail; } else { return vybuild::actions::ActionStatus::Success; } } else { return vybuild::actions::ActionStatus::Success; } } } else if (this->show_output=="on_failure") { int pipefd[2]; if (pipe(pipefd)==-1) { print_deepness("RunCommandWaitAction: An error happened while creating pipe."); return vybuild::actions::ActionStatus::SystemError; } pid_t pid=fork(); if (pid<0) { print_deepness("RunCommandWaitAction: An error happened while forking."); return vybuild::actions::ActionStatus::SystemError; } if (pid==0) { close(pipefd[0]); dup2(pipefd[1],STDOUT_FILENO); dup2(pipefd[1],STDERR_FILENO); close(pipefd[1]); std::vector argv; for (int i=0;icommand.size();i++) { argv.push_back(command[i].c_str()); } argv.push_back(nullptr); execvp(command[0].c_str(),const_cast(argv.data())); print_deepness("RunCommandWaitAction: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer<ignore_success_status) { if (find(this->success_status.begin(),this->success_status.end(),WEXITSTATUS(status))==this->success_status.end()) { print_deepness("RunCommandWaitAction: Command exited with error (status "+to_string(WEXITSTATUS(status))+"), showing output:\n"); cout< argv; for (int i=0;icommand.size();i++) { argv.push_back(command[i].c_str()); } argv.push_back(nullptr); execvp(command[0].c_str(),const_cast(argv.data())); print_deepness("RunCommandWaitAction: An error happened while executing command."); _exit(1); } else if (this->show_output=="live") { int status; waitpid(pid,&status,0); if (!WIFEXITED(status)) { print_deepness("RunCommandWaitAction: An error happened in forked process."); return vybuild::actions::ActionStatus::SystemError; } cout<ignore_success_status) { if (find(this->success_status.begin(),this->success_status.end(),WEXITSTATUS(status))==this->success_status.end()) { print_deepness("RunCommandWaitAction: Command exited with error (status "+to_string(WEXITSTATUS(status))+"). Stopping."); return vybuild::actions::ActionStatus::Fail; } else { return vybuild::actions::ActionStatus::Success; } } else { return vybuild::actions::ActionStatus::Success; } } } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus parallel_action(vybuild::actions::ParallelAction &act) { vybuild::actions::ActionStatus status=std::visit([](auto &act) { return act.run_action(); },act); return status; } vybuild::actions::ActionStatus parallel_action_worker(size_t thread_num) { while (true) { function task; { std::lock_guard lock(tasks_mutex); if (tasks.empty()) break; task=std::move(tasks.front()); tasks.pop(); } auto status=task(); if (status!=vybuild::actions::ActionStatus::Success) { return status; } } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::RunActionsParallelAction::run_action() { size_t nb_thread=thread::hardware_concurrency(); size_t original_quantity=this->actions.size(); for (int i=0;iactions.size();i++) { ParallelAction act=std::move(this->actions[i]); tasks.push([act=std::move(act)]() mutable -> vybuild::actions::ActionStatus { return parallel_action(act); }); } vector threads; vector results; mutex results_mutex; size_t real_threads=0; if (original_quantity>nb_thread) { real_threads=nb_thread; } else { real_threads=original_quantity; } results.resize(real_threads,vybuild::actions::ActionStatus::Success); for (size_t i=0;i lock(results_mutex); results[i]=status; }); } for (auto &t:threads) { t.join(); } for (auto status:results) { if (status!=vybuild::actions::ActionStatus::Success) { return status; } } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::SaveCommandContentAction::run_action() { if (command_variables) { this->command=vybuild::parser::vector_string_vars(this->command); } if (command_output_file_variables) { this->command_output_file=vybuild::parser::string_vars(this->command_output_file); } int pipefd[2]; if (pipe(pipefd)==-1) { print_deepness("SaveCommandContentAction: An error happened while creating pipe."); return vybuild::actions::ActionStatus::SystemError; } pid_t pid=fork(); if (pid<0) { print_deepness("SaveCommandContentAction: An error happened while forking."); return vybuild::actions::ActionStatus::SystemError; } if (pid==0) { close(pipefd[0]); dup2(pipefd[1],STDOUT_FILENO); dup2(pipefd[1],STDERR_FILENO); close(pipefd[1]); std::vector argv; for (int i=0;icommand.size();i++) { argv.push_back(command[i].c_str()); } argv.push_back(nullptr); execvp(command[0].c_str(),const_cast(argv.data())); print_deepness("SaveCommandContentAction: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer<ignore_success_status) { if (find(this->success_status.begin(),this->success_status.end(),WEXITSTATUS(status))==this->success_status.end()) { print_deepness("SaveCommandContentAction: Command exited with error (status "+to_string(WEXITSTATUS(status))+"). Stopping."); return vybuild::actions::ActionStatus::Fail; } else { ofstream out(this->command_output_file,std::ios::trunc); out<command_output_file,std::ios::trunc); out<source_variables) this->source=vybuild::parser::string_vars(this->source); if (this->assembler_variables) this->assembler=vybuild::parser::string_vars(this->assembler); if (this->pre_arguments_variables) this->pre_arguments=vybuild::parser::vector_string_vars(this->pre_arguments); if (this->post_arguments_variables) this->post_arguments=vybuild::parser::vector_string_vars(this->post_arguments); if (this->output_file_variables) this->output_file=vybuild::parser::string_vars(this->output_file); if (this->cache_authorized) { ifstream asm_file(this->source,ios::binary); if (!asm_file) { print_deepness("CompileOneAsmAction: Couldn't open source file"); return vybuild::actions::ActionStatus::SystemError; } vector asm_data(fs::file_size(this->source)); asm_file.read(asm_data.data(),asm_data.size()); if (!asm_file) { print_deepness("CompileOneAsmAction: Couldn't read source file"); return vybuild::actions::ActionStatus::SystemError; } string asm_content(asm_data.begin(),asm_data.end()); asm_file.close(); string assembler_path=find_executable(this->assembler); string assembler_hash=""; if (assembler_path_hash.find(assembler_path)==assembler_path_hash.end()) { ifstream assembler_bin(assembler_path,ios::binary); if (!assembler_bin) { print_deepness("CompileOneAsmAction: Cannot open assembler binary: "+assembler_path); return vybuild::actions::ActionStatus::SystemError; } size_t assembler_size=fs::file_size(assembler_path); vector assembler_data(assembler_size); assembler_bin.read(reinterpret_cast(assembler_data.data()),assembler_size); if (!assembler_bin) { print_deepness("CompileOneAsmAction: Cannot read assembler binary: "+assembler_path); return vybuild::actions::ActionStatus::SystemError; } assembler_hash=vybuild::cache::xxhash_to_string(XXH3_128bits(assembler_data.data(),assembler_data.size())); assembler_path_hash[assembler_path]=assembler_hash; } else { assembler_hash=assembler_path_hash.at(assembler_path); } asm_content+="\n"+assembler_path; asm_content+="\n"+assembler_hash; for (auto arg:this->pre_arguments) asm_content+="\n"+arg; for (auto arg:this->post_arguments) asm_content+="\n"+arg; string key=vybuild::cache::xxhash_to_string(XXH3_128bits(asm_content.data(),asm_content.size())); if (vybuild::cache::cache_contains(key)) { vybuild::cache::extract_file(key,this->output_file); return vybuild::actions::ActionStatus::Success; } int pipefd[2]; if (pipe(pipefd)==-1) { print_deepness("CompileOneAsmAction: An error happened while creating pipe."); return vybuild::actions::ActionStatus::SystemError; } pid_t pid=fork(); if (pid<0) { print_deepness("CompileOneAsmAction: An error happened while forking."); return vybuild::actions::ActionStatus::SystemError; } if (pid==0) { close(pipefd[0]); dup2(pipefd[1],STDOUT_FILENO); dup2(pipefd[1],STDERR_FILENO); close(pipefd[1]); std::vector argv={this->assembler.c_str()}; for (int i=0;ipre_arguments.size();i++) { argv.push_back(pre_arguments[i].c_str()); } argv.push_back(this->source.c_str()); for (int i=0;ipost_arguments.size();i++) { argv.push_back(post_arguments[i].c_str()); } argv.push_back(nullptr); execvp(this->assembler.c_str(),const_cast(argv.data())); print_deepness("CompileOneAsmAction: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer<ignore_success_status) { if (find(this->success_status.begin(),this->success_status.end(),WEXITSTATUS(status))==this->success_status.end()) { print_deepness("CompileOneAsmAction: Command exited with error (status "+to_string(WEXITSTATUS(status))+"), showing output:\n"); cout<output_file); return vybuild::actions::ActionStatus::Success; } } else { vybuild::cache::store_file(key,this->output_file); return vybuild::actions::ActionStatus::Success; } } } else { int pipefd[2]; if (pipe(pipefd)==-1) { print_deepness("CompileOneAsmAction: An error happened while creating pipe."); return vybuild::actions::ActionStatus::SystemError; } pid_t pid=fork(); if (pid<0) { print_deepness("CompileOneAsmAction: An error happened while forking."); return vybuild::actions::ActionStatus::SystemError; } if (pid==0) { close(pipefd[0]); dup2(pipefd[1],STDOUT_FILENO); dup2(pipefd[1],STDERR_FILENO); close(pipefd[1]); std::vector argv={this->assembler.c_str()}; for (int i=0;ipre_arguments.size();i++) { argv.push_back(pre_arguments[i].c_str()); } argv.push_back(this->source.c_str()); for (int i=0;ipost_arguments.size();i++) { argv.push_back(post_arguments[i].c_str()); } argv.push_back(nullptr); execvp(this->assembler.c_str(),const_cast(argv.data())); print_deepness("CompileOneAsmAction: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer<ignore_success_status) { if (find(this->success_status.begin(),this->success_status.end(),WEXITSTATUS(status))==this->success_status.end()) { print_deepness("CompileOneAsmAction: Command exited with error (status "+to_string(WEXITSTATUS(status))+"), showing output:\n"); cout<path_variables) this->path=vybuild::parser::string_vars(this->path); if (!fs::exists(this->path)) fs::create_directories(this->path); return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::MoveFileAction::run_action() { if (this->source_file_variables) this->source_file=vybuild::parser::string_vars(this->source_file); if (this->destination_folder_variables) this->destination_folder=vybuild::parser::string_vars(this->destination_folder); if (!fs::exists(this->source_file)) { print_deepness("MoveFileAction: Source file doesn't exist: "+this->source_file); return vybuild::actions::ActionStatus::Fail; } if (!fs::exists(this->destination_folder)) { print_deepness("MoveFileAction: Destination doesn't exist: "+this->destination_folder); return vybuild::actions::ActionStatus::Fail; } if (!fs::is_directory(this->destination_folder)) { print_deepness("MoveFileAction: Destination isn't a folder: "+this->destination_folder); return vybuild::actions::ActionStatus::Fail; } fs::rename(fs::absolute(this->source_file),fs::path(this->destination_folder)/(fs::path(this->source_file).filename())); return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::CompileOneCppAction::run_action() { if (this->source_variables) this->source=vybuild::parser::string_vars(this->source); if (this->compiler_variables) this->compiler=vybuild::parser::string_vars(this->compiler); if (this->pre_arguments_variables) this->pre_arguments=vybuild::parser::vector_string_vars(this->pre_arguments); if (this->post_arguments_variables) this->post_arguments=vybuild::parser::vector_string_vars(this->post_arguments); if (this->output_file_variables) this->output_file=vybuild::parser::string_vars(this->output_file); if (this->headers_command_variables) this->headers_command=vybuild::parser::string_vars(this->headers_command); if (this->cache_authorized) { string cpp_content=""; string compiler_path=find_executable(this->compiler); string compiler_hash=""; auto vars_list=vybuild::vars::get_global_variables_list(); ifstream compiler_bin(compiler_path,ios::binary); if (!compiler_bin) { print_deepness("CompileOneCppAction: Cannot open compiler binary: "+compiler_path); return vybuild::actions::ActionStatus::SystemError; } size_t compiler_size=fs::file_size(compiler_path); vector compiler_data(compiler_size); compiler_bin.read(reinterpret_cast(compiler_data.data()),compiler_size); if (!compiler_bin) { print_deepness("CompileOneCppAction: Cannot read compiler binary: "+compiler_path); return vybuild::actions::ActionStatus::SystemError; } compiler_hash=vybuild::cache::xxhash_to_string(XXH3_128bits(compiler_data.data(),compiler_data.size())); for (auto &v:vars_list) { if (v.starts_with("\\$__VYBUILD_COMPILER_HASH_")) { compiler_hash+="\n"+get_command_output(vybuild::vars::get_global_variable(v)); } } cpp_content+="\n"+compiler_path; cpp_content+="\n"+compiler_hash; for (auto arg:this->pre_arguments) cpp_content+="\n"+arg; for (auto arg:this->post_arguments) cpp_content+="\n"+arg; if (headers_command!="") { string dep_command=headers_command; regex regvar1("%dfile%"); dep_command=regex_replace(dep_command,regvar1,".vybuild_dep.d"); regex regvar2("%sourcefile%"); dep_command=regex_replace(dep_command,regvar2,this->source); int res=std::system(dep_command.c_str()); if (res!=0) { print_deepness("CompileOneCppAction: Cannot get headers list for file \""+this->source+"\" with command: "+dep_command); return vybuild::actions::ActionStatus::Fail; } auto headers=headers_from_dfile(".vybuild_dep.d"); for (auto &h:headers) { ifstream header(h); vector header_data(fs::file_size(fs::absolute(h))); if (!header) { print_deepness("CompileOneCppAction: Couldn't open source/header file: "+h); return vybuild::actions::ActionStatus::SystemError; } header.read(header_data.data(),header_data.size()); if (!header) { print_deepness("CompileOneCppAction: Couldn't read source/header file: "+h); return vybuild::actions::ActionStatus::SystemError; } cpp_content+=string(header_data.begin(),header_data.end()); header.close(); } fs::remove(".vybuild_dep.d"); } string key=vybuild::cache::xxhash_to_string(XXH3_128bits(cpp_content.data(),cpp_content.size())); if (vybuild::cache::cache_contains(key)) { vybuild::cache::extract_file(key,this->output_file); return vybuild::actions::ActionStatus::Success; } int pipefd[2]; if (pipe(pipefd)==-1) { print_deepness("CompileOneCppAction: An error happened while creating pipe."); return vybuild::actions::ActionStatus::SystemError; } pid_t pid=fork(); if (pid<0) { print_deepness("CompileOneCppAction: An error happened while forking."); return vybuild::actions::ActionStatus::SystemError; } if (pid==0) { close(pipefd[0]); dup2(pipefd[1],STDOUT_FILENO); dup2(pipefd[1],STDERR_FILENO); close(pipefd[1]); std::vector argv={this->compiler.c_str()}; for (int i=0;ipre_arguments.size();i++) { argv.push_back(pre_arguments[i].c_str()); } argv.push_back(this->source.c_str()); for (int i=0;ipost_arguments.size();i++) { argv.push_back(post_arguments[i].c_str()); } argv.push_back(nullptr); execvp(this->compiler.c_str(),const_cast(argv.data())); print_deepness("CompileOneCppAction: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer<ignore_success_status) { if (find(this->success_status.begin(),this->success_status.end(),WEXITSTATUS(status))==this->success_status.end()) { print_deepness("CompileOneCppAction: Command exited with error (status "+to_string(WEXITSTATUS(status))+"), showing output:\n"); cout<output_file); return vybuild::actions::ActionStatus::Success; } } else { vybuild::cache::store_file(key,this->output_file); return vybuild::actions::ActionStatus::Success; } } } else { int pipefd[2]; if (pipe(pipefd)==-1) { print_deepness("CompileOneCppAction: An error happened while creating pipe."); return vybuild::actions::ActionStatus::SystemError; } pid_t pid=fork(); if (pid<0) { print_deepness("CompileOneCppAction: An error happened while forking."); return vybuild::actions::ActionStatus::SystemError; } if (pid==0) { close(pipefd[0]); dup2(pipefd[1],STDOUT_FILENO); dup2(pipefd[1],STDERR_FILENO); close(pipefd[1]); std::vector argv={this->compiler.c_str()}; for (int i=0;ipre_arguments.size();i++) { argv.push_back(pre_arguments[i].c_str()); } argv.push_back(this->source.c_str()); for (int i=0;ipost_arguments.size();i++) { argv.push_back(post_arguments[i].c_str()); } argv.push_back(nullptr); execvp(this->compiler.c_str(),const_cast(argv.data())); print_deepness("CompileOneCppAction: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer<ignore_success_status) { if (find(this->success_status.begin(),this->success_status.end(),WEXITSTATUS(status))==this->success_status.end()) { print_deepness("CompileOneCppAction: Command exited with error (status "+to_string(WEXITSTATUS(status))+"), showing output:\n"); cout<file_variables) { this->file=vybuild::parser::string_vars(file); } if (vybuild::condition::evaluate_condition(this->condition)==false) return vybuild::actions::ActionStatus::Success; if (this->wait==true) { deepness++; vybuild::module::Module submodule(this->file,true); print_deepness("Running submodule on main process: "+submodule.name); auto status=submodule.run_module(false,false); if (status==vybuild::actions::ActionStatus::Success) { deepness--; return status; } else { if (status==vybuild::actions::ActionStatus::Fail) { print_deepness("RunSubmoduleIfAction: Submodule failed to execute."); } else if (status==vybuild::actions::ActionStatus::SystemError) { print_deepness("RunSubmoduleIfAction: A system error happened in submodule."); } else if (status==vybuild::actions::ActionStatus::ExitFail) { print_deepness("RunSubmoduleAction: The submodule exited by itself, raising an error."); } if (this->block_if_fail) { deepness--; return status; } else { deepness--; return vybuild::actions::ActionStatus::Success; } } } else { deepness++; vybuild::module::Module submodule(this->file,true); print_deepness("Running submodule on secondary process: "+submodule.name); pid_t pid=fork(); if (pid<0) { print_deepness("RunSubmoduleIfAction: An error happened while forking."); if (this->block_if_fail) { deepness--; return vybuild::actions::ActionStatus::SystemError; } else { deepness--; return vybuild::actions::ActionStatus::Success; } } if (pid==0) { auto status=submodule.run_module(false,false); exit(0); } else { return vybuild::actions::ActionStatus::Success; } } } vybuild::actions::ActionStatus vybuild::actions::RunCommandWaitDirAction::run_action() { if (command_variables) { this->command=vybuild::parser::vector_string_vars(this->command); } if (this->working_dir_variables) { this->working_dir=vybuild::parser::string_vars(this->working_dir); } char path[PATH_MAX]; getcwd(path,PATH_MAX); chdir(this->working_dir.c_str()); if (this->show_output=="silent") { int pipefd[2]; if (pipe(pipefd)==-1) { print_deepness("RunCommandWaitDirAction: An error happened while creating pipe."); return vybuild::actions::ActionStatus::SystemError; } pid_t pid=fork(); if (pid<0) { print_deepness("RunCommandWaitDirAction: An error happened while forking."); return vybuild::actions::ActionStatus::SystemError; } if (pid==0) { close(pipefd[0]); dup2(pipefd[1],STDOUT_FILENO); dup2(pipefd[1],STDERR_FILENO); close(pipefd[1]); std::vector argv; for (int i=0;icommand.size();i++) { argv.push_back(command[i].c_str()); } argv.push_back(nullptr); execvp(command[0].c_str(),const_cast(argv.data())); print_deepness("RunCommandWaitDirAction: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer<ignore_success_status) { if (find(this->success_status.begin(),this->success_status.end(),WEXITSTATUS(status))==this->success_status.end()) { print_deepness("RunCommandWaitDirAction: Command exited with error (status "+to_string(WEXITSTATUS(status))+"). Stopping."); return vybuild::actions::ActionStatus::Fail; } else { chdir(path); return vybuild::actions::ActionStatus::Success; } } else { chdir(path); return vybuild::actions::ActionStatus::Success; } } } else if (this->show_output=="on_failure") { int pipefd[2]; if (pipe(pipefd)==-1) { print_deepness("RunCommandWaitDirAction: An error happened while creating pipe."); return vybuild::actions::ActionStatus::SystemError; } pid_t pid=fork(); if (pid<0) { print_deepness("RunCommandWaitDirAction: An error happened while forking."); return vybuild::actions::ActionStatus::SystemError; } if (pid==0) { close(pipefd[0]); dup2(pipefd[1],STDOUT_FILENO); dup2(pipefd[1],STDERR_FILENO); close(pipefd[1]); std::vector argv; for (int i=0;icommand.size();i++) { argv.push_back(command[i].c_str()); } argv.push_back(nullptr); execvp(command[0].c_str(),const_cast(argv.data())); print_deepness("RunCommandWaitDirAction: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer<ignore_success_status) { if (find(this->success_status.begin(),this->success_status.end(),WEXITSTATUS(status))==this->success_status.end()) { print_deepness("RunCommandWaitDirAction: Command exited with error (status "+to_string(WEXITSTATUS(status))+"), showing output:\n"); cout< argv; for (int i=0;icommand.size();i++) { argv.push_back(command[i].c_str()); } argv.push_back(nullptr); execvp(command[0].c_str(),const_cast(argv.data())); print_deepness("RunCommandWaitDirAction: An error happened while executing command."); _exit(1); } else { int status; waitpid(pid,&status,0); if (!WIFEXITED(status)) { print_deepness("RunCommandWaitDirAction: An error happened in forked process."); return vybuild::actions::ActionStatus::SystemError; } cout<ignore_success_status) { if (find(this->success_status.begin(),this->success_status.end(),WEXITSTATUS(status))==this->success_status.end()) { print_deepness("RunCommandWaitDirAction: Command exited with error (status "+to_string(WEXITSTATUS(status))+"). Stopping."); return vybuild::actions::ActionStatus::Fail; } else { chdir(path); return vybuild::actions::ActionStatus::Success; } } else { chdir(path); return vybuild::actions::ActionStatus::Success; } } } chdir(path); return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::CopyFolderAction::run_action() { if (this->source_folder_variables) this->source_folder=vybuild::parser::string_vars(this->source_folder); if (this->destination_folder_variables) this->destination_folder=vybuild::parser::string_vars(this->destination_folder); if (!fs::exists(this->source_folder) || !fs::is_directory(this->source_folder)) { print_deepness("CopyFolderAction: Source folder doesn't exist or isn't a folder: "+this->source_folder); return vybuild::actions::ActionStatus::Fail; } if (!fs::exists(this->destination_folder)) { print_deepness("CopyFolderAction: Destination doesn't exist: "+this->destination_folder); return vybuild::actions::ActionStatus::Fail; } if (!fs::is_directory(this->destination_folder)) { print_deepness("CopyFolderAction: Destination isn't a folder: "+this->destination_folder); return vybuild::actions::ActionStatus::Fail; } fs::copy(fs::absolute(this->source_folder),fs::path(this->destination_folder),fs::copy_options::overwrite_existing | fs::copy_options::recursive); return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus parallel_submodule(string& submodule) { vybuild::module::Module module(submodule,false); { lock_guard print(print_mutex); print_deepness("Running submodule in parallel: "+module.name); } return module.run_module(false,true); } vybuild::actions::ActionStatus parallel_submodule_worker(size_t thread_num) { while (true) { function task; { std::lock_guard lock(tasks_mutex); if (tasks.empty()) break; task=std::move(tasks.front()); tasks.pop(); } auto status=task(); if (status!=vybuild::actions::ActionStatus::Success) { return status; } } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::RunSubmoduleParallelAction::run_action() { if (this->submodules_variables) this->submodules=vybuild::parser::vector_string_vars(this->submodules); size_t nb_thread=thread::hardware_concurrency(); size_t original_quantity=this->submodules.size(); for (int i=0;isubmodules.size();i++) { string module=std::move(this->submodules[i]); tasks.push([module=std::move(module)]() mutable -> vybuild::actions::ActionStatus { return parallel_submodule(module); }); } vector threads; vector results; mutex results_mutex; size_t real_threads=0; if (original_quantity>nb_thread) { real_threads=nb_thread; } else { real_threads=original_quantity; } results.resize(real_threads,vybuild::actions::ActionStatus::Success); deepness++; for (size_t i=0;i lock(results_mutex); results[i]=status; }); } for (auto &t:threads) { t.join(); } deepness--; for (auto status:results) { if (status!=vybuild::actions::ActionStatus::Success) { return status; } } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::ExtractFromCacheAction::run_action() { if (this->target_variables) this->target=vybuild::parser::string_vars(this->target); if (this->output_file_variables) this->output_file=vybuild::parser::string_vars(this->output_file); string key=vybuild::cache::xxhash_to_string(XXH3_128bits(this->target.data(),this->target.size())); if (!vybuild::cache::cache_contains(key)) { return vybuild::actions::ActionStatus::Success; } else { vybuild::cache::extract_file(key,this->output_file); return vybuild::actions::ActionStatus::Success; } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::StoreInCacheAction::run_action() { if (this->key_variables) this->key=vybuild::parser::string_vars(this->key); if (this->file_variables) this->file=vybuild::parser::string_vars(this->file); string key=vybuild::cache::xxhash_to_string(XXH3_128bits(this->key.data(),this->key.size())); if (!vybuild::cache::cache_contains(key)) { vybuild::cache::store_file(key,this->file); return vybuild::actions::ActionStatus::Success; } else { if (this->overwrite_allowed) { vybuild::cache::store_file(key,this->file); return vybuild::actions::ActionStatus::Success; } else { return vybuild::actions::ActionStatus::Success; } } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::EnsureFolderResetAction::run_action() { if (this->path_variables) this->path=vybuild::parser::string_vars(this->path); if (!fs::exists(this->path)) { fs::create_directories(this->path); } else { fs::remove_all(this->path); fs::create_directories(this->path); } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::RunActionsIfAction::run_action() { if (vybuild::condition::evaluate_condition(this->condition)==false) { return vybuild::actions::ActionStatus::Success; } for (const auto& act_j:this->actions) { if (act_j["action"].get()=="exit_current_module") { if (act_j["args"]["is_fail"].get()) { return vybuild::actions::ActionStatus::ExitFail; } else { return vybuild::actions::ActionStatus::ExitSuccess; } } auto act=vybuild::actions::return_action(act_j["args"],act_j["action"],this->is_threading_allowed,this->root_folder); vybuild::actions::ActionStatus status=std::visit([](auto& act) { return act.run_action(); },act); if (status!=vybuild::actions::ActionStatus::Success) { return status; } } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::RunCommandStrAction::run_action() { if (command_variables) { this->command=vybuild::parser::string_vars(this->command); } char path[PATH_MAX]; getcwd(path,PATH_MAX); if (this->show_output=="silent") { int pipefd[2]; if (pipe(pipefd)==-1) { print_deepness("RunCommandStrAction: An error happened while creating pipe."); return vybuild::actions::ActionStatus::SystemError; } pid_t pid=fork(); if (pid<0) { print_deepness("RunCommandStrAction: An error happened while forking."); return vybuild::actions::ActionStatus::SystemError; } if (pid==0) { close(pipefd[0]); dup2(pipefd[1],STDOUT_FILENO); dup2(pipefd[1],STDERR_FILENO); close(pipefd[1]); std::vector argv; auto args=split_args_shell(this->command); for (int i=0;i(argv.data())); print_deepness("RunCommandStrAction: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer<ignore_success_status) { if (find(this->success_status.begin(),this->success_status.end(),WEXITSTATUS(status))==this->success_status.end()) { print_deepness("RunCommandStrAction: Command exited with error (status "+to_string(WEXITSTATUS(status))+"). Stopping."); return vybuild::actions::ActionStatus::Fail; } else { return vybuild::actions::ActionStatus::Success; } } else { return vybuild::actions::ActionStatus::Success; } } } else if (this->show_output=="on_failure") { int pipefd[2]; if (pipe(pipefd)==-1) { print_deepness("RunCommandStrAction: An error happened while creating pipe."); return vybuild::actions::ActionStatus::SystemError; } pid_t pid=fork(); if (pid<0) { print_deepness("RunCommandStrAction: An error happened while forking."); return vybuild::actions::ActionStatus::SystemError; } if (pid==0) { close(pipefd[0]); dup2(pipefd[1],STDOUT_FILENO); dup2(pipefd[1],STDERR_FILENO); close(pipefd[1]); std::vector argv; auto args=split_args_shell(this->command); for (int i=0;i(argv.data())); print_deepness("RunCommandStrAction: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer<ignore_success_status) { if (find(this->success_status.begin(),this->success_status.end(),WEXITSTATUS(status))==this->success_status.end()) { print_deepness("RunCommandStrAction: Command exited with error (status "+to_string(WEXITSTATUS(status))+"), showing output:\n"); cout< argv; auto args=split_args_shell(this->command); for (int i=0;i(argv.data())); print_deepness("RunCommandStrAction: An error happened while executing command."); _exit(1); } else if (this->show_output=="live") { int status; waitpid(pid,&status,0); if (!WIFEXITED(status)) { print_deepness("RunCommandStrAction: An error happened in forked process."); return vybuild::actions::ActionStatus::SystemError; } cout<ignore_success_status) { if (find(this->success_status.begin(),this->success_status.end(),WEXITSTATUS(status))==this->success_status.end()) { print_deepness("RunCommandStrAction: Command exited with error (status "+to_string(WEXITSTATUS(status))+"). Stopping."); return vybuild::actions::ActionStatus::Fail; } else { return vybuild::actions::ActionStatus::Success; } } else { return vybuild::actions::ActionStatus::Success; } } } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::RunActionsIfElseAction::run_action() { if (vybuild::condition::evaluate_condition(this->condition)) { for (const auto& act_j:this->actions_if) { if (act_j["action"].get()=="exit_current_module") { if (act_j["args"]["is_fail"].get()) { return vybuild::actions::ActionStatus::ExitFail; } else { return vybuild::actions::ActionStatus::ExitSuccess; } } auto act=vybuild::actions::return_action(act_j["args"],act_j["action"],this->is_threading_allowed,this->root_folder); vybuild::actions::ActionStatus status=std::visit([](auto& act) { return act.run_action(); },act); if (status!=vybuild::actions::ActionStatus::Success) { return status; } } } else { for (const auto& act_j:this->actions_else) { if (act_j["action"].get()=="exit_current_module") { if (act_j["args"]["is_fail"].get()) { return vybuild::actions::ActionStatus::ExitFail; } else { return vybuild::actions::ActionStatus::ExitSuccess; } } auto act=vybuild::actions::return_action(act_j["args"],act_j["action"],this->is_threading_allowed,this->root_folder); vybuild::actions::ActionStatus status=std::visit([](auto& act) { return act.run_action(); },act); if (status!=vybuild::actions::ActionStatus::Success) { return status; } } } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::ComputeElementHashAction::run_action() { if (this->element_variables) this->element=vybuild::parser::string_vars(this->element); if (!fs::exists(this->element)) { print_deepness("ComputeElementHashAction: Provided element doesn't exists."); return vybuild::actions::ActionStatus::Fail; } string value=""; if (fs::is_directory(this->element)) { string content; for (auto &element:fs::recursive_directory_iterator(this->element)) { content+="\n"+fs::absolute(element.path()).string(); if (element.is_regular_file()) { ifstream el_file(fs::path(element.path())); vector el_data(fs::file_size(fs::absolute(element.path()))); if (!el_file) { print_deepness("ComputeElementHashAction: Couldn't open element: "+element.path().string()); return vybuild::actions::ActionStatus::SystemError; } el_file.read(el_data.data(),el_data.size()); if (!el_file) { print_deepness("ComputeElementHashAction: Couldn't read element file: "+element.path().string()); return vybuild::actions::ActionStatus::SystemError; } content+="\n"+string(el_data.begin(),el_data.end()); } } value=vybuild::cache::xxhash_to_string(XXH3_128bits(content.data(),content.size())); } else if (fs::is_regular_file(this->element)) { string content=""; content+="\n"+fs::absolute(this->element).string(); ifstream el_file(fs::path(this->element)); vector el_data(fs::file_size(fs::absolute(this->element))); if (!el_file) { print_deepness("ComputeElementHashAction: Couldn't open element: "+this->element); return vybuild::actions::ActionStatus::SystemError; } el_file.read(el_data.data(),el_data.size()); if (!el_file) { print_deepness("ComputeElementHashAction: Couldn't read element file: "+this->element); return vybuild::actions::ActionStatus::SystemError; } content+="\n"+string(el_data.begin(),el_data.end()); value=vybuild::cache::xxhash_to_string(XXH3_128bits(content.data(),content.size())); } else { print_deepness("ComputeElementHashAction: Path isn't a hashable element: "+this->element); return vybuild::actions::ActionStatus::SystemError; } vybuild::vars::set_global_variable("\\$"+this->output_variable+"\\$",value); return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus parallel_compilation(string source,string compiler_path,string compiler_hash,vector pre_compiler_args,vector post_compiler_args,string header_command,vector success_status,bool ignore_success_status,pair args_variables,string output_file) { if (args_variables.first) pre_compiler_args=vybuild::parser::vector_string_vars(pre_compiler_args); if (args_variables.second) post_compiler_args=vybuild::parser::vector_string_vars(post_compiler_args); vector temp; for (auto arg:pre_compiler_args) { auto element=arg; regex regvar("%outputfile%"); arg=regex_replace(arg,regvar,output_file); temp.push_back(arg); } pre_compiler_args=temp; for (auto arg:post_compiler_args) { auto element=arg; regex regvar("%outputfile%"); arg=regex_replace(arg,regvar,output_file); temp.push_back(arg); } post_compiler_args=temp; if (compiler_hash!="") { string c_content=""; auto vars_list=vybuild::vars::get_global_variables_list(); fs::path dfile_path(source+".d"); if (header_command!="") { string dep_command=header_command; regex regvar1("%dfile%"); dep_command=regex_replace(dep_command,regvar1,dfile_path.string()); regex regvar2("%sourcefile%"); dep_command=regex_replace(dep_command,regvar2,source); int res=std::system(dep_command.c_str()); if (res!=0) { print_deepness("ParallelCompilation: Cannot get headers list for file \""+source+"\" with command: "+dep_command); return vybuild::actions::ActionStatus::Fail; } auto headers=headers_from_dfile(dfile_path); for (auto &h:headers) { ifstream header(h); vector header_data(fs::file_size(fs::absolute(h))); if (!header) { print_deepness("ParallelCompilation: Couldn't open source/header file: "+h); return vybuild::actions::ActionStatus::SystemError; } header.read(header_data.data(),header_data.size()); if (!header) { print_deepness("ParallelCompilation: Couldn't read source/header file: "+h); return vybuild::actions::ActionStatus::SystemError; } c_content+=string(header_data.begin(),header_data.end()); header.close(); } fs::remove(dfile_path); } c_content+="\n"+compiler_path; c_content+="\n"+compiler_hash; for (auto& a:pre_compiler_args) c_content+="\n"+a; for (auto& a:post_compiler_args) c_content+="\n"+a; string key=vybuild::cache::xxhash_to_string(XXH3_128bits(c_content.data(),c_content.size())); if (vybuild::cache::cache_contains(key)) { vybuild::cache::extract_file(key,output_file); return vybuild::actions::ActionStatus::Success; } int pipefd[2]; if (pipe(pipefd)==-1) { print_deepness("ParallelCompilation: An error happened while creating pipe."); return vybuild::actions::ActionStatus::SystemError; } pid_t pid=fork(); if (pid<0) { print_deepness("ParallelCompilation: An error happened while forking."); return vybuild::actions::ActionStatus::SystemError; } if (pid==0) { close(pipefd[0]); dup2(pipefd[1],STDOUT_FILENO); dup2(pipefd[1],STDERR_FILENO); close(pipefd[1]); std::vector argv={compiler_path.c_str()}; for (int i=0;i(argv.data())); print_deepness("ParallelCompilation: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer< argv={compiler_path.c_str()}; for (int i=0;i(argv.data())); print_deepness("ParallelCompilation: An error happened while executing command."); _exit(1); } else { close(pipefd[1]); std::stringstream buffer; char c; while (read(pipefd[0],&c,1)>0) { buffer< task; { std::lock_guard lock(tasks_mutex); if (tasks.empty()) break; task=std::move(tasks.front()); tasks.pop(); } auto status=task(); if (status!=vybuild::actions::ActionStatus::Success) { return status; } } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::CompileMultipleCppAction::run_action() { if (this->compiler_variables) this->compiler=vybuild::parser::string_vars(this->compiler); if (this->headers_command_variables) this->headers_command=vybuild::parser::string_vars(this->headers_command); if (!this->source_files_path_completion) { if (this->source_files_variables) this->source_files=vybuild::parser::vector_string_vars(this->source_files); if (this->output_files_variables) this->output_files=vybuild::parser::vector_string_vars(this->output_files); if (this->source_files.size()!=this->output_files.size()) { print_deepness("CompileMultipleCppAction: Source and output files doesn't have the same amount of elements."); return vybuild::actions::ActionStatus::Fail; } } else { if (this->output_files.size()!=0) { print_deepness("CompileMultipleCppAction: When source files path completion is enabled, output files must be empty as it's automatically generated."); return vybuild::actions::ActionStatus::Fail; } if (this->source_files_variables) this->source_files=vybuild::parser::vector_string_vars(this->source_files); this->source_files=vybuild::parser::vector_string_path_completions(this->source_files,this->path_completion_source_ext); for (auto &s:this->source_files) { fs::path source_path(s); fs::path parent=source_path.parent_path(); string filename=source_path.filename().string(); string base_name=filename.substr(0,filename.size()-this->path_completion_source_ext.size()); string output_name=base_name+this->path_completion_output_ext; this->output_files.push_back(parent/output_name); } } string compiler_path=find_executable(this->compiler); string compiler_hash=""; auto vars_list=vybuild::vars::get_global_variables_list(); if (this->cache_authorized) { ifstream compiler_bin(compiler_path,ios::binary); if (!compiler_bin) { print_deepness("CompileMultipleCppAction: Cannot open compiler binary: "+compiler_path); return vybuild::actions::ActionStatus::SystemError; } size_t compiler_size=fs::file_size(compiler_path); vector compiler_data(compiler_size); compiler_bin.read(reinterpret_cast(compiler_data.data()),compiler_size); if (!compiler_bin) { print_deepness("CompileMultipleCppAction: Cannot read compiler binary: "+compiler_path); return vybuild::actions::ActionStatus::SystemError; } compiler_hash=vybuild::cache::xxhash_to_string(XXH3_128bits(compiler_data.data(),compiler_data.size())); for (auto &v:vars_list) { if (v.starts_with("\\$__VYBUILD_COMPILER_HASH_")) { compiler_hash+="\n"+get_command_output(vybuild::vars::get_global_variable(v)); } } } size_t nb_thread=thread::hardware_concurrency(); size_t original_quantity=this->source_files.size(); for (int i=0;isource_files[i]); string headers_cmd=this->headers_command; auto pre_args=this->pre_arguments; auto post_args=this->post_arguments; auto success_status=this->success_status; auto ignore_success_status=this->ignore_success_status; auto output_file=std::move(this->output_files[i]); pair args_vars={this->pre_arguments_variables,this->post_arguments_variables}; tasks.push([source_file=std::move(source_file),compiler_path,compiler_hash,pre_args=std::move(pre_args),post_args=std::move(post_args),headers_cmd=std::move(headers_cmd),success_status=std::move(success_status),ignore_success_status=std::move(ignore_success_status),args_vars=std::move(args_vars),output_file=std::move(output_file)]() mutable -> vybuild::actions::ActionStatus { return parallel_compilation(source_file,compiler_path,compiler_hash,pre_args,post_args,headers_cmd,success_status,ignore_success_status,args_vars,output_file); }); } vector threads; vector results; mutex results_mutex; size_t real_threads=0; if (original_quantity>nb_thread) { real_threads=nb_thread; } else { real_threads=original_quantity; } results.resize(real_threads,vybuild::actions::ActionStatus::Success); for (size_t i=0;i lock(results_mutex); results[i]=status; }); } for (auto &t:threads) { t.join(); } for (auto status:results) { if (status!=vybuild::actions::ActionStatus::Success) { return status; } } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::CopyFileAction::run_action() { if (this->source_file_variables) this->source_file=vybuild::parser::string_vars(this->source_file); if (this->destination_folder_variables) this->destination_folder=vybuild::parser::string_vars(this->destination_folder); if (!fs::exists(this->source_file) || !fs::is_regular_file(this->source_file)) { print_deepness("CopyFolderAction: Source file doesn't exist or isn't a folder: "+this->source_file); return vybuild::actions::ActionStatus::Fail; } if (!fs::exists(fs::path(this->destination_folder).parent_path())) { print_deepness("CopyFolderAction: Destination parent folder doesn't exist: "+this->destination_folder); return vybuild::actions::ActionStatus::Fail; } fs::copy_file(fs::absolute(this->source_file),fs::path(this->destination_folder),fs::copy_options::overwrite_existing); return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::CreateFat32Action::run_action() { if (this->key_variables) this->key=vybuild::parser::string_vars(this->key); if (this->size==0) { print_deepness("CreateFat32Action: Size is null."); return vybuild::actions::ActionStatus::Fail; } if (this->key=="") { print_deepness("CreateFat32Action: Key is null."); return vybuild::actions::ActionStatus::Fail; } try { vybuild::fat32::create_fat32(this->key,this->size); } catch (const exception& e) { print_deepness("CreateFat32Action: "+string(e.what())); return vybuild::actions::ActionStatus::Fail; } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::CreateFolderFat32Action::run_action() { if (this->key_variables) this->key=vybuild::parser::string_vars(this->key); if (this->full_path_variables) this->full_path=vybuild::parser::string_vars(this->full_path); if (this->full_path=="") { print_deepness("CreateFolderFat32Action: Full path is null."); return vybuild::actions::ActionStatus::Fail; } if (this->key=="") { print_deepness("CreateFolderFat32Action: Key is null."); return vybuild::actions::ActionStatus::Fail; } try { vybuild::fat32::add_directory_fat32(this->key,this->full_path); } catch (const exception& e) { print_deepness("CreateFolderFat32Action: "+string(e.what())); return vybuild::actions::ActionStatus::Fail; } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::CreateFileFat32Action::run_action() { if (this->key_variables) this->key=vybuild::parser::string_vars(this->key); if (this->source_path_variables) this->source_path=vybuild::parser::string_vars(this->source_path); if (this->full_path_variables) this->full_path=vybuild::parser::string_vars(this->full_path); if (this->source_path=="") { print_deepness("CreateFileFat32Action: Source path is null."); return vybuild::actions::ActionStatus::Fail; } if (this->full_path=="") { print_deepness("CreateFileFat32Action: Full path is null."); return vybuild::actions::ActionStatus::Fail; } if (this->key=="") { print_deepness("CreateFileFat32Action: Key is null."); return vybuild::actions::ActionStatus::Fail; } if (!fs::exists(this->source_path)) { print_deepness("CreateFileFat32Action: Source path doesn't exist."); return vybuild::actions::ActionStatus::Fail; } if (!fs::is_regular_file(this->source_path)) { print_deepness("CreateFileFat32Action: Source path isn't a regular file."); return vybuild::actions::ActionStatus::Fail; } ifstream source(this->source_path,ios::binary); if (!source) { print_deepness("CreateFileFat32Action: Couldn't open source file."); return vybuild::actions::ActionStatus::SystemError; } vector source_data(fs::file_size(this->source_path)); if (!source.read(reinterpret_cast(source_data.data()),source_data.size())) { print_deepness("CreateFileFat32Action: Couldn't read source file."); return vybuild::actions::ActionStatus::SystemError; } if (this->is_vftm_target && vybuild::fat32::is_vftm_target_set()) { print_deepness("CreateFileFat32Action: VFTM target is already set."); return vybuild::actions::ActionStatus::Fail; } try { vybuild::fat32::add_file_fat32(this->key,this->full_path,source_data.data(),source_data.size(),this->is_vftm_target); } catch (const exception& e) { print_deepness("CreateFileFat32Action: "+string(e.what())); return vybuild::actions::ActionStatus::Fail; } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::ExportFat32Action::run_action() { if (this->key_variables) this->key=vybuild::parser::string_vars(this->key); if (this->export_path_variables) this->export_path=vybuild::parser::string_vars(this->export_path); if (this->export_path=="") { print_deepness("ExportFat32Action: Export path is null."); return vybuild::actions::ActionStatus::Fail; } if (this->key=="") { print_deepness("ExportFat32Action: Key is null."); return vybuild::actions::ActionStatus::Fail; } vector image; try { image=vybuild::fat32::export_fat32(this->key,this->generate_vftm,this->disk_guid_var,this->efi_part_unique_guid_var); } catch (const exception& e) { print_deepness("ExportFat32Action: "+string(e.what())); return vybuild::actions::ActionStatus::Fail; } ofstream output(this->export_path,ios::binary); if (!output) { print_deepness("ExportFat32Action: Couldn't write image in specified output file."); return vybuild::actions::ActionStatus::SystemError; } if (!output.write(reinterpret_cast(image.data()),image.size())) { print_deepness("ExportFat32Action: Couldn't write image in specified output file."); return vybuild::actions::ActionStatus::SystemError; } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::CreateDiskAction::run_action() { if (this->key_variables) this->key=vybuild::parser::string_vars(this->key); if (this->disk_guid_variables) this->disk_guid=vybuild::parser::string_vars(this->disk_guid); if (this->size==0) { print_deepness("CreateDiskAction: Size is null."); return vybuild::actions::ActionStatus::Fail; } if (this->key=="") { print_deepness("CreateDiskAction: Key is null."); return vybuild::actions::ActionStatus::Fail; } if (this->disk_guid=="") { print_deepness("CreateDiskAction: Disk GUID is null."); return vybuild::actions::ActionStatus::Fail; } try { vybuild::disk::create_disk(this->key,this->size,this->disk_guid); } catch (const exception& e) { print_deepness("CreateDiskAction: "+string(e.what())); return vybuild::actions::ActionStatus::Fail; } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::AddPartitionDiskAction::run_action() { if (this->key_variables) this->key=vybuild::parser::string_vars(this->key); if (this->type_guid_variables) this->type_guid=vybuild::parser::string_vars(this->type_guid); if (this->unique_guid_variables) this->unique_guid=vybuild::parser::string_vars(this->unique_guid); if (this->name_variables) this->name=vybuild::parser::string_vars(this->name); if (this->source_file_variables) this->source_file=vybuild::parser::string_vars(this->source_file); if (this->key=="") { print_deepness("AddPartitionDiskAction: Key is null."); return vybuild::actions::ActionStatus::Fail; } if (this->type_guid=="") { print_deepness("AddPartitionDiskAction: Type GUID is null."); return vybuild::actions::ActionStatus::Fail; } if (this->unique_guid=="") { print_deepness("AddPartitionDiskAction: Unique GUID is null."); return vybuild::actions::ActionStatus::Fail; } if (this->name=="") { print_deepness("AddPartitionDiskAction: Name is null."); return vybuild::actions::ActionStatus::Fail; } if (this->source_file=="") { print_deepness("AddPartitionDiskAction: Source file is null."); return vybuild::actions::ActionStatus::Fail; } if (!fs::exists(this->source_file)) { print_deepness("AddPartitionDiskAction: Source file doesn't exists."); return vybuild::actions::ActionStatus::Fail; } if (!fs::is_regular_file(this->source_file)) { print_deepness("AddPartitionDiskAction: Source file isn't a regular file."); return vybuild::actions::ActionStatus::Fail; } ifstream source(this->source_file,ios::binary); if (!source) { print_deepness("AddPartitionDiskAction: Couldn't open source file."); return vybuild::actions::ActionStatus::SystemError; } vector source_data(fs::file_size(this->source_file)); if (!source.read(reinterpret_cast(source_data.data()),source_data.size())) { print_deepness("AddPartitionDiskAction: Couldn't read source file."); return vybuild::actions::ActionStatus::SystemError; } size_t buf_size=fs::file_size(this->source_file); size_t part_size=((buf_size+511)/512)*512; try { vybuild::disk::add_partition(this->key,part_size,this->type_guid,this->unique_guid,0,this->name,source_data.data(),buf_size); } catch (const exception& e) { print_deepness("AddPartitionDiskAction: "+string(e.what())); return vybuild::actions::ActionStatus::Fail; } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::ExportDiskAction::run_action() { if (this->key_variables) this->key=vybuild::parser::string_vars(this->key); if (this->export_path_variables) this->export_path=vybuild::parser::string_vars(this->export_path); if (this->export_path=="") { print_deepness("ExportDiskAction: Export path is null."); return vybuild::actions::ActionStatus::Fail; } if (this->key=="") { print_deepness("ExportDiskAction: Key is null."); return vybuild::actions::ActionStatus::Fail; } vector image; try { image=vybuild::disk::export_disk(this->key); } catch (const exception& e) { print_deepness("ExportDiskAction: "+string(e.what())); return vybuild::actions::ActionStatus::Fail; } ofstream output(this->export_path,ios::binary); if (!output) { print_deepness("ExportDiskAction: Couldn't write image in specified output file."); return vybuild::actions::ActionStatus::SystemError; } if (!output.write(reinterpret_cast(image.data()),image.size())) { print_deepness("ExportDiskAction: Couldn't write image in specified output file."); return vybuild::actions::ActionStatus::SystemError; } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::VFTMLoadCryptoLibAction::run_action() { if (this->lib_path_variables) this->lib_path=vybuild::parser::string_vars(this->lib_path); if (vybuild::fat32::is_crypto_lib_loaded()) { print_deepness("VFTMLoadCryptoLib: cryptography library is already loaded."); return vybuild::actions::ActionStatus::Fail; } try { vybuild::fat32::load_crypto_lib(this->lib_path); } catch (const exception& e) { print_deepness("VFTMLoadCryptoLib: "+string(e.what())); return vybuild::actions::ActionStatus::Fail; } return vybuild::actions::ActionStatus::Success; } vybuild::actions::ActionStatus vybuild::actions::VFTMRunKeysGeneratorAction::run_action() { if (this->command_variables) this->command=vybuild::parser::vector_string_vars(this->command); if (this->working_dir_variables) this->working_dir=vybuild::parser::string_vars(this->working_dir); if (this->socket_path_variables) this->socket_path=vybuild::parser::string_vars(this->socket_path); char path[PATH_MAX]; getcwd(path,PATH_MAX); chdir(this->working_dir.c_str()); print_deepness("VFTMRunKeysGeneratorAction: Live output of command:\n"); int fd=socket(AF_UNIX,SOCK_SEQPACKET,0); if (fd<0) { print_deepness("VFTMRunKeysGeneratorAction: An error happened while creating socket."); return vybuild::actions::ActionStatus::SystemError; } struct timeval tv; tv.tv_sec=30; tv.tv_usec=0; setsockopt(fd,SOL_SOCKET,SO_RCVTIMEO,&tv,sizeof(tv)); sockaddr_un addr{}; addr.sun_family=AF_UNIX; strcpy(addr.sun_path,this->socket_path.c_str()); unlink(this->socket_path.c_str()); int r=bind(fd,(sockaddr*)&addr,sizeof(addr)); if (r<0) { print_deepness("VFTMRunKeysGeneratorAction: An error happened while binding socket."); return vybuild::actions::ActionStatus::SystemError; } listen(fd,1); pid_t pid=fork(); if (pid<0) { print_deepness("VFTMRunKeysGeneratorAction: An error happened while forking."); return vybuild::actions::ActionStatus::SystemError; } if (pid==0) { std::vector argv; for (int i=0;icommand.size();i++) { argv.push_back(this->command[i].c_str()); } argv.push_back(nullptr); execvp(command[0].c_str(),const_cast(argv.data())); print_deepness("VFTMRunKeysGeneratorAction: An error happened while executing command."); _exit(1); } else { uint8_t manifest_keypair[192]; memset(manifest_keypair,0,192); int client=accept(fd,nullptr,nullptr); struct ucred cred; socklen_t len=sizeof(cred); if (getsockopt(client,SOL_SOCKET,SO_PEERCRED,&cred,&len)<0) { print_deepness("VFTMRunKeysGeneratorAction: getsockopt fail."); return vybuild::actions::ActionStatus::SystemError; } if (cred.uid!=0) { print_deepness("VFTMRunKeysGeneratorAction: Wrong UID."); return vybuild::actions::ActionStatus::Fail; } ssize_t n=recv(client,manifest_keypair,sizeof(manifest_keypair),0); if (n<0 || n!=sizeof(manifest_keypair)) { print_deepness("VFTMRunKeysGeneratorAction: Couldn't receive manifest keypair."); return vybuild::actions::ActionStatus::SystemError; } int status; waitpid(pid,&status,0); if (!WIFEXITED(status)) { print_deepness("VFTMRunKeysGeneratorAction: An error happened in forked process."); return vybuild::actions::ActionStatus::SystemError; } cout<ignore_success_status) { if (find(this->success_status.begin(),this->success_status.end(),WEXITSTATUS(status))==this->success_status.end()) { print_deepness("VFTMRunKeysGeneratorAction: Command exited with error (status "+to_string(WEXITSTATUS(status))+"). Stopping."); return vybuild::actions::ActionStatus::Fail; } else { memcpy(vybuild::fat32::get_manifest_keypair_buffer(),manifest_keypair,192); chdir(path); return vybuild::actions::ActionStatus::Success; } } else { memcpy(vybuild::fat32::get_manifest_keypair_buffer(),manifest_keypair,192); chdir(path); return vybuild::actions::ActionStatus::Success; } } chdir(path); return vybuild::actions::ActionStatus::Success; }