diff --git a/.gitignore b/.gitignore index 660f1cb..f5ebb16 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ ccc linux/ +*.tar.xz +*.ccc diff --git a/ccc.cpp b/ccc.cpp index deb5359..21d0ea3 100644 --- a/ccc.cpp +++ b/ccc.cpp @@ -20,11 +20,14 @@ #include #include #include +#include #include #include #include #include #include +#include +#include using namespace std; namespace fs=filesystem; const uint64_t CCC_DELIMITER_0_HEAD=0b0; @@ -44,6 +47,11 @@ struct XXH3HasherString { return static_cast(XXH3_64bits(s.data(),s.size())); } }; +size_t get_terminal_width() { + struct winsize w; + ioctl(STDOUT_FILENO,TIOCGWINSZ,&w); + return w.ws_col?w.ws_col:80; +} class bit_streamer { private: vector out; @@ -226,11 +234,13 @@ struct header { size_t size_payload; }; #pragma pack(pop) +#pragma pack(push,1) struct node { uint32_t type; uint32_t start; uint32_t end; }; +#pragma pack(pop) struct file_entry { string name; string content; @@ -240,10 +250,10 @@ struct file_entry { struct thread_iterate_input_loop_call { string &source_code; vector &thread_local_node_list; - map& thread_local_rec_map; + unordered_map& thread_local_rec_map; }; struct thread_rec_map_result { - map thread_local_rec_map; + unordered_map thread_local_rec_map; }; struct thread_encoding_input_loop_call { string &source_code; @@ -253,6 +263,13 @@ struct thread_encoding_input_loop_call { struct thread_encoding_result { vector encoded_files; }; +atomic parsed_files=0; +atomic encoded_files=0; +size_t total_files=0; +atomic parsing_done=false; +atomic encoding_started=false; +atomic encoding_done=false; +mutex ui_mutex; queue rec_map_files_queue; mutex rec_map_queue_mutex; queue encoding_files_queue; @@ -271,6 +288,35 @@ bool show_warning=false; bool fail_on_warning=false; bool enable_malloc_trim=true; mutex type_alloc; +void print_progress_line(const string& label,size_t done,size_t total) { + size_t width=get_terminal_width(); + size_t percent=(total==0)?0:(done*100/total); + string prefix=label+" "+to_string(done)+"/"+to_string(total)+" files"; + size_t bar_width=width-prefix.size()-10; + if (bar_width>width) bar_width=10; + size_t filled=(total==0)?0:(done*bar_width/total); + string bar="["; + for (size_t i=0;i lock(ui_mutex); + while (!parsing_done) { + print_progress_line("Parsing",parsed_files.load(),total_files); + this_thread::sleep_for(chrono::milliseconds(50)); + } + cout<<"\n"; + encoding_started=true; + while (!encoding_done) { + print_progress_line("Encoding",encoded_files.load(),total_files); + this_thread::sleep_for(chrono::milliseconds(50)); + } + cout<<"\n"; + return; +} uint32_t get_id(const string& type) { { auto it=type_to_id.find(type); @@ -311,7 +357,6 @@ void iterate_all_nodes_loop_call(thread_iterate_input_loop_call &settings,TSNode } } thread_rec_map_result run_thread_rec_map(size_t thread_num) { - auto start=chrono::high_resolution_clock::now(); thread_rec_map_result res; TSParser *parser=ts_parser_new(); ts_parser_set_language(parser,tree_sitter_c()); @@ -344,11 +389,9 @@ thread_rec_map_result run_thread_rec_map(size_t thread_num) { encoding_files_queue.push(std::move(f)); } if (++counter%20==0 && enable_malloc_trim) malloc_trim(0); + parsed_files++; } ts_parser_delete(parser); - auto end=chrono::high_resolution_clock::now(); - auto ms=chrono::duration_cast(end-start).count(); - cout<<"Recccurences map thread number "<>=1; - ++bits; +void generate_rec(bit_streamer& bitstream,size_t index) { + if (index==0) { + bitstream.align(); + bitstream.write_bits(CCC_REC_TABLE_REF_HEAD,4); + bitstream.write_bits(0,1); + bitstream.write_bits(index,3); + bitstream.align(); + return; } + size_t tmp=index+1; + size_t k=0; + size_t threshold=1ULL<<3; + while (tmp>=threshold) { + tmp>>=3; + ++k; + } + size_t payload_bits=3*(k+1); bitstream.align(); bitstream.write_bits(CCC_REC_TABLE_REF_HEAD,4); - bitstream.write_bits(index,bits); + for (size_t i=0;isecond; - generate_rec(out,index,rec_list.size()); + generate_rec(out,index); } } else if (type==ID_PRIMITIVE_TYPE || type==ID_TYPE_IDENTIFIER) { auto it=c_keyword_lookup.find(string(text)); @@ -440,7 +497,7 @@ void process_file_nodes_loop_call(thread_encoding_input_loop_call& settings) { } } else { size_t index=it->second; - generate_rec(out,index,rec_list.size()); + generate_rec(out,index); } } } else if (delimiter0_lookup.find(id_to_type[type])!=delimiter0_lookup.end() || delimiter1_lookup.find(id_to_type[type])!=delimiter1_lookup.end() || type==ID_QUOTE) { @@ -542,7 +599,7 @@ void process_file_nodes_loop_call(thread_encoding_input_loop_call& settings) { } } else { size_t index=it->second; - generate_rec(out,index,rec_list.size()); + generate_rec(out,index); } } } @@ -566,7 +623,7 @@ void process_file_nodes_loop_call(thread_encoding_input_loop_call& settings) { } } else { size_t index=it->second; - generate_rec(out,index,rec_list.size()); + generate_rec(out,index); } } else { auto it=rec_lookup.find(id_to_type[type]); @@ -579,7 +636,7 @@ void process_file_nodes_loop_call(thread_encoding_input_loop_call& settings) { } } else { size_t index=it->second; - generate_rec(out,index,rec_list.size()); + generate_rec(out,index); } } } @@ -587,11 +644,9 @@ void process_file_nodes_loop_call(thread_encoding_input_loop_call& settings) { return; } thread_encoding_result run_thread_encoding(size_t thread_num) { - auto start=chrono::high_resolution_clock::now(); thread_encoding_result res; vector thread_local_encoded_files; int counter=0; - int max=0; while (true) { file_entry f; { @@ -611,11 +666,9 @@ thread_encoding_result run_thread_encoding(size_t thread_num) { vector().swap(encoding_loop_settings.node_list); string().swap(f.content); if (++counter%20==0 && enable_malloc_trim) malloc_trim(0); + encoded_files++; } res.encoded_files=std::move(thread_local_encoded_files); - auto end=chrono::high_resolution_clock::now(); - auto ms=chrono::duration_cast(end-start).count(); - cout<<"Parsing/encoding thread number "<> rec_map_futures; for (size_t i=0;i=2 && str.size()>=3) { rec_list.push_back(str); @@ -756,6 +811,7 @@ int main(int argc,char **argv) { for (auto& fut:encoding_futures) { all_encoding_results.push_back(fut.get()); } + encoding_done=true; vector globals_bit_stream; for (auto& res:all_encoding_results) { globals_bit_stream.insert(globals_bit_stream.end(),res.encoded_files.begin(),res.encoded_files.end()); @@ -775,6 +831,8 @@ int main(int argc,char **argv) { final_payloads.insert(final_payloads.end(),encoded_file.begin(),encoded_file.end()); current_offset+=encoded_file.size(); } + ui.join(); + lock_guard lock(ui_mutex); // // Payload compression diff --git a/ccc.cpp.save b/ccc.cpp.save deleted file mode 100644 index d47946c..0000000 --- a/ccc.cpp.save +++ /dev/null @@ -1,950 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -using namespace std; -namespace fs=filesystem; -const uint64_t CCC_DELIMITER_0_HEAD=0b0; -const uint64_t CCC_DELIMITER_1_HEAD=0b10; -const uint64_t CCC_C_KEYWORD_HEAD=0b1100; -const uint64_t CCC_MISCELANEOUS_HEAD=0b1101; -const uint64_t CCC_STRING_INLINE_HEAD=0b1110; -const uint64_t CCC_REC_TABLE_REF_HEAD=0b1111; -const uint64_t CCC_STRING_INLINE_END=0b00000000; -#define CCC_ADD_COMPONENT(vec,tail) \ - do { \ - auto tmp=tail; \ - vec.insert(vec.end(),tmp.begin(),tmp.end()); \ - } while (0) -struct XXH3HasherString { - size_t operator()(const std::string& s) const { - return static_cast(XXH3_64bits(s.data(),s.size())); - } -}; -class bit_streamer { - private: - vector out; - uint8_t current_byte=0; - uint8_t bit_pos=0; - public: - size_t index; - bit_streamer(size_t index) { - out.reserve(1024*1024); - this->index=index; - } - size_t get_size() { - return out.size(); - } - void write_bits(uint64_t value,uint8_t count) { - for (int i=count-1;i>=0;--i) { - if ((value>>i) & 1) { - current_byte|=(1<<(7-bit_pos)); - } - bit_pos++; - if (bit_pos==8) { - out.push_back(current_byte); - current_byte=0; - bit_pos=0; - } - } - } - void align() { - if (bit_pos>0) { - out.push_back(current_byte); - current_byte=0; - bit_pos=0; - } - } - const vector& get_out() const { - return out; - } - vector extract_buffer() { - align(); - return std::move(out); - } -}; -const vector delimiter0={ - "{", - "}", - "(", - ")", - "[", - "]", - ",", - "." -}; -const vector delimiter1={ - "{}", - "()", - "[]", - ";" -}; -const vector miscellaneous={ - "!", - "%", - "'", - "*", - "+", - "-", - "/", - ":", - "<", - ">", - "=", - "?", - "^", - "|", - "&", - "~", - "+=", - "-=", - "*=", - "/=", - "%=", - "&=", - "|=", - "^=", - "<<=", - ">>=", - "++", - "--", - "<<", - ">>", - "==", - "!=", - "<=", - ">=", - "->", - "...", - "||", - "&&", - "NULL", - "size_t", - "uint8_t", - "uint16_t", - "uint32_t", - "uint64_t", - "int8_t", - "int16_t", - "int32_t", - "int64_t" -}; -const vector c_keywords={ - "#if", - "#ifdef", - "#ifndef", - "#else", - "#elif", - "#elifdef", - "#elifndef", - "#endif", - "#define", - "#undef", - "#include", - "#error", - "#warning", - "#pragma", - "#line", - "alignas", - "alignof", - "auto", - "bool", - "break", - "case", - "char", - "const", - "constexpr", - "continue", - "default", - "do", - "double", - "else", - "enum", - "extern", - "false", - "float", - "for", - "goto", - "if", - "inline", - "int", - "long", - "nullptr", - "register", - "restrict", - "return", - "short", - "signed", - "sizeof", - "static", - "static_assert", - "struct", - "switch", - "thread_local", - "true", - "typedef", - "typeof", - "typeof_unequal", - "union", - "unsigned", - "void", - "volatile", - "while", - "__asm__", - "__attribute__", - "defined", -}; -#pragma pack(push,1) -struct header { - uint8_t sig[3]; - uint8_t flags; - size_t size_rec_table; - size_t entry_count; - size_t size_payload; -}; -#pragma pack(pop) -struct node { - uint16_t type; - uint32_t start; - uint32_t end; -}; -struct file_entry { - string name; - string content; - size_t size; - size_t index; -}; -struct thread_iterate_input_loop_call { - string &source_code; - vector &thread_local_node_list; - unordered_map& thread_local_type_map; - unordered_map& thread_local_type_u16_map; - uint16_t thread_local_next_type_id; - map thread_local_rec_map; -}; -struct thread_rec_map_result { - map thread_local_rec_map; -}; -struct thread_encoding_input_loop_call { - string &source_code; - vector &node_list; - unordered_map& thread_local_type_map; - bit_streamer& thread_local_bit_stream; -}; -struct thread_encoding_result { - vector encoded_files; -}; -enum iterating_mode { - REC_MAP, - PARSING -}; -queue rec_map_files_queue; -mutex rec_map_queue_mutex; -queue encoding_files_queue; -mutex encoding_queue_mutex; -vector rec_list; -unordered_map> rec_lookup; -unordered_map> c_keyword_lookup; -unordered_map> miscelaneous_lookup; -unordered_map> delimiter0_lookup; -unordered_map> delimiter1_lookup; -bool show_warning=false; -bool fail_on_warning=false; -bool enable_malloc_trim=true; -void iterate_all_nodes_loop_call(thread_iterate_input_loop_call &settings,TSNode current_node,iterating_mode mode) { - if (ts_node_child_count(current_node)==0) { - uint32_t start=ts_node_start_byte(current_node); - uint32_t end=ts_node_end_byte(current_node); - string_view text{settings.source_code.data()+start,end-start}; - string type=string(ts_node_type(current_node)); - if (mode==iterating_mode::REC_MAP) { - if (type=="string_content" || type=="system_lib_string" || type=="identifier" || type=="number_literal" || type=="type_identifier" || type=="field_identifier" || type=="escape_sequence" || type=="statement_identifier") { - settings.thread_local_rec_map[string(text)]++; - } - if (type=="primitive_type" && find(c_keywords.begin(),c_keywords.end(),text)==c_keywords.end()) { - settings.thread_local_rec_map[string(text)]++; - } - if (type=="comment") { - settings.thread_local_rec_map[string(text)]=2; - } - } else if (mode==iterating_mode::PARSING) { - if (settings.thread_local_type_u16_map.find(type)==settings.thread_local_type_u16_map.end()) { - settings.thread_local_type_u16_map[type]=settings.thread_local_next_type_id; - settings.thread_local_type_map[settings.thread_local_type_u16_map.at(type)]=type; - settings.thread_local_next_type_id++; - } - settings.thread_local_node_list.push_back({.type=settings.thread_local_type_u16_map[type],.start=start,.end=end}); - } - } else { - uint32_t child_count=ts_node_child_count(current_node); - for (uint32_t i=0;i useless_type_u16_map; - vector useless_node_vector; - unordered_map useless_type_map; - TSParser *parser=ts_parser_new(); - ts_parser_set_language(parser,tree_sitter_c()); - int counter=0; - while (true) { - file_entry f; - { - lock_guard lock(rec_map_queue_mutex); - if (rec_map_files_queue.empty()) break; - f=std::move(rec_map_files_queue.front()); - rec_map_files_queue.pop(); - } - thread_iterate_input_loop_call loop_settings { - .source_code=f.content, - .thread_local_node_list=useless_node_vector, - .thread_local_type_map=useless_type_map, - .thread_local_type_u16_map=useless_type_u16_map, - .thread_local_next_type_id=0, - .thread_local_rec_map=res.thread_local_rec_map - }; - TSTree *tree=ts_parser_parse_string(parser,nullptr,f.content.c_str(),f.content.size()); - TSNode root=ts_tree_root_node(tree); - loop_settings.source_code=f.content; - iterate_all_nodes_loop_call(loop_settings,root,iterating_mode::REC_MAP); - ts_tree_delete(tree); - { - lock_guard lock(encoding_queue_mutex); - encoding_files_queue.push(std::move(f)); - } - if (++counter%20==0 && enable_malloc_trim) malloc_trim(0); - } - ts_parser_delete(parser); - auto end=chrono::high_resolution_clock::now(); - auto ms=chrono::duration_cast(end-start).count(); - cout<<"Recccurences map thread number "<>=1; - ++bits; - } - bitstream.align(); - bitstream.write_bits(CCC_REC_TABLE_REF_HEAD,4); - bitstream.write_bits(index,bits); - bitstream.align(); - return; -} -void generate_delimiter0(bit_streamer& bitstream,size_t index) { - bitstream.align(); - bitstream.write_bits(CCC_DELIMITER_0_HEAD,1); - bitstream.write_bits(index,3); - bitstream.align(); - return; -} -void generate_delimiter1(bit_streamer& bitstream,size_t index) { - bitstream.align(); - bitstream.write_bits(CCC_DELIMITER_1_HEAD,2); - bitstream.write_bits(index,2); - bitstream.align(); - return; -} -void generate_miscellaneous(bit_streamer& bitstream,size_t index) { - bitstream.align(); - bitstream.write_bits(CCC_MISCELANEOUS_HEAD,4); - bitstream.write_bits(index,6); - bitstream.align(); -} -void generate_string_content(bit_streamer& bitstream,const char *text,size_t text_len) { - bitstream.align(); - bitstream.write_bits(CCC_STRING_INLINE_HEAD,4); - for (int i=0;isecond; - generate_rec(out,index,rec_list.size()); - } - } else if (type=="primitive_type" || type=="type_identifier") { - auto it=c_keyword_lookup.find(string(text)); - if (it!=c_keyword_lookup.end()) { - size_t index=it->second; - generate_c_keyword(out,index); - } else { - auto it=rec_lookup.find(string(text)); - if (it==rec_lookup.end()) { - if (!text.empty()) { - generate_string_content(out,text.data(),text.size()); - } else { - print_warning("Warning: type node is empty: "+string(text)); - fail_if_warning(); - } - } else { - size_t index=it->second; - generate_rec(out,index,rec_list.size()); - } - } - } else if (delimiter0_lookup.find(type)!=delimiter0_lookup.end() || delimiter1_lookup.find(type)!=delimiter1_lookup.end() || type=="\"") { - string insert; - if (type=="(" && i+1second; - generate_delimiter0(out,index); - } else { - if (insert!="{}" && insert!="\"") { - auto it=delimiter1_lookup.find(insert); - if (it!=delimiter1_lookup.end()) { - size_t index=it->second; - generate_delimiter1(out,index); - } else { - print_warning("Warning: unknow delimiter, that shouldn't happen: "+insert); - fail_if_warning(); - } - } else { - if (insert=="{}") { - auto it=delimiter1_lookup.find("{}"); - if (it!=delimiter1_lookup.end()) { - size_t index=it->second; - out.align(); - out.write_bits(CCC_DELIMITER_1_HEAD,2); - out.write_bits(index,2); - out.write_bits(0b0,1); - out.align(); - } else { - print_warning("Warning: unknow delimiter, that shouldn't happen: "+insert); - fail_if_warning(); - } - } else if (insert=="\"") { - auto it=delimiter1_lookup.find("{}"); - if (it!=delimiter1_lookup.end()) { - size_t index=it->second; - out.align(); - out.write_bits(CCC_DELIMITER_1_HEAD,2); - out.write_bits(index,2); - out.write_bits(0b1,1); - out.align(); - } else { - print_warning("Warning: unknow delimiter, that shouldn't happen: "+insert); - fail_if_warning(); - } - } else { - print_warning("Warning: unknow delimiter, that shouldn't happen: "+insert); - fail_if_warning(); - } - } - } - } else if (c_keyword_lookup.find(type)!=c_keyword_lookup.end() || type=="preproc_directive") { - if (type!="preproc_directive") { - auto it=c_keyword_lookup.find(type); - if (it!=c_keyword_lookup.end()) { - size_t index=it->second; - generate_c_keyword(out,index); - } else { - print_warning("Warning: unknow C keyword, that shouldn't happen: "+type+" "+string(text)); - fail_if_warning(); - } - } else { - auto it=c_keyword_lookup.find(string(text)); - if (it!=c_keyword_lookup.end()) { - size_t index=it->second; - generate_c_keyword(out,index); - } else { - auto it=rec_lookup.find(string(text)); - if (it==rec_lookup.end()) { - if (!text.empty()) { - generate_string_content(out,text.data(),text.size()); - } else { - print_warning("Warning: C keyword is empty: "+string(text)); - fail_if_warning(); - } - } else { - size_t index=it->second; - generate_rec(out,index,rec_list.size()); - } - } - } - } else if (miscelaneous_lookup.find(type)!=miscelaneous_lookup.end()) { - auto it=miscelaneous_lookup.find(type); - if (it!=miscelaneous_lookup.end()) { - size_t index=it->second; - generate_miscellaneous(out,index); - } else { - print_warning("Warning: unknow miscellaneous, that shouldn't happen: "+type); - fail_if_warning(); - } - } else if (type=="comment") { - auto it=rec_lookup.find(string(text)); - if (it==rec_lookup.end()) { - if (!text.empty()) { - generate_string_content(out,text.data(),text.size()); - } else { - print_warning("Warning: comment is empty: "+string(text)); - fail_if_warning(); - } - } else { - size_t index=it->second; - generate_rec(out,index,rec_list.size()); - } - } else { - auto it=rec_lookup.find(type); - if (it==rec_lookup.end()) { - if (!text.empty()) { - generate_string_content(out,text.data(),text.size()); - } else { - print_warning("Warning: unknow node is empty: "+string(text)); - fail_if_warning(); - } - } else { - size_t index=it->second; - generate_rec(out,index,rec_list.size()); - } - } - } - out.align(); - return; -} -thread_encoding_result run_thread_encoding(size_t thread_num) { - auto start=chrono::high_resolution_clock::now(); - thread_encoding_result res; - map useless_rec_map; - unordered_map thread_local_type_map; - unordered_map thread_local_type_u16_map; - vector thread_local_node_list; - vector thread_local_encoded_files; - TSParser *parser=ts_parser_new(); - ts_parser_set_language(parser,tree_sitter_c()); - int counter=0; - int max=0; - while (true) { - file_entry f; - { - lock_guard lock(encoding_queue_mutex); - if (encoding_files_queue.empty()) break; - f=std::move(encoding_files_queue.front()); - encoding_files_queue.pop(); - } - thread_iterate_input_loop_call iterate_loop_settings { - .source_code=f.content, - .thread_local_node_list=thread_local_node_list, - .thread_local_type_map=thread_local_type_map, - .thread_local_type_u16_map=thread_local_type_u16_map, - .thread_local_next_type_id=0, - .thread_local_rec_map=useless_rec_map - }; - thread_local_encoded_files.emplace_back(f.index); - thread_encoding_input_loop_call encoding_loop_settings { - .source_code=f.content, - .node_list=thread_local_node_list, - .thread_local_type_map=thread_local_type_map, - .thread_local_bit_stream=thread_local_encoded_files[counter] - }; - TSTree *tree=ts_parser_parse_string(parser,nullptr,f.content.c_str(),f.content.size()); - TSNode root=ts_tree_root_node(tree); - iterate_loop_settings.source_code=f.content; - iterate_all_nodes_loop_call(iterate_loop_settings,root,iterating_mode::PARSING); - ts_tree_delete(tree); - encoding_loop_settings.source_code=f.content; - process_file_nodes_loop_call(encoding_loop_settings); - vector().swap(thread_local_node_list); - thread_local_type_map.clear(); - thread_local_type_u16_map.clear(); - string().swap(f.content); - if (++counter%20==0 && enable_malloc_trim) malloc_trim(0); - } - ts_parser_delete(parser); - res.encoded_files=std::move(thread_local_encoded_files); - auto end=chrono::high_resolution_clock::now(); - auto ms=chrono::duration_cast(end-start).count(); - cout<<"Parsing/encoding thread number "< files; - for (int i=1;i(file)),istreambuf_iterator()); - file_entry f{files[i],std::move(code),code.size()}; - f.index=i; - rec_map_files_queue.push(std::move(f)); - } - size_t nb_threads=thread::hardware_concurrency()/2; - size_t total_files=files.size(); - size_t files_per_thread=(total_files+nb_threads-1)/nb_threads; - vector> rec_map_futures; - for (size_t i=0;i all_rec_map_results; - map global_rec_map; - for (auto& fut:rec_map_futures) { - all_rec_map_results.push_back(fut.get()); - for (auto const& [str,count]:all_rec_map_results.back().thread_local_rec_map) { - global_rec_map[str]+=count; - } - } - for (auto const& [str,count]:global_rec_map) { - if (count>=2 && str.size()>=3) { - rec_list.push_back(str); - rec_lookup[str]=rec_list.size()-1; - } - } - global_rec_map.clear(); - vector encoding_files_vec; - while (!encoding_files_queue.empty()) { - encoding_files_vec.push_back(std::move(encoding_files_queue.front())); - encoding_files_queue.pop(); - } - sort(encoding_files_vec.begin(),encoding_files_vec.end(),[](const file_entry& a,const file_entry& b) { - return a.size>b.size; - }); - for (auto& f:encoding_files_vec) { - encoding_files_queue.push(std::move(f)); - } - vector> encoding_futures; - for (size_t i=0;i all_encoding_results; - for (auto& fut:encoding_futures) { - all_encoding_results.push_back(fut.get()); - } - vector globals_bit_stream; - for (auto& res:all_encoding_results) { - globals_bit_stream.insert(globals_bit_stream.end(),res.encoded_files.begin(),res.encoded_files.end()); - } - sort(globals_bit_stream.begin(),globals_bit_stream.end(),[](const bit_streamer& a,const bit_streamer& b) { - return a.index final_payloads; - vector global_payloads_start; - size_t total_size2=0; - for(auto& bstr:globals_bit_stream) total_size2+=bstr.get_size(); - final_payloads.reserve(total_size2); - size_t current_offset=0; - for (auto& bstr:globals_bit_stream) { - global_payloads_start.push_back(current_offset); - auto encoded_file=std::move(bstr.extract_buffer()); - final_payloads.insert(final_payloads.end(),encoded_file.begin(),encoded_file.end()); - current_offset+=encoded_file.size(); - } - vector payload_compressed; - payload_compressed.resize(final_payloads.size()+final_payloads.size()/3+128); - lzma_mt mt_options={}; - mt_options.flags=0; - mt_options.threads=thread::hardware_concurrency()/4; - mt_options.block_size=max((size_t)8*1024*1024,final_payloads.size()/mt_options.threads); - mt_options.timeout=0; - mt_options.filters=nullptr; - mt_options.check=LZMA_CHECK_CRC64; - lzma_options_lzma opt_lzma; - if (lzma_lzma_preset(&opt_lzma,compression_ratio)) { - cout<<"Error: couldn't initialize LZMA compressor for files archive."<(end-start).count(); - if (ret!=LZMA_STREAM_END) { - cout<<"Error: couldn't compress files archive."<=original_size) { - flags&= ~(0b00000001); - payload_total_size=original_size; - } else { - flags|=0b00000001; - payload_total_size=compressed_size; - } - vector rec_table; - for (int i=0;i rec_table_compressed; - rec_table_compressed.resize(rec_table.size()+rec_table.size()/3+128); - strm=LZMA_STREAM_INIT; - if (lzma_easy_encoder(&strm,9,LZMA_CHECK_CRC64)!=LZMA_OK) { - cout<<"Error: couldn't initialize LZMA compressor for reccurences table."<=original_size) { - flags&= ~(0b00000010); - rec_table_total_size=original_size; - } else { - flags|=0b00000010; - rec_table_total_size=compressed_size; - } - vector files_table; - for (int i=0;i files_table_compressed; - files_table_compressed.resize(files_table.size()+files_table.size()/3+128); - strm=LZMA_STREAM_INIT; - if (lzma_easy_encoder(&strm,9,LZMA_CHECK_CRC64)!=LZMA_OK) { - cout<<"Error: couldn't initialize LZMA compressor for files table."<=original_size) { - flags&= ~(0b00000100); - files_table_total_size=original_size; - } else { - flags|=0b00000100; - files_table_total_size=compressed_size; - } - header head; - head.sig[0]='C'; - head.sig[1]='C'; - head.sig[2]='C'; - head.flags=flags; - head.size_payload=payload_total_size; - head.size_rec_table=rec_table_total_size; - head.entry_count=files.size(); - vector out; - for (int i=0;i(out.data()),out.size()); - fileout.close(); - cout<<"Finished !"<