lolo859/ccc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

whole archive start

Browse Source
This commit is contained in:
lolo859
2026-02-04 23:13:24 +01:00
parent dea8f2425e
commit f2640c70a5
9 changed files with 1377 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
all.tar.gz Normal file
View File

Binary file not shown.

2
build.sh
View File

@@ -1 +1 @@
g++ ccc.cpp -o ccc -ltree-sitter -ltree-sitter-c g++ ccc.cpp -o ccc -ltree-sitter -ltree-sitter-c -llzma

344
ccc.cpp
View File

@@ -9,6 +9,7 @@
#include <algorithm> #include <algorithm>
#include <tree_sitter/api.h> #include <tree_sitter/api.h>
#include <tree_sitter/tree-sitter-c.h> #include <tree_sitter/tree-sitter-c.h>
#include <lzma.h>
using namespace std; using namespace std;
namespace fs=filesystem; namespace fs=filesystem;
const vector<bool> CCC_C_KEYYORD_HEAD {0,0,0}; const vector<bool> CCC_C_KEYYORD_HEAD {0,0,0};
@@ -157,7 +158,9 @@ const vector<string> c_keywords={
"unsigned", "unsigned",
"void", "void",
"volatile", "volatile",
"while" "while",
"__asm__",
"__attribute__"
}; };
struct symbol { struct symbol {
string name; string name;
@@ -177,10 +180,19 @@ void insert(node* root,string str,int id) {
} }
curr->token_id=id; curr->token_id=id;
} }
vector<TSNode> all_tokens; struct processed_file {
void get_all_nodes(TSNode node,const string &source_code,map<string,int> &rec_map) { string path;
uint32_t payload_size;
vector<unsigned char> payload;
bool is_payload_compressed;
};
map<string,vector<TSNode>> all_tokens;
map<string,int> rec_map;
vector<string> rec_list;
bool debug=false;
void get_all_nodes(TSNode node,const string &source_code,map<string,int> &rec_map,const string& file) {
if (ts_node_child_count(node)==0) { if (ts_node_child_count(node)==0) {
all_tokens.push_back(node); all_tokens[file].push_back(node);
string text=source_code.substr(ts_node_start_byte(node),ts_node_end_byte(node)-ts_node_start_byte(node)); string text=source_code.substr(ts_node_start_byte(node),ts_node_end_byte(node)-ts_node_start_byte(node));
if (string(ts_node_type(node))=="string_content" || string(ts_node_type(node))=="system_lib_string" || string(ts_node_type(node))=="identifier" || string(ts_node_type(node))=="number_literal" || string(ts_node_type(node))=="type_identifier" || string(ts_node_type(node))=="field_identifier" || string(ts_node_type(node))=="escape_sequence" || string(ts_node_type(node))=="statement_identifier") { if (string(ts_node_type(node))=="string_content" || string(ts_node_type(node))=="system_lib_string" || string(ts_node_type(node))=="identifier" || string(ts_node_type(node))=="number_literal" || string(ts_node_type(node))=="type_identifier" || string(ts_node_type(node))=="field_identifier" || string(ts_node_type(node))=="escape_sequence" || string(ts_node_type(node))=="statement_identifier") {
rec_map[text]++; rec_map[text]++;
@@ -188,11 +200,14 @@ void get_all_nodes(TSNode node,const string &source_code,map<string,int> &rec_ma
if (string(ts_node_type(node))=="primitive_type" && find(c_keywords.begin(),c_keywords.end(),text)==c_keywords.end()) { if (string(ts_node_type(node))=="primitive_type" && find(c_keywords.begin(),c_keywords.end(),text)==c_keywords.end()) {
rec_map[text]++; rec_map[text]++;
} }
if (string(ts_node_type(node))=="comment") {
rec_map[text]=2;
}
} else { } else {
uint32_t child_count=ts_node_child_count(node); uint32_t child_count=ts_node_child_count(node);
for (uint32_t i=0;i<child_count;++i) { for (uint32_t i=0;i<child_count;++i) {
TSNode child=ts_node_child(node,i); TSNode child=ts_node_child(node,i);
get_all_nodes(child,source_code,rec_map); get_all_nodes(child,source_code,rec_map,file);
} }
} }
} }
@@ -286,55 +301,60 @@ vector<bool> generate_string_content(string str) {
} }
return out; return out;
} }
vector<bool> process_all_nodes(vector<TSNode> *nodes,string code,vector<string> &rec_list) { void print_debug(string text) {
if (debug==true) {
cout<<text<<endl;
}
}
vector<unsigned char> process_file_nodes(vector<TSNode> *nodes,string code,vector<string> &rec_list) {
vector<bool> out; vector<bool> out;
for (int i=0;i<nodes->size();i++) { for (int i=0;i<nodes->size();i++) {
string type=string(ts_node_type(nodes->at(i))); string type=string(ts_node_type(nodes->at(i)));
if (type=="#if") { if (type=="#if") {
CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_CONDITIONAL_IF); CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_CONDITIONAL_IF);
cout<<"if"<<endl; print_debug("if");
} else if (type=="#ifdef") { } else if (type=="#ifdef") {
CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_CONDITIONAL_IFDEF); CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_CONDITIONAL_IFDEF);
cout<<"ifdef"<<endl; print_debug("ifdef");
} else if (type=="#ifndef") { } else if (type=="#ifndef") {
CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_CONDITIONAL_IFNDEF); CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_CONDITIONAL_IFNDEF);
cout<<"ifndef"<<endl; print_debug("ifndef");
} else if (type=="#else") { } else if (type=="#else") {
CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_CONDITIONAL_ELSE); CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_CONDITIONAL_ELSE);
cout<<"else"<<endl; print_debug("else");
} else if (type=="#elif") { } else if (type=="#elif") {
CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_CONDITIONAL_ELIF); CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_CONDITIONAL_ELIF);
cout<<"elif"<<endl; print_debug("elif");
} else if (type=="#elifdef") { } else if (type=="#elifdef") {
CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_CONDITIONAL_ELIFDEF); CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_CONDITIONAL_ELIFDEF);
cout<<"elifdef"<<endl; print_debug("elifdef");
} else if (type=="#elifndef") { } else if (type=="#elifndef") {
CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_CONDITIONAL_ELIFNDEF); CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_CONDITIONAL_ELIFNDEF);
cout<<"elifndef"<<endl; print_debug("elifndef");
} else if (type=="#endif") { } else if (type=="#endif") {
CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_CONDITIONAL_ENDIF); CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_CONDITIONAL_ENDIF);
cout<<"endif"<<endl; print_debug("endif");
} else if (type=="#define") { } else if (type=="#define") {
CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_OTHER_DEFINE); CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_OTHER_DEFINE);
cout<<"define"<<endl; print_debug("define");
} else if (type=="#undef") { } else if (type=="#undef") {
CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_OTHER_UNDEF); CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_OTHER_UNDEF);
cout<<"undef"<<endl; print_debug("undef");
} else if (type=="#include") { } else if (type=="#include") {
CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_OTHER_INCLUDE); CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_OTHER_INCLUDE);
cout<<"include"<<endl; print_debug("include");
} else if (type=="#error") { } else if (type=="#error") {
CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_OTHER_ERROR); CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_OTHER_ERROR);
cout<<"error"<<endl; print_debug("error");
} else if (type=="#warning") { } else if (type=="#warning") {
CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_OTHER_WARNING); CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_OTHER_WARNING);
cout<<"warning"<<endl; print_debug("warning");
} else if (type=="#pragma") { } else if (type=="#pragma") {
CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_OTHER_PRAGMA); CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_OTHER_PRAGMA);
cout<<"pragma"<<endl; print_debug("pragma");
} else if (type=="#line") { } else if (type=="#line") {
CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_OTHER_LINE); CCC_ADD_COMPOMENT(out,CCC_PREPROCESSOR_OTHER_LINE);
cout<<"line"<<endl; print_debug("line");
} else if (type=="string_content" || type=="system_lib_string" || type=="identifier" || type=="number_literal" || type=="type_identifier" || type=="field_identifier" || type=="preproc_arg" || type=="escape_sequence" || type=="character" || type=="statement_identifier") { } else if (type=="string_content" || type=="system_lib_string" || type=="identifier" || type=="number_literal" || type=="type_identifier" || type=="field_identifier" || type=="preproc_arg" || type=="escape_sequence" || type=="character" || type=="statement_identifier") {
string text=code.substr(ts_node_start_byte(nodes->at(i)),ts_node_end_byte(nodes->at(i))-ts_node_start_byte(nodes->at(i))); string text=code.substr(ts_node_start_byte(nodes->at(i)),ts_node_end_byte(nodes->at(i))-ts_node_start_byte(nodes->at(i)));
auto it=find(rec_list.begin(),rec_list.end(),text); auto it=find(rec_list.begin(),rec_list.end(),text);
@@ -342,17 +362,17 @@ vector<bool> process_all_nodes(vector<TSNode> *nodes,string code,vector<string>
if (!text.empty()) { if (!text.empty()) {
string text=code.substr(ts_node_start_byte(nodes->at(i)),ts_node_end_byte(nodes->at(i))-ts_node_start_byte(nodes->at(i))); string text=code.substr(ts_node_start_byte(nodes->at(i)),ts_node_end_byte(nodes->at(i))-ts_node_start_byte(nodes->at(i)));
CCC_ADD_COMPOMENT(out,generate_string_content(text)); CCC_ADD_COMPOMENT(out,generate_string_content(text));
cout<<"string ("<<type<<"): "<<text<<endl; print_debug("string ("+type+"): "+text);
} else { } else {
auto it=find(delimiter.begin(),delimiter.end(),""); auto it=find(delimiter.begin(),delimiter.end(),"");
size_t index=distance(delimiter.begin(),it); size_t index=distance(delimiter.begin(),it);
CCC_ADD_COMPOMENT(out,generate_delimiter(index)); CCC_ADD_COMPOMENT(out,generate_delimiter(index));
cout<<"delimiter for empty string"<<endl; print_debug("delimiter for empty string");
} }
} else { } else {
size_t index=distance(rec_list.begin(),it); size_t index=distance(rec_list.begin(),it);
CCC_ADD_COMPOMENT(out,generate_rec(index,rec_list.size())); CCC_ADD_COMPOMENT(out,generate_rec(index,rec_list.size()));
cout<<"rec_table for string ("<<type<<"): "<<text<<endl; print_debug("rec_table for string ("+type+"): "+text);
} }
} else if (type=="primitive_type") { } else if (type=="primitive_type") {
string text=code.substr(ts_node_start_byte(nodes->at(i)),ts_node_end_byte(nodes->at(i))-ts_node_start_byte(nodes->at(i))); string text=code.substr(ts_node_start_byte(nodes->at(i)),ts_node_end_byte(nodes->at(i))-ts_node_start_byte(nodes->at(i)));
@@ -360,21 +380,21 @@ vector<bool> process_all_nodes(vector<TSNode> *nodes,string code,vector<string>
if (it!=c_keywords.end()) { if (it!=c_keywords.end()) {
size_t index=distance(c_keywords.begin(),it); size_t index=distance(c_keywords.begin(),it);
CCC_ADD_COMPOMENT(out,generate_c_keyword(index)); CCC_ADD_COMPOMENT(out,generate_c_keyword(index));
cout<<"primitive_type: "<<text<<endl; print_debug("primitive_type: "+text);
} else { } else {
auto it=find(rec_list.begin(),rec_list.end(),text); auto it=find(rec_list.begin(),rec_list.end(),text);
if (it==rec_list.end()) { if (it==rec_list.end()) {
if (!text.empty()) { if (!text.empty()) {
CCC_ADD_COMPOMENT(out,generate_string_content(text)); CCC_ADD_COMPOMENT(out,generate_string_content(text));
cout<<"string ("<<type<<"): "<<text<<endl; print_debug("string ("+type+"): "+text);
} else { } else {
cout<<"Error: provided primitive is empty: "<<text; cout<<"Error: provided primitive is empty: "<<text<<endl;;
exit(-1); exit(-1);
} }
} else { } else {
size_t index=distance(rec_list.begin(),it); size_t index=distance(rec_list.begin(),it);
CCC_ADD_COMPOMENT(out,generate_rec(index,rec_list.size())); CCC_ADD_COMPOMENT(out,generate_rec(index,rec_list.size()));
cout<<"rec_table for string ("<<type<<"): "<<text<<endl; print_debug("rec_table for string ("+type+"): "+text);
} }
} }
} else if (find(delimiter.begin(),delimiter.end(),type)!=delimiter.end()) { } else if (find(delimiter.begin(),delimiter.end(),type)!=delimiter.end()) {
@@ -407,7 +427,7 @@ vector<bool> process_all_nodes(vector<TSNode> *nodes,string code,vector<string>
if (it!=delimiter.end()) { if (it!=delimiter.end()) {
size_t index=distance(delimiter.begin(),it); size_t index=distance(delimiter.begin(),it);
CCC_ADD_COMPOMENT(out,generate_delimiter(index)); CCC_ADD_COMPOMENT(out,generate_delimiter(index));
cout<<"delimiter: "<<text<<endl; print_debug("delimiter: "+text);
} else { } else {
cout<<"Error: unknow delimiter, that shouldn't happen: "<<text<<endl;; cout<<"Error: unknow delimiter, that shouldn't happen: "<<text<<endl;;
exit(-1); exit(-1);
@@ -417,7 +437,7 @@ vector<bool> process_all_nodes(vector<TSNode> *nodes,string code,vector<string>
if (it!=other_grammer.end()) { if (it!=other_grammer.end()) {
size_t index=distance(other_grammer.begin(),it); size_t index=distance(other_grammer.begin(),it);
CCC_ADD_COMPOMENT(out,generate_other_grammar(index)); CCC_ADD_COMPOMENT(out,generate_other_grammar(index));
cout<<"other grammar: "<<type<<endl; print_debug("other grammar: "+type);
} else { } else {
cout<<"Error: unknow other grammar symbol, that shouldn't happen: "<<type<<endl;; cout<<"Error: unknow other grammar symbol, that shouldn't happen: "<<type<<endl;;
exit(-1); exit(-1);
@@ -427,7 +447,7 @@ vector<bool> process_all_nodes(vector<TSNode> *nodes,string code,vector<string>
if (it!=c_keywords.end()) { if (it!=c_keywords.end()) {
size_t index=distance(c_keywords.begin(),it); size_t index=distance(c_keywords.begin(),it);
CCC_ADD_COMPOMENT(out,generate_c_keyword(index)); CCC_ADD_COMPOMENT(out,generate_c_keyword(index));
cout<<"c keyword: "<<type<<endl; print_debug("c keyword: "+type);
} else { } else {
cout<<"Error: unknow C keyword, that shouldn't happen: "<<type<<endl;; cout<<"Error: unknow C keyword, that shouldn't happen: "<<type<<endl;;
exit(-1); exit(-1);
@@ -437,111 +457,227 @@ vector<bool> process_all_nodes(vector<TSNode> *nodes,string code,vector<string>
if (it!=miscellaneous.end()) { if (it!=miscellaneous.end()) {
size_t index=distance(miscellaneous.begin(),it); size_t index=distance(miscellaneous.begin(),it);
CCC_ADD_COMPOMENT(out,generate_miscellaneous(index)); CCC_ADD_COMPOMENT(out,generate_miscellaneous(index));
cout<<"miscellaneous: "<<type<<endl; print_debug("miscellaneous: "+type);
} else { } else {
cout<<"Error: unknow miscellaneous, that shouldn't happen: "<<type<<endl;; cout<<"Error: unknow miscellaneous, that shouldn't happen: "<<type<<endl;;
exit(-1); exit(-1);
} }
} else if (type=="comment") {
string text=code.substr(ts_node_start_byte(nodes->at(i)),ts_node_end_byte(nodes->at(i))-ts_node_start_byte(nodes->at(i)));
auto it=find(rec_list.begin(),rec_list.end(),text);
if (it==rec_list.end()) {
cout<<"Error: comment in reccurences map not found: "<<text<<endl;;
exit(-1);
} else {
size_t index=distance(rec_list.begin(),it);
CCC_ADD_COMPOMENT(out,generate_rec(index,rec_list.size()));
print_debug("rec_table for comment");
}
} else if (type=="\"") { } else if (type=="\"") {
if (i+1<nodes->size()) { if (i+1<nodes->size()) {
if (string(ts_node_type(nodes->at(i+1)))=="\"") { if (string(ts_node_type(nodes->at(i+1)))=="\"") {
auto it=find(delimiter.begin(),delimiter.end(),""); auto it=find(delimiter.begin(),delimiter.end(),"");
size_t index=distance(delimiter.begin(),it); size_t index=distance(delimiter.begin(),it);
CCC_ADD_COMPOMENT(out,generate_delimiter(index)); CCC_ADD_COMPOMENT(out,generate_delimiter(index));
cout<<"double quotes mark, inserting delimiter for empty string"<<endl; print_debug("double quotes mark, inserting delimiter for empty string");
i++; i++;
} else { } else {
CCC_ADD_COMPOMENT(out,CCC_QUOTE); CCC_ADD_COMPOMENT(out,CCC_QUOTE);
cout<<"single quote mark"<<endl; print_debug("single quote mark");
} }
} }
} else if (type=="comment") {
continue;
} else { } else {
string text=code.substr(ts_node_start_byte(nodes->at(i)),ts_node_end_byte(nodes->at(i))-ts_node_start_byte(nodes->at(i))); string text=code.substr(ts_node_start_byte(nodes->at(i)),ts_node_end_byte(nodes->at(i))-ts_node_start_byte(nodes->at(i)));
cout<<"unknow node type: "<<type<<endl; cout<<"Error: unknow node type: "<<type<<endl;
cout<<"unknow node text: "<<text<<endl; cout<<"Error: unknow node text: "<<text<<endl;
exit(-1); exit(-1);
} }
} }
return out; vector<unsigned char> payload_bytes;
}
int main(int argc,char **argv) {
if (argc!=2) {
cout<<"Usage: ccc <c file>"<<endl;
return -1;
}
string filepath=string(argv[1]);
if (!fs::exists(filepath)) {
cout<<"Error: provided file doesn't exist."<<endl;
return -1;
}
ifstream file(filepath,ios::binary);
if (!file) {
cout<<"Error: couldn't open provided file."<<endl;
return -1;
}
string code((istreambuf_iterator<char>(file)),istreambuf_iterator<char>());
TSParser *parser=ts_parser_new();
ts_parser_set_language(parser,tree_sitter_c());
TSTree *tree=ts_parser_parse_string(parser,nullptr,code.c_str(),code.size());
TSNode root=ts_tree_root_node(tree);
map<string,int> rec_map;
vector<string> rec_list;
get_all_nodes(root,code,rec_map);
for (auto s:rec_map) {
if (s.second>=2 and s.first.size()>=3 && s.first.size()<=256) {
rec_list.push_back(s.first);
}
}
auto payload=process_all_nodes(&all_tokens,code,rec_list);
vector<bool> out={0,1,0,0,0,0,1,1,0,1,0,0,0,0,1,1,0,1,0,0,0,0,1,1};
for (int i=63;i>=0;i--) {
bool enabled=(rec_list.size()>>i)&0x01;
out.push_back(enabled);
}
for (int i=0;i<rec_list.size();i++) {
uint8_t size=(uint8_t)rec_list[i].size();
for (int i=7;i>=0;i--) {
bool enabled=(size>>i)&0x01;
out.push_back(enabled);
}
for (auto c:rec_list[i]) {
for (int i=7;i>=0;i--) {
bool enabled=(c>>i)&0x01;
out.push_back(enabled);
}
}
}
CCC_ADD_COMPOMENT(out,payload);
vector<unsigned char> outbytes;
unsigned char current=0; unsigned char current=0;
int bit_index=0; size_t bit_index=0;
for (bool b:out) { for (bool b:out) {
current|=(b<<(7-bit_index)); current|=(b<<(7-bit_index));
bit_index++; bit_index++;
if (bit_index==8) { if (bit_index==8) {
outbytes.push_back(current); payload_bytes.push_back(current);
current=0; current=0;
bit_index=0; bit_index=0;
} }
} }
if (bit_index!=0) { if (bit_index!=0) {
outbytes.push_back(current); payload_bytes.push_back(current);
} }
ofstream fileout(filepath+".ccc",ios::binary); return payload_bytes;
if (!fileout) { }
cout<<"Error: couldn't open output file."<<endl; void construct_rec_table(vector<string> &files_content,vector<string> files_names) {
for (int i=0;i<files_content.size();i++) {
TSParser *parser=ts_parser_new();
ts_parser_set_language(parser,tree_sitter_c());
TSTree *tree=ts_parser_parse_string(parser,nullptr,files_content[i].c_str(),files_content[i].size());
TSNode root=ts_tree_root_node(tree);
get_all_nodes(root,files_content[i],rec_map,files_names[i]);
}
for (auto s:rec_map) {
if (s.second>=2 and s.first.size()>=3) {
rec_list.push_back(s.first);
}
}
}
int main(int argc,char **argv) {
if (argc<2) {
cout<<"Usage: ccc [FILES]"<<endl;
return -1; return -1;
} }
fileout.write(reinterpret_cast<const char*>(outbytes.data()),outbytes.size()); vector<string> files;
fileout.close(); for (int i=1;i<argc;i++) {
cout<<"Reccurences map entry count: "<<rec_list.size()<<endl; string file=string(argv[i]);
size_t total_bytes=0; if (file=="-v") {
for (int i=0;i<rec_list.size();i++) { debug=true;
total_bytes++; continue;
total_bytes+=rec_list[i].size();
} }
cout<<"Total spaces taken by reccurences map in bytes: "<<total_bytes<<endl; if (!fs::exists(file)) {
return 0; cout<<"Error: file doesn't exist: "<<file<<endl;
return -1;
}
files.push_back(file);
}
vector<string> files_content;
for (auto f:files) {
ifstream file(f,ios::binary);
if (!file) {
cout<<"Error: couldn't open provided file."<<endl;
return -1;
}
string code((istreambuf_iterator<char>(file)),istreambuf_iterator<char>());
files_content.push_back(code);
}
construct_rec_table(files_content,files);
vector<processed_file> files_archive;
for (int i=0;i<files_content.size();i++) {
processed_file pfile;
pfile.path=files[i];
auto payload_bytes=process_file_nodes(&(all_tokens.at(pfile.path)),files_content[i],rec_list);
vector<unsigned char> payload_compressed;
payload_compressed.resize(payload_bytes.size()+payload_bytes.size()/3+128);
lzma_stream strm=LZMA_STREAM_INIT;
if (lzma_easy_encoder(&strm,9,LZMA_CHECK_CRC64)!=LZMA_OK) {
cout<<"Error: couldn't initialize LZMA compressor for file: "<<files[i]<<endl;
return -1;
}
strm.next_in=payload_bytes.data();
strm.avail_in=payload_bytes.size();
strm.next_out=payload_compressed.data();
strm.avail_out=payload_compressed.size();
auto ret=lzma_code(&strm,LZMA_FINISH);
if (ret!=LZMA_STREAM_END) {
cout<<"Error: couldn't compress payload for file: "<<files[i]<<endl;
return -1;
}
size_t compressed_size=payload_compressed.size()-strm.avail_out;
payload_compressed.resize(compressed_size);
size_t original_size=payload_bytes.size();
lzma_end(&strm);
if (compressed_size>=original_size) {
pfile.is_payload_compressed=false;
pfile.payload=payload_bytes;
pfile.payload_size=original_size;
} else {
pfile.is_payload_compressed=true;
pfile.payload=payload_compressed;
pfile.payload_size=compressed_size;
}
cout<<i+1<<" file(s) done on "<<files.size()<<": "<<files[i]<<endl;
}
exit(0);
// auto payload=process_all_nodes(&all_tokens[0],files_content[0],rec_list);
// vector<unsigned char> rec_table;
// for (int i=0;i<rec_list.size();i++) {
// for (auto c:rec_list[i]) {
// rec_table.push_back(c);
// }
// rec_table.push_back('\0');
// }
// vector<unsigned char> rec_table_compressed;
// rec_table_compressed.resize(rec_table.size()+rec_table.size()/3+128);
// lzma_stream strm=LZMA_STREAM_INIT;
// if (lzma_easy_encoder(&strm,9,LZMA_CHECK_CRC64)!=LZMA_OK) {
// cout<<"Error: couldn't initialize LZMA compressor."<<endl;
// return -1;
// }
// strm.next_in=rec_table.data();
// strm.avail_in=rec_table.size();
// strm.next_out=rec_table_compressed.data();
// strm.avail_out=rec_table_compressed.size();
// lzma_ret ret=lzma_code(&strm,LZMA_FINISH);
// if (ret!=LZMA_STREAM_END) {
// cout<<"Error: couldn't compress reccurences table."<<endl;
// return -1;
// }
// size_t compressed_size=rec_table_compressed.size()-strm.avail_out;
// rec_table_compressed.resize(compressed_size);
// size_t original_size=rec_table.size();
// lzma_end(&strm);
// vector<unsigned char> out;
// out.push_back('C');
// out.push_back(compressed_size>=original_size?'C':'c');
// vector<unsigned char> payload_bytes;
// unsigned char current=0;
// size_t bit_index=0;
// for (bool b:payload) {
// current|=(b<<(7-bit_index));
// bit_index++;
// if (bit_index==8) {
// payload_bytes.push_back(current);
// current=0;
// bit_index=0;
// }
// }
// if (bit_index!=0) {
// payload_bytes.push_back(current);
// }
// vector<unsigned char> payload_compressed;
// payload_compressed.resize(payload_bytes.size()+payload_bytes.size()/3+128);
// strm=LZMA_STREAM_INIT;
// if (lzma_easy_encoder(&strm,9,LZMA_CHECK_CRC64)!=LZMA_OK) {
// cout<<"Error: couldn't initialize LZMA compressor."<<endl;
// return -1;
// }
// strm.next_in=payload_bytes.data();
// strm.avail_in=payload_bytes.size();
// strm.next_out=payload_compressed.data();
// strm.avail_out=payload_compressed.size();
// ret=lzma_code(&strm,LZMA_FINISH);
// if (ret!=LZMA_STREAM_END) {
// cout<<"Error: couldn't compress reccurences table."<<endl;
// return -1;
// }
// size_t compressed_size1=payload_compressed.size()-strm.avail_out;
// payload_compressed.resize(compressed_size1);
// size_t original_size1=payload_bytes.size();
// lzma_end(&strm);
// out.push_back(compressed_size1>=original_size1?'C':'c');
// if (compressed_size>=original_size) {
// CCC_ADD_COMPOMENT(out,rec_table);
// } else {
// CCC_ADD_COMPOMENT(out,rec_table_compressed);
// }
// if (compressed_size1>=original_size1) {
// CCC_ADD_COMPOMENT(out,payload_bytes);
// } else {
// CCC_ADD_COMPOMENT(out,payload_compressed);
// }
// ofstream fileout("test.ccc",ios::binary);
// if (!fileout) {
// cout<<"Error: couldn't open output file."<<endl;
// return -1;
// }
// fileout.write(reinterpret_cast<const char*>(out.data()),out.size());
// fileout.close();
// return 0;
} }

BIN
ccc.cpp.ccc
View File

Binary file not shown.

6
hello.c
View File

@@ -1,6 +1,12 @@
#include <stdio.h> #include <stdio.h>
#include <stdint.h> #include <stdint.h>
typedef static unsigned char HEY; typedef static unsigned char HEY;
// hello
// hello
// hello
// hello
// hello
// hello
int main() { int main() {
hello[]="hello"; hello[]="hello";
HEY res=8; HEY res=8;

BIN
hello.c.ccc
View File

Binary file not shown.

409
log.c Normal file
View File

@@ -0,0 +1,409 @@
#include "../include/log.h"
#include "../include/page.h"
sh_uint8 kernel_log_level=0;
static inline sh_uint8 inb(sh_uint16 port) {
sh_uint8 val;
__asm__ volatile ("inb %1, %0":"=a"(val):"Nd"(port));
return val;
}
static inline void outb(sh_uint16 port,sh_uint8 val) {
__asm__ volatile ("outb %0, %1"::"a"(val),"Nd"(port));
}
SH_STATUS sh_log_send_byte(sh_uint8 b) {
while (!(inb(SH_LOG_SERIAL_PORT_COM1+5) & 0x20));
outb(SH_LOG_SERIAL_PORT_COM1,b);
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_send_string(const char* str) {
while (*str) {
sh_log_send_byte(*str++);
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_send_uintn(sh_uint64 n) {
char buf[20];
sh_uint32 i=0;
if (n==0) {
sh_log_send_byte('0');
return SH_STATUS_SUCCESS;
}
while (n>0) {
buf[i++]='0'+(n%10);
n/=10;
}
while (i--) {
sh_log_send_byte(buf[i]);
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_send_uintn_hex(sh_uint64 n) {
char buf[16];
sh_uint32 i=0;
const char hex_digits[]="0123456789ABCDEF";
if (n==0) {
sh_log_send_byte('0');
return SH_STATUS_SUCCESS;
}
while (n>0) {
buf[i++]=hex_digits[n & 0xF];
n>>=4;
}
while (i--) {
sh_log_send_byte(buf[i]);
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_send_double(double value) {
if (value<0) {
sh_log_send_byte('-');
value=-value;
}
sh_uint64 integer_part=(sh_uint64)value;
double fractional_part=value-(double)integer_part;
SH_STATUS status=sh_log_send_uintn(integer_part);
if (status!=SH_STATUS_SUCCESS) return status;
status=sh_log_send_byte('.');
if (status!=SH_STATUS_SUCCESS) return status;
for (int i=0;i<6;i++) {
fractional_part*=10.0;
sh_uint64 digit=(sh_uint64)fractional_part;
status=sh_log_send_byte('0'+(sh_uint8)digit);
if (status!=SH_STATUS_SUCCESS) return status;
fractional_part-=(double)digit;
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_send_payload(sh_log_OUTPUT_PAYLOAD *payload) {
if (payload==SH_NULLPTR) {
return SH_STATUS_INVALID_PARAMETER;
}
if (!(sh_log_output_type_valid(payload->output_type))) {
return SH_STATUS_INVALID_PARAMETER;
}
if (!(sh_log_output_source_valid(payload->output_source))) {
return SH_STATUS_INVALID_PARAMETER;
}
if (payload->output_source!=SH_LOG_SOURCE_TEST && payload->output_type<kernel_log_level) {
return SH_STATUS_SUCCESS;
}
sh_log_send_string("[Shelter:");
if (payload->output_source==SH_LOG_SOURCE_MAIN) {
sh_log_send_string("Main@");
} else if (payload->output_source==SH_LOG_SOURCE_CONF) {
sh_log_send_string("Conf@");
} else if (payload->output_source==SH_LOG_SOURCE_PAGE) {
sh_log_send_string("Page@");
} else if (payload->output_source==SH_LOG_SOURCE_SLAB) {
sh_log_send_string("Slab@");
} else if (payload->output_source==SH_LOG_SOURCE_TEST) {
sh_log_send_string("Test@");
}
if (payload->output_type==SH_LOG_DEBUG) {
sh_log_send_string("Debug] ");
} else if (payload->output_type==SH_LOG_LOG) {
sh_log_send_string("Log] ");
} else if (payload->output_type==SH_LOG_WARNING) {
sh_log_send_string("Warning] ");
} else if (payload->output_type==SH_LOG_ERROR) {
sh_log_send_string("Error] ");
} else if (payload->output_type==SH_LOG_CRITICAL) {
sh_log_send_string("Critical] ");
} else if (payload->output_type==SH_LOG_FATAL) {
sh_log_send_string("Fatal] ");
} else if (payload->output_type==SH_LOG_TEST) {
sh_log_send_string("Test] ");
}
sh_log_send_uintn(payload->tsc_value);
sh_log_send_string(" : ");
sh_log_send_string(payload->message_pointer);
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_load_log_level(sh_uint8 log_level) {
kernel_log_level=log_level;
return SH_STATUS_SUCCESS;
}
sh_uint8 sh_log_get_log_level() {
return kernel_log_level;
}
SH_STATUS sh_log_test(const char* str) {
if (str==SH_NULLPTR) {
return SH_STATUS_INVALID_PARAMETER;
}
sh_log_OUTPUT_PAYLOAD payload={
.output_type=SH_LOG_TEST,
.output_source=SH_LOG_SOURCE_TEST,
.tsc_value=sh_tsc_get_kernel_current_tsc(),
.message_pointer=str
};
SH_STATUS status=sh_log_send_payload(&payload);
if (sh_status_error(status)) {
return status;
}
sh_log_send_string("\n");
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_debug(const char* str,sh_log_OUTPUT_SOURCE source) {
if (str==SH_NULLPTR || !(sh_log_output_source_valid(source))) {
return SH_STATUS_INVALID_PARAMETER;
}
sh_log_OUTPUT_PAYLOAD payload={
.output_type=SH_LOG_DEBUG,
.output_source=source,
.tsc_value=sh_tsc_get_kernel_current_tsc(),
.message_pointer=str
};
SH_STATUS status=sh_log_send_payload(&payload);
if (sh_status_error(status)) {
return status;
}
sh_log_send_string("\n");
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_log(const char* str,sh_log_OUTPUT_SOURCE source) {
if (str==SH_NULLPTR || !(sh_log_output_source_valid(source))) {
return SH_STATUS_INVALID_PARAMETER;
}
sh_log_OUTPUT_PAYLOAD payload={
.output_type=SH_LOG_LOG,
.output_source=source,
.tsc_value=sh_tsc_get_kernel_current_tsc(),
.message_pointer=str
};
SH_STATUS status=sh_log_send_payload(&payload);
if (sh_status_error(status)) {
return status;
}
sh_log_send_string("\n");
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_warning(const char* str,sh_log_OUTPUT_SOURCE source) {
if (str==SH_NULLPTR || !(sh_log_output_source_valid(source))) {
return SH_STATUS_INVALID_PARAMETER;
}
sh_log_OUTPUT_PAYLOAD payload={
.output_type=SH_LOG_WARNING,
.output_source=source,
.tsc_value=sh_tsc_get_kernel_current_tsc(),
.message_pointer=str
};
SH_STATUS status=sh_log_send_payload(&payload);
if (sh_status_error(status)) {
return status;
}
sh_log_send_string("\n");
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_error(const char* str,sh_log_OUTPUT_SOURCE source) {
if (str==SH_NULLPTR || !(sh_log_output_source_valid(source))) {
return SH_STATUS_INVALID_PARAMETER;
}
sh_log_OUTPUT_PAYLOAD payload={
.output_type=SH_LOG_ERROR,
.output_source=source,
.tsc_value=sh_tsc_get_kernel_current_tsc(),
.message_pointer=str
};
SH_STATUS status=sh_log_send_payload(&payload);
if (sh_status_error(status)) {
return status;
}
sh_log_send_string("\n");
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_critical(const char* str,sh_log_OUTPUT_SOURCE source) {
if (str==SH_NULLPTR || !(sh_log_output_source_valid(source))) {
return SH_STATUS_INVALID_PARAMETER;
}
sh_log_OUTPUT_PAYLOAD payload={
.output_type=SH_LOG_CRITICAL,
.output_source=source,
.tsc_value=sh_tsc_get_kernel_current_tsc(),
.message_pointer=str
};
SH_STATUS status=sh_log_send_payload(&payload);
if (sh_status_error(status)) {
return status;
}
sh_log_send_string("\n");
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_fatal(const char* str,sh_log_OUTPUT_SOURCE source) {
if (str==SH_NULLPTR || !(sh_log_output_source_valid(source))) {
return SH_STATUS_INVALID_PARAMETER;
}
sh_log_OUTPUT_PAYLOAD payload={
.output_type=SH_LOG_FATAL,
.output_source=source,
.tsc_value=sh_tsc_get_kernel_current_tsc(),
.message_pointer=str
};
SH_STATUS status=sh_log_send_payload(&payload);
if (sh_status_error(status)) {
return status;
}
sh_log_send_string("\n");
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_ltest(const char* str) {
if (str==SH_NULLPTR) {
return SH_STATUS_INVALID_PARAMETER;
}
sh_log_OUTPUT_PAYLOAD payload={
.output_type=SH_LOG_TEST,
.output_source=SH_LOG_SOURCE_TEST,
.tsc_value=sh_tsc_get_kernel_current_tsc(),
.message_pointer=str
};
SH_STATUS status=sh_log_send_payload(&payload);
if (sh_status_error(status)) {
return status;
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_ldebug(const char* str,sh_log_OUTPUT_SOURCE source) {
if (str==SH_NULLPTR || !(sh_log_output_source_valid(source))) {
return SH_STATUS_INVALID_PARAMETER;
}
sh_log_OUTPUT_PAYLOAD payload={
.output_type=SH_LOG_DEBUG,
.output_source=source,
.tsc_value=sh_tsc_get_kernel_current_tsc(),
.message_pointer=str
};
SH_STATUS status=sh_log_send_payload(&payload);
if (sh_status_error(status)) {
return status;
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_llog(const char* str,sh_log_OUTPUT_SOURCE source) {
if (str==SH_NULLPTR || !(sh_log_output_source_valid(source))) {
return SH_STATUS_INVALID_PARAMETER;
}
sh_log_OUTPUT_PAYLOAD payload={
.output_type=SH_LOG_LOG,
.output_source=source,
.tsc_value=sh_tsc_get_kernel_current_tsc(),
.message_pointer=str
};
SH_STATUS status=sh_log_send_payload(&payload);
if (sh_status_error(status)) {
return status;
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_lwarning(const char* str,sh_log_OUTPUT_SOURCE source) {
if (str==SH_NULLPTR || !(sh_log_output_source_valid(source))) {
return SH_STATUS_INVALID_PARAMETER;
}
sh_log_OUTPUT_PAYLOAD payload={
.output_type=SH_LOG_WARNING,
.output_source=source,
.tsc_value=sh_tsc_get_kernel_current_tsc(),
.message_pointer=str
};
SH_STATUS status=sh_log_send_payload(&payload);
if (sh_status_error(status)) {
return status;
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_lerror(const char* str,sh_log_OUTPUT_SOURCE source) {
if (str==SH_NULLPTR || !(sh_log_output_source_valid(source))) {
return SH_STATUS_INVALID_PARAMETER;
}
sh_log_OUTPUT_PAYLOAD payload={
.output_type=SH_LOG_ERROR,
.output_source=source,
.tsc_value=sh_tsc_get_kernel_current_tsc(),
.message_pointer=str
};
SH_STATUS status=sh_log_send_payload(&payload);
if (sh_status_error(status)) {
return status;
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_lcritical(const char* str,sh_log_OUTPUT_SOURCE source) {
if (str==SH_NULLPTR || !(sh_log_output_source_valid(source))) {
return SH_STATUS_INVALID_PARAMETER;
}
sh_log_OUTPUT_PAYLOAD payload={
.output_type=SH_LOG_CRITICAL,
.output_source=source,
.tsc_value=sh_tsc_get_kernel_current_tsc(),
.message_pointer=str
};
SH_STATUS status=sh_log_send_payload(&payload);
if (sh_status_error(status)) {
return status;
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_lfatal(const char* str,sh_log_OUTPUT_SOURCE source) {
if (str==SH_NULLPTR || !(sh_log_output_source_valid(source))) {
return SH_STATUS_INVALID_PARAMETER;
}
sh_log_OUTPUT_PAYLOAD payload={
.output_type=SH_LOG_FATAL,
.output_source=source,
.tsc_value=sh_tsc_get_kernel_current_tsc(),
.message_pointer=str
};
SH_STATUS status=sh_log_send_payload(&payload);
if (sh_status_error(status)) {
return status;
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_log_mem_stats(sh_log_OUTPUT_SOURCE source) {
sh_page_MEM_STATS mem_stats;
sh_page_get_memory_stats(&mem_stats);
if (sh_log_get_log_level()<=1) {
sh_log_llog("Total memory installed (bytes) : ",source);
sh_log_send_uintn(mem_stats.memory_total_bytes);
sh_log_send_string(" / 0x");
sh_log_send_uintn_hex(mem_stats.memory_total_bytes);
sh_log_send_string("\n");
sh_log_llog("Total memory installed (pages) : ",source);
sh_log_send_uintn(mem_stats.memory_total_pages);
sh_log_send_string(" / 0x");
sh_log_send_uintn_hex(mem_stats.memory_total_pages);
sh_log_send_string("\n");
sh_log_llog("Free memory : ",source);
sh_log_send_double(mem_stats.free_ratio*100);
sh_log_send_string("%\n");
sh_log_llog("Used memory : ",source);
sh_log_send_double(mem_stats.used_ratio*100);
sh_log_send_string("%\n");
sh_log_llog("Free pages : ",source);
sh_log_send_uintn(mem_stats.free_pages);
sh_log_send_string("\n");
sh_log_llog("Used pages : ",source);
sh_log_send_uintn(mem_stats.used_pages);
sh_log_send_string("\n");
sh_log_llog("Largest free block (pages) : ",source);
sh_log_send_uintn(mem_stats.largest_free_block);
sh_log_send_string("\n");
sh_log_llog("Largest used block (pages) : ",source);
sh_log_send_uintn(mem_stats.largest_used_block);
sh_log_send_string("\n");
sh_log_llog("Free block count : ",source);
sh_log_send_uintn(mem_stats.free_blocks_count);
sh_log_send_string("\n");
sh_log_llog("Used block count : ",source);
sh_log_send_uintn(mem_stats.used_blocks_count);
sh_log_send_string("\n");
sh_log_llog("Total memory taken by physical bitmap (bytes) : ",source);
sh_log_send_uintn(mem_stats.physical_bitmap_size_bytes);
sh_log_send_string(" / 0x");
sh_log_send_uintn_hex(mem_stats.physical_bitmap_size_bytes);
sh_log_send_string("\n");
sh_log_llog("Total memory taken by physical bitmap (pages) : ",source);
sh_log_send_uintn(mem_stats.physical_bitmap_size_pages);
sh_log_send_string(" / 0x");
sh_log_send_uintn_hex(mem_stats.physical_bitmap_size_pages);
sh_log_send_string("\n");
}
return SH_STATUS_SUCCESS;
}

721
page.c Normal file
View File

@@ -0,0 +1,721 @@
#include "../include/page.h"
#include <stdint.h>
__attribute__((section(".bss")))
static sh_uint8 memory_map_buffer[64*1024];
static sh_uint8 *physical_bitmap;
__attribute__((section(".bss")))
static sh_uint64 physical_memory_pages_count=0;
__attribute__((section(".bss")))
static sh_uint64 physical_memory_bytes_count=0;
__attribute__((section(".bss")))
static sh_uint64 physical_bitmap_size_bytes=0;
__attribute__((section(".bss")))
static sh_uint64 physical_bitmap_size_pages=0;
static sh_page_VIRTUAL_ADRESS page_table_pool_va_ptr=SH_PAGE_NULL_VA;
SH_STATUS sh_page_load_boot_ptp_va(sh_page_VIRTUAL_ADRESS pt_pool_va) {
page_table_pool_va_ptr=pt_pool_va;
sh_log_ldebug("Page table pool VA: 0x",SH_LOG_SOURCE_PAGE);
sh_log_send_uintn_hex((sh_uint64)page_table_pool_va_ptr);
sh_log_send_string("\n");
sh_uint8 first_byte=*(sh_uint8*)(page_table_pool_va_ptr);
sh_log_debug("If you can see this message, no fault happened.",SH_LOG_SOURCE_PAGE);
return SH_STATUS_SUCCESS;
}
sh_page_VIRTUAL_ADRESS sh_page_get_boot_ptp_va() {
return page_table_pool_va_ptr;
}
SH_STATUS sh_page_copy_memory_map() {
return sh_mem_copy(memory_map_buffer,(void*)SH_PAGE_MEMORY_MAP_VA,sizeof(memory_map_buffer));
}
SH_STATUS sh_page_check_memory_map() {
static const sh_uint8 memory_map_sig[8]={'S','h','e','M','m','a','p','B'};
if (sh_mem_compare(memory_map_sig,memory_map_buffer,sizeof(memory_map_sig))==SH_STATUS_MEM_NOT_EQUAL) {
sh_log_critical("Memory map doesn't have signature on.",SH_LOG_SOURCE_PAGE);
return SH_STATUS_INVALID_SIGNATURE;
}
sh_page_MEMORY_MAP_HEADER *memory_map_header=(sh_page_MEMORY_MAP_HEADER *)memory_map_buffer;
sh_log_ldebug("Memory map entry count: ",SH_LOG_SOURCE_PAGE);
sh_log_send_uintn((sh_uint64)memory_map_header->entry_count);
sh_log_send_string("\n");
sh_log_ldebug("Memory map entry size: ",SH_LOG_SOURCE_PAGE);
sh_log_send_uintn((sh_uint64)memory_map_header->entry_size);
sh_log_send_string("\n");
sh_log_ldebug("Memory map syntax version: ",SH_LOG_SOURCE_PAGE);
sh_log_send_uintn((sh_uint64)memory_map_header->mmap_syntax_version);
sh_log_send_string("\n");
if (memory_map_header->entry_count*memory_map_header->entry_size+sizeof(sh_page_MEMORY_MAP_HEADER)>sizeof(memory_map_buffer)) {
sh_log_error("Memory map overflow allocated buffer.",SH_LOG_SOURCE_PAGE);
return SH_STATUS_MMAP_BUFFER_OVERFLOW;
}
return SH_STATUS_SUCCESS;
}
void sh_page_dump_memory_map() {
sh_page_MEMORY_MAP_HEADER *memory_map_header=(sh_page_MEMORY_MAP_HEADER *)memory_map_buffer;
sh_log_send_string("Memory map dump:\n");
sh_log_send_string("Header:\n");
for (sh_uint64 i=0;i<sizeof(sh_page_MEMORY_MAP_HEADER);++i) {
sh_log_send_uintn((sh_uint64)memory_map_buffer[i]);
sh_log_send_string(" ");
}
sh_log_send_string("\n");
for (sh_uint64 i=0;i<memory_map_header->entry_count;++i) {
sh_log_send_string("Entry number ");
sh_log_send_uintn(i);
sh_log_send_string(" : ");
for (sh_uint64 y=0;y<sizeof(sh_page_MEMORY_MAP_ENTRY);++y) {
sh_log_send_uintn((sh_uint64)memory_map_buffer[sizeof(sh_page_MEMORY_MAP_HEADER)+i*sizeof(sh_page_MEMORY_MAP_ENTRY)+y]);
sh_log_send_string(" ");
}
sh_log_send_string("\n");
}
}
sh_uint64 sh_page_get_physical_memory_amount_pages() {
return physical_memory_pages_count;
}
sh_uint64 sh_page_get_physical_memory_amount_bytes() {
return physical_memory_bytes_count;
}
sh_uint64 sh_page_get_one_page_na() {
sh_uint64 page_count=physical_memory_pages_count;
sh_uint64 bitmap_word_count=(page_count+63)/64;
for (sh_uint64 word=0;word<bitmap_word_count;word++) {
sh_uint64 value=physical_bitmap[word];
if (value==0xFFFFFFFFFFFFFFFFULL) {
continue;
}
for (sh_uint64 bit=0;bit<64;bit++) {
sh_uint64 page_index=(word*64)+bit;
if (page_index>=page_count) {
return 0;
}
if ((value & (1ULL<<bit))==0) {
return page_index*SH_PAGE_SIZE;
}
}
}
return 0;
}
SH_STATUS sh_page_set_pages_range_bitmap(sh_uint8 *bitmap,sh_uint64 page_count_in_bitmap,sh_uint64 page_index,sh_uint64 page_count,sh_bool state) {
if (bitmap==SH_NULLPTR) {
return SH_STATUS_INVALID_PARAMETER;
}
if (page_index+page_count>page_count_in_bitmap) {
return SH_STATUS_INVALID_PARAMETER;
}
for (sh_uint64 i=0;i<page_count;++i) {
sh_uint64 page=page_index+i;
sh_uint64 byte_index=page/8;
sh_uint8 bit_index=page%8;
if (state) {
bitmap[byte_index]|=(sh_uint8)(1u<<bit_index);
} else {
bitmap[byte_index]&=(sh_uint8)~(1u<<bit_index);
}
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_page_init_ptp(sh_page_PHYSICAL_ADRESS ptp_pa,sh_page_VIRTUAL_ADRESS ptp_va,sh_uint64 initial_fill_level,sh_page_PAGE_TABLE_POOL *page_table_pool) {
page_table_pool->page_table_pa=ptp_pa;
page_table_pool->page_table_va=ptp_va;
page_table_pool->ptp_pages_count=SH_PAGE_PTP_ALLOCATOR_PAGES_COUNT;
page_table_pool->ptp_alloc_bitmap_uint64_count=SH_PAGE_PTP_ALLOCATOR_BITMAP_UINT64;
SH_STATUS status=sh_mem_set_8((sh_uint8*)page_table_pool->ptp_alloc_bitmap,SH_FALSE,sizeof(page_table_pool->ptp_alloc_bitmap));
if (sh_status_error(status)) {
sh_log_error("Error: couldn't initialize page table pool bitmap.",SH_LOG_SOURCE_PAGE);
return SH_STATUS_PT_POOL_NO_BITMAP_INIT;
}
if (initial_fill_level!=0) {
status=sh_page_set_pages_range_bitmap((sh_uint8*)page_table_pool->ptp_alloc_bitmap,page_table_pool->ptp_pages_count,0,initial_fill_level,SH_TRUE);
if (sh_status_error(status)) {
sh_log_error("Error: couldn't initialize pages tables already alocated.",SH_LOG_SOURCE_PAGE);
return SH_STATUS_PT_POOL_NO_PAGE_SET;
}
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_page_dump_ptp_bitmap(sh_page_PAGE_TABLE_POOL *ptp) {
for (sh_uint64 i=0;i<ptp->ptp_alloc_bitmap_uint64_count;++i) {
sh_log_send_string(" 0x");
sh_log_send_uintn_hex(ptp->ptp_alloc_bitmap[i]);
}
sh_log_send_string("\n");
return SH_STATUS_SUCCESS;
}
sh_page_PHYSICAL_ADRESS sh_page_ptp_alloc_one_page(sh_page_PAGE_TABLE_POOL *pt_pool) {
if (pt_pool==SH_NULLPTR) {
return SH_STATUS_INVALID_PARAMETER;
}
sh_uint64 page_count=pt_pool->ptp_pages_count;
sh_uint64 bitmap_word_count=(page_count+63)/64;
for (sh_uint64 word=0;word<bitmap_word_count;word++) {
sh_uint64 value=pt_pool->ptp_alloc_bitmap[word];
if (value==0xFFFFFFFFFFFFFFFFULL) {
continue;
}
for (sh_uint64 bit=0;bit<64;bit++) {
sh_uint64 page_index=(word*64)+bit;
if (page_index>=page_count) {
return 0;
}
if ((value & (1ULL<<bit))==0) {
pt_pool->ptp_alloc_bitmap[word]|=(1ULL<<bit);
sh_page_PHYSICAL_ADRESS pa=pt_pool->page_table_pa+page_index*SH_PAGE_SIZE;
return pa;
}
}
}
return 0;
}
SH_STATUS sh_page_map_one_page_ptp(sh_page_PAGE_TABLE_POOL *ptp,sh_page_VIRTUAL_ADRESS va,sh_page_PHYSICAL_ADRESS pa,sh_uint64 flags) {
if (ptp==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if (va%SH_PAGE_SIZE!=0 || pa%SH_PAGE_SIZE!=0) return SH_STATUS_INVALID_PARAMETER;
sh_uint64 pml4_i=(va>>39) & 0x1FF;
sh_uint64 pdpt_i=(va>>30) & 0x1FF;
sh_uint64 pd_i=(va>>21) & 0x1FF;
sh_uint64 pt_i=(va>>12) & 0x1FF;
sh_uint64 *pdpt;
sh_uint64 *pd;
sh_uint64 *pt;
sh_uint64 *pml4=(sh_uint64*)ptp->page_table_va;
if (!(pml4[pml4_i] & SH_PAGE_PRESENT)) {
sh_page_PHYSICAL_ADRESS pdpt_pa=sh_page_ptp_alloc_one_page(ptp);
if (!pdpt_pa) return SH_STATUS_OUT_OF_MEMORY;
pdpt=sh_page_ptp_pa_to_va(ptp,pdpt_pa);
if (!pdpt) return SH_STATUS_INVALID_INTERNAL_PA;
sh_mem_set_8((sh_uint8*)pdpt,0,SH_PAGE_SIZE);
pml4[pml4_i]=pdpt_pa | SH_PAGE_TABLE_FLAGS | SH_PAGE_PRESENT;
} else {
pdpt=sh_page_ptp_pa_to_va(ptp,(pml4[pml4_i] & ~0xFFFULL));
if (!pdpt) return SH_STATUS_INVALID_INTERNAL_PA;
}
if (!(pdpt[pdpt_i] & SH_PAGE_PRESENT)) {
sh_page_PHYSICAL_ADRESS pd_pa=sh_page_ptp_alloc_one_page(ptp);
if (!pd_pa) return SH_STATUS_OUT_OF_MEMORY;
pd=sh_page_ptp_pa_to_va(ptp,pd_pa);
if (!pd) return SH_STATUS_INVALID_INTERNAL_PA;
sh_mem_set_8((sh_uint8*)pd,0,SH_PAGE_SIZE);
pdpt[pdpt_i]=pd_pa | SH_PAGE_TABLE_FLAGS | SH_PAGE_PRESENT;
} else {
pd=sh_page_ptp_pa_to_va(ptp,(pdpt[pdpt_i] & ~0xFFFULL));
if (!pd) return SH_STATUS_INVALID_INTERNAL_PA;
}
if (!(pd[pd_i] & SH_PAGE_PRESENT)) {
sh_page_PHYSICAL_ADRESS pt_pa=sh_page_ptp_alloc_one_page(ptp);
if (!pt_pa) return SH_STATUS_OUT_OF_MEMORY;
pt=sh_page_ptp_pa_to_va(ptp,pt_pa);
if (!pt) return SH_STATUS_INVALID_INTERNAL_PA;
sh_mem_set_8((sh_uint8*)pt,0,SH_PAGE_SIZE);
pd[pd_i]=pt_pa | SH_PAGE_TABLE_FLAGS | SH_PAGE_PRESENT;
} else {
pt=sh_page_ptp_pa_to_va(ptp,pd[pd_i] & ~0xFFFULL);
if (!pt) return SH_STATUS_INVALID_INTERNAL_PA;
}
pt[pt_i]=(pa & ~0xFFFULL) | flags | SH_PAGE_PRESENT;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_page_is_va_mapped_ptp(sh_page_PAGE_TABLE_POOL *ptp,sh_page_VIRTUAL_ADRESS va) {
if (ptp==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
sh_uint64 pml4_i=(va>>39) & 0x1FF;
sh_uint64 pdpt_i=(va>>30) & 0x1FF;
sh_uint64 pd_i=(va>>21) & 0x1FF;
sh_uint64 pt_i=(va>>12) & 0x1FF;
sh_uint64 *pdpt;
sh_uint64 *pd;
sh_uint64 *pt;
sh_uint64 *pml4=(sh_uint64*)ptp->page_table_va;
if (!(pml4[pml4_i] & SH_PAGE_PRESENT)) {
return SH_STATUS_VA_NOT_MAPPED;
} else {
pdpt=sh_page_ptp_pa_to_va(ptp,(pml4[pml4_i] & ~0xFFFULL));
if (pdpt==0) return SH_STATUS_INVALID_INTERNAL_PA;
}
if (!(pdpt[pdpt_i] & SH_PAGE_PRESENT)) {
return SH_STATUS_VA_NOT_MAPPED;
} else {
pd=sh_page_ptp_pa_to_va(ptp,(pdpt[pdpt_i] & ~0xFFFULL));
if (pd==0) return SH_STATUS_INVALID_INTERNAL_PA;
}
if (!(pd[pd_i] & SH_PAGE_PRESENT)) {
return SH_STATUS_VA_NOT_MAPPED;
} else {
pt=sh_page_ptp_pa_to_va(ptp,(pd[pd_i] & ~0xFFFULL));
if (pt==0) return SH_STATUS_INVALID_INTERNAL_PA;
}
if (!(pt[pt_i] & SH_PAGE_PRESENT)) {
return SH_STATUS_VA_NOT_MAPPED;
} else {
return SH_STATUS_VA_MAPPED;
}
}
SH_STATUS sh_page_is_va_range_mapped_ptp(sh_page_PAGE_TABLE_POOL *ptp,sh_page_VIRTUAL_ADRESS va,sh_uint64 size_bytes) {
if (ptp==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if (va%SH_PAGE_SIZE!=0 || size_bytes%SH_PAGE_SIZE!=0 || size_bytes==0) return SH_STATUS_INVALID_PARAMETER;
sh_uint64 counter=0;
for (sh_uint64 i=0;i<size_bytes/SH_PAGE_SIZE;i++) {
if (sh_page_is_va_mapped_ptp(ptp,va+i*SH_PAGE_SIZE)==SH_STATUS_VA_MAPPED) {
counter++;
}
}
if (counter==0) return SH_STATUS_VA_NOT_MAPPED;
if (counter==size_bytes/SH_PAGE_SIZE) return SH_STATUS_VA_FULLY_MAPPED;
return SH_STATUS_VA_PARTIALLY_MAPPED;
}
SH_STATUS sh_page_search_available_va_range(sh_page_PAGE_TABLE_POOL *ptp,sh_page_VIRTUAL_ADRESS range_base,sh_page_VIRTUAL_ADRESS range_size_bytes,sh_uint64 size_bytes,sh_page_VIRTUAL_ADRESS *adress_found) {
if (ptp==SH_NULLPTR || adress_found==SH_NULLPTR) {
sh_log_send_string("1\n");
return SH_STATUS_INVALID_PARAMETER;
}
if (size_bytes==0 || size_bytes%SH_PAGE_SIZE!=0) {
sh_log_send_string("2\n");
return SH_STATUS_INVALID_PARAMETER;
};
if (range_base%SH_PAGE_SIZE!=0 || range_size_bytes%4096!=0 || range_size_bytes==0) {
sh_log_send_string("3\n");
return SH_STATUS_INVALID_PARAMETER;
};
sh_uint64 pages_needed=size_bytes/SH_PAGE_SIZE;
sh_page_VIRTUAL_ADRESS current_va=range_base;
sh_uint64 contiguous=0;
sh_page_VIRTUAL_ADRESS candidate_start=0;
while (current_va<range_base+range_size_bytes) {
SH_STATUS status=sh_page_is_va_mapped_ptp(ptp,current_va);
if (status==SH_STATUS_VA_NOT_MAPPED) {
if (contiguous==0) {
candidate_start=current_va;
}
contiguous++;
if (contiguous==pages_needed) {
*adress_found=(sh_page_VIRTUAL_ADRESS)candidate_start;
return SH_STATUS_SUCCESS;
}
} else if (status==SH_STATUS_VA_MAPPED) {
contiguous=0;
} else {
return SH_STATUS_INVALID_INTERNAL_PA;
}
current_va+=SH_PAGE_SIZE;
}
return SH_STATUS_OUT_OF_MEMORY;
}
SH_STATUS sh_page_map_contiguous_pages_range_ptp(sh_page_PAGE_TABLE_POOL *ptp,sh_page_VIRTUAL_ADRESS va,sh_page_PHYSICAL_ADRESS pa,sh_uint64 flags,sh_uint64 size_bytes) {
if (ptp==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if (va%SH_PAGE_SIZE!=0 || pa%SH_PAGE_SIZE!=0 || size_bytes==0 || size_bytes%SH_PAGE_SIZE!=0) return SH_STATUS_INVALID_PARAMETER;
if ((flags & SH_PAGE_PS)==SH_PAGE_PS) return SH_STATUS_INVALID_PARAMETER;
sh_uint64 pages=size_bytes/SH_PAGE_SIZE;
SH_STATUS status=sh_page_is_va_range_mapped_ptp(ptp,va,size_bytes);
if (status==SH_STATUS_VA_PARTIALLY_MAPPED) return SH_STATUS_ERROR_VA_PARTIALLY_MAPPED;
if (status==SH_STATUS_VA_FULLY_MAPPED) return SH_STATUS_ERROR_VA_FULLY_MAPPED;
for (sh_uint64 i=0;i<pages;i++) {
status=sh_page_map_one_page_ptp(ptp,va+i*SH_PAGE_SIZE,pa+i*SH_PAGE_SIZE,flags);
if (status!=SH_STATUS_SUCCESS) {
return status;
}
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_page_search_physical_contiguous_block_na(sh_uint64 pages_needed,sh_page_PHYSICAL_ADRESS *pa) {
if (pages_needed==0) return SH_STATUS_INVALID_PARAMETER;
sh_uint64 page_count=sh_page_get_physical_memory_amount_pages();
sh_uint64 contiguous=0;
sh_uint64 candidate_start=0;
for (sh_uint64 page_index=0;page_index<page_count;page_index++) {
if (!sh_page_is_allocated(physical_bitmap,page_index)) {
if (contiguous==0) candidate_start=page_index;
contiguous++;
if (contiguous==pages_needed) {
*pa=candidate_start*SH_PAGE_SIZE;
return SH_STATUS_SUCCESS;
}
} else {
contiguous=0;
}
}
return SH_STATUS_OUT_OF_MEMORY;
}
SH_STATUS sh_page_alloc_contiguous(sh_page_PAGE_TABLE_POOL *ptp,sh_uint64 size_bytes,sh_page_VIRTUAL_ADRESS *va) {
if (ptp==SH_NULLPTR || va==SH_NULLPTR || size_bytes==0) return SH_STATUS_INVALID_PARAMETER;
sh_uint64 pages_needed=size_bytes/SH_PAGE_SIZE;
if (size_bytes%SH_PAGE_SIZE!=0) pages_needed++;
sh_page_VIRTUAL_ADRESS candidate_va=0;
SH_STATUS status=sh_page_search_available_va_range(ptp,0x0,0x00007FFFFFFFF000,pages_needed*SH_PAGE_SIZE,&candidate_va);
if (status==SH_STATUS_OUT_OF_MEMORY) {
return SH_STATUS_OUT_OF_MEMORY;
} else if (status==SH_STATUS_INVALID_INTERNAL_PA) {
return SH_STATUS_KERNEL_PANIC;
} else if (status==SH_STATUS_INVALID_PARAMETER) {
return SH_STATUS_OUT_OF_MEMORY;
} else if (sh_status_error(status)) {
return SH_STATUS_OUT_OF_MEMORY;
}
sh_page_PHYSICAL_ADRESS candidate_pa=0;
status=sh_page_search_physical_contiguous_block_na(pages_needed,&candidate_pa);
if (status!=SH_STATUS_SUCCESS) {
return SH_STATUS_OUT_OF_MEMORY;
}
status=sh_page_map_contiguous_pages_range_ptp(ptp,candidate_va,candidate_pa,SH_PAGE_PRESENT | SH_PAGE_NX | SH_PAGE_RW,pages_needed*SH_PAGE_SIZE);
if (status!=SH_STATUS_SUCCESS) return status;
sh_page_set_pages_range_bitmap(physical_bitmap,physical_memory_pages_count,(sh_uint64)candidate_pa/SH_PAGE_SIZE,pages_needed,SH_TRUE);
*va=candidate_va;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_page_alloc_contiguous_extended(sh_page_PAGE_TABLE_POOL *ptp,sh_uint64 size_bytes,sh_page_VIRTUAL_ADRESS* va,DEFAULT sh_uint64 flags,DEFAULT sh_page_VIRTUAL_ADRESS va_range_start,DEFAULT sh_uint64 va_range_size_bytes) {
if (ptp==SH_NULLPTR || va==SH_NULLPTR || size_bytes==0) return SH_STATUS_INVALID_PARAMETER;
if (flags==SH_DEFVALUE) flags=SH_PAGE_PRESENT | SH_PAGE_NX | SH_PAGE_RW;
if (va_range_start==SH_DEFVALUE) va_range_start=0x0;
if (va_range_size_bytes==SH_DEFVALUE) va_range_size_bytes=0x00007FFFFFFFF000;
if (va_range_start%SH_PAGE_SIZE!=0 || va_range_size_bytes==0 || va_range_size_bytes%SH_PAGE_SIZE!=0) return SH_STATUS_INVALID_PARAMETER;
sh_uint64 pages_needed=size_bytes/SH_PAGE_SIZE;
if (size_bytes%SH_PAGE_SIZE!=0) pages_needed++;
sh_page_VIRTUAL_ADRESS candidate_va=0;
SH_STATUS status=sh_page_search_available_va_range(ptp,va_range_start,va_range_size_bytes,pages_needed*SH_PAGE_SIZE,&candidate_va);
if (status==SH_STATUS_OUT_OF_MEMORY) {
return SH_STATUS_OUT_OF_MEMORY;
} else if (status==SH_STATUS_INVALID_INTERNAL_PA) {
return SH_STATUS_KERNEL_PANIC;
} else if (status==SH_STATUS_INVALID_PARAMETER) {
return SH_STATUS_OUT_OF_MEMORY;
} else if (sh_status_error(status)) {
return SH_STATUS_OUT_OF_MEMORY;
}
sh_page_PHYSICAL_ADRESS candidate_pa=0;
status=sh_page_search_physical_contiguous_block_na(pages_needed,&candidate_pa);
if (status!=SH_STATUS_SUCCESS) {
return SH_STATUS_OUT_OF_MEMORY;
}
status=sh_page_map_contiguous_pages_range_ptp(ptp,candidate_va,candidate_pa,flags,pages_needed*SH_PAGE_SIZE);
if (status!=SH_STATUS_SUCCESS) return status;
sh_page_set_pages_range_bitmap(physical_bitmap,physical_memory_pages_count,(sh_uint64)candidate_pa/SH_PAGE_SIZE,pages_needed,SH_TRUE);
*va=candidate_va;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_page_unmap_one_page_ptp(sh_page_PAGE_TABLE_POOL *ptp,sh_page_VIRTUAL_ADRESS va) {
if (ptp==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if (va%SH_PAGE_SIZE!=0) return SH_STATUS_INVALID_PARAMETER;
sh_uint64 pml4_i=(va>>39) & 0x1FF;
sh_uint64 pdpt_i=(va>>30) & 0x1FF;
sh_uint64 pd_i=(va>>21) & 0x1FF;
sh_uint64 pt_i=(va>>12) & 0x1FF;
sh_uint64 *pdpt;
sh_uint64 *pd;
sh_uint64 *pt;
sh_uint64 *pml4=(sh_uint64*)ptp->page_table_va;
if (!(pml4[pml4_i] & SH_PAGE_PRESENT)) {
return SH_STATUS_ERROR_VA_NOT_MAPPED;
} else {
pdpt=sh_page_ptp_pa_to_va(ptp,(pml4[pml4_i] & ~0xFFFULL));
if (!pdpt) return SH_STATUS_INVALID_INTERNAL_PA;
}
if (!(pdpt[pdpt_i] & SH_PAGE_PRESENT)) {
return SH_STATUS_ERROR_VA_NOT_MAPPED;
} else {
pd=sh_page_ptp_pa_to_va(ptp,(pdpt[pdpt_i] & ~0xFFFULL));
if (!pd) return SH_STATUS_INVALID_INTERNAL_PA;
}
if (!(pd[pd_i] & SH_PAGE_PRESENT)) {
return SH_STATUS_ERROR_VA_NOT_MAPPED;
} else {
pt=sh_page_ptp_pa_to_va(ptp,pd[pd_i] & ~0xFFFULL);
if (!pt) return SH_STATUS_INVALID_INTERNAL_PA;
}
pt[pt_i]=0x0ULL;
__asm__ volatile("invlpg (%0)" :: "r"(va) : "memory");
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_page_unmap_contiguous_pages_range_ptp(sh_page_PAGE_TABLE_POOL *ptp,sh_page_VIRTUAL_ADRESS va,sh_uint64 size_bytes) {
if (ptp==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if (va%SH_PAGE_SIZE!=0 || size_bytes==0 || size_bytes%SH_PAGE_SIZE!=0) return SH_STATUS_INVALID_PARAMETER;
sh_uint64 pages=size_bytes/SH_PAGE_SIZE;
SH_STATUS status=sh_page_is_va_range_mapped_ptp(ptp,va,size_bytes);
if (status==SH_STATUS_VA_PARTIALLY_MAPPED) return SH_STATUS_ERROR_VA_PARTIALLY_MAPPED;
if (status==SH_STATUS_VA_NOT_MAPPED) return SH_STATUS_ERROR_VA_NOT_MAPPED;
for (sh_uint64 i=0;i<pages;i++) {
status=sh_page_unmap_one_page_ptp(ptp,va+i*SH_PAGE_SIZE);
if (status!=SH_STATUS_SUCCESS) {
return status;
}
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_page_ptp_va_to_pa(sh_page_PAGE_TABLE_POOL *ptp,sh_page_VIRTUAL_ADRESS va,sh_page_PHYSICAL_ADRESS *pa) {
if (ptp==SH_NULLPTR || pa==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if (va%SH_PAGE_SIZE!=0) return SH_STATUS_INVALID_PARAMETER;
sh_uint64 pml4_i=(va>>39) & 0x1FF;
sh_uint64 pdpt_i=(va>>30) & 0x1FF;
sh_uint64 pd_i=(va>>21) & 0x1FF;
sh_uint64 pt_i=(va>>12) & 0x1FF;
sh_uint64 *pdpt;
sh_uint64 *pd;
sh_uint64 *pt;
sh_uint64 *pml4=(sh_uint64*)ptp->page_table_va;
if (!(pml4[pml4_i] & SH_PAGE_PRESENT)) {
return SH_STATUS_ERROR_VA_NOT_MAPPED;
} else {
pdpt=sh_page_ptp_pa_to_va(ptp,(pml4[pml4_i] & ~0xFFFULL));
if (!pdpt) return SH_STATUS_INVALID_INTERNAL_PA;
}
if (!(pdpt[pdpt_i] & SH_PAGE_PRESENT)) {
return SH_STATUS_ERROR_VA_NOT_MAPPED;
} else {
pd=sh_page_ptp_pa_to_va(ptp,(pdpt[pdpt_i] & ~0xFFFULL));
if (!pd) return SH_STATUS_INVALID_INTERNAL_PA;
}
if (!(pd[pd_i] & SH_PAGE_PRESENT)) {
return SH_STATUS_ERROR_VA_NOT_MAPPED;
} else {
pt=sh_page_ptp_pa_to_va(ptp,pd[pd_i] & ~0xFFFULL);
if (!pt) return SH_STATUS_INVALID_INTERNAL_PA;
}
if (!(pt[pt_i] & SH_PAGE_PRESENT)) return SH_STATUS_ERROR_VA_NOT_MAPPED;
*pa=pt[pt_i] & 0x000FFFFFFFFFF000;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_page_unalloc_one_page(sh_page_PAGE_TABLE_POOL *ptp,sh_page_VIRTUAL_ADRESS va) {
if (ptp==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if (va%SH_PAGE_SIZE!=0) return SH_STATUS_INVALID_PARAMETER;
sh_page_PHYSICAL_ADRESS equivalent_pa;
SH_STATUS status=sh_page_ptp_va_to_pa(ptp,va,&equivalent_pa);
if (status!=SH_STATUS_SUCCESS) return status;
status=sh_page_unmap_one_page_ptp(ptp,va); // If this call return SH_STATUS_ERROR_VA_NOT_MAPPED, there is a severe bug that should cause kernel panic because sh_page_ptp_va_to_pa should already have returned exact same error code.
if (status!=SH_STATUS_SUCCESS) return status;
status=sh_page_set_pages_range_bitmap(physical_bitmap,physical_memory_pages_count,(sh_uint64)(equivalent_pa/SH_PAGE_SIZE),1,SH_FALSE);
if (status!=SH_STATUS_SUCCESS) return status;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_page_unalloc_contiguous(sh_page_PAGE_TABLE_POOL *ptp,sh_page_VIRTUAL_ADRESS va,sh_uint64 size_bytes) {
if (ptp==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if (va%SH_PAGE_SIZE!=0 || size_bytes==0 || size_bytes%SH_PAGE_SIZE!=0) return SH_STATUS_INVALID_PARAMETER;
SH_STATUS status=sh_page_is_va_range_mapped_ptp(ptp,va,size_bytes);
if (status==SH_STATUS_VA_NOT_MAPPED) return SH_STATUS_ERROR_VA_NOT_MAPPED;
if (status==SH_STATUS_VA_PARTIALLY_MAPPED) return SH_STATUS_ERROR_VA_PARTIALLY_MAPPED;
sh_uint64 pages=size_bytes/SH_PAGE_SIZE;
for (sh_uint64 i=0;i<pages;i++) {
status=sh_page_unalloc_one_page(ptp,va+i*SH_PAGE_SIZE);
if (status!=SH_STATUS_SUCCESS) {
return status;
}
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_page_analyse_memory_map(sh_page_PAGE_TABLE_POOL *ptp) {
if (ptp==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
sh_page_MEMORY_MAP_HEADER *memory_map_header=(sh_page_MEMORY_MAP_HEADER *)memory_map_buffer;
sh_page_MEMORY_MAP_ENTRY *memory_map_cursor=(sh_page_MEMORY_MAP_ENTRY *)(memory_map_buffer+sizeof(sh_page_MEMORY_MAP_HEADER));
sh_bool verbose=sh_log_get_log_level()==0;
sh_bool log=sh_log_get_log_level()<=1;
sh_uint64 highest_usable_segment=0;
sh_uint64 highest_usable_page=0;
sh_uint64 biggest_segment_index=0;
sh_uint64 biggest_segment_pages=0;
if (log) {
sh_log_llog("Max pages count is currently set to 0x",SH_LOG_SOURCE_PAGE);
sh_log_send_uintn_hex(SH_PAGE_MAX_PAGES_COUNT);
sh_log_send_string(" pages or 0x");
sh_log_send_uintn_hex(SH_PAGE_MAX_PAGES_COUNT*4096);
sh_log_send_string(" bytes.\n");
}
// first loop : identify memory amount and bigest free region
for (sh_uint64 i=0;i<memory_map_header->entry_count;i++) {
sh_uint64 start_page=memory_map_cursor[i].physical_start/4096;
sh_uint64 end_page=start_page+memory_map_cursor[i].pages_count;
if (memory_map_cursor[i].type==SH_PAGE_CONVENTIONAL_MEMORY && memory_map_cursor[i].pages_count>biggest_segment_pages) {
biggest_segment_pages=memory_map_cursor[i].pages_count;
biggest_segment_index=i;
}
if (verbose) {
sh_log_ldebug("Found memory map segment #",SH_LOG_SOURCE_PAGE);
sh_log_send_uintn(i);
sh_log_send_string(": [0x");
sh_log_send_uintn_hex(start_page*4096);
sh_log_send_string(" - 0x");
sh_log_send_uintn_hex(end_page*4096);
sh_log_send_string("] Memory type: ");
sh_log_send_uintn(memory_map_cursor[i].type);
if (memory_map_cursor[i].type==SH_PAGE_CONVENTIONAL_MEMORY) {
sh_log_send_string(" --> usable\n");
} else {
sh_log_send_string(" --> not usable\n");
}
if (!(end_page<=SH_PAGE_MAX_PAGES_COUNT)) {
sh_log_lwarning("Memory map segment #",SH_LOG_SOURCE_PAGE);
sh_log_send_uintn(i);
sh_log_send_string(" isn't usable because it overflow over max page count. Enable debug log channel to see more.\n");
}
}
if (memory_map_cursor[i].type==SH_PAGE_CONVENTIONAL_MEMORY && end_page<=SH_PAGE_MAX_PAGES_COUNT) {
if (end_page>highest_usable_page) {
highest_usable_segment=i;
highest_usable_page=end_page;
}
}
}
physical_memory_pages_count=highest_usable_page;
physical_memory_bytes_count=physical_memory_pages_count*SH_PAGE_SIZE;
if (verbose) {
sh_log_ldebug("Total memory was given by memory map segment #",SH_LOG_SOURCE_PAGE);
sh_log_send_uintn(highest_usable_segment);
sh_log_send_string("\n");
sh_log_ldebug("Total memory (pages): 0x",SH_LOG_SOURCE_PAGE);
sh_log_send_uintn_hex(physical_memory_pages_count);
sh_log_send_string(". Total memory (bytes) : 0x");
sh_log_send_uintn_hex(physical_memory_bytes_count);
sh_log_send_string("\n");
}
if (biggest_segment_pages==0) {
sh_log_error("No suitable conventional memory segment found.",SH_LOG_SOURCE_PAGE);
return SH_STATUS_OUT_OF_MEMORY;
}
if (memory_map_cursor[biggest_segment_index].pages_count<(physical_memory_pages_count/8)) {
sh_log_error("Memory is too low or too fragmented to allocate physical bitmap.",SH_LOG_SOURCE_PAGE);
return SH_STATUS_OUT_OF_MEMORY;
}
sh_page_PHYSICAL_ADRESS pa=memory_map_cursor[biggest_segment_index].physical_start;
sh_page_VIRTUAL_ADRESS va;
physical_bitmap_size_bytes=physical_memory_pages_count/8;
if (physical_memory_pages_count%8!=0) physical_bitmap_size_bytes++;
physical_bitmap_size_pages=physical_bitmap_size_bytes/SH_PAGE_SIZE;
if (physical_bitmap_size_bytes%SH_PAGE_SIZE!=0) physical_bitmap_size_pages++;
SH_STATUS status=sh_page_search_available_va_range(ptp,SH_PAGE_KERNEL_PERM_VA_BASE,(SH_PAGE_KERNEL_PERM_VA_END-SH_PAGE_KERNEL_PERM_VA_BASE+1-0x1000),physical_bitmap_size_pages*SH_PAGE_SIZE,&va);
if (status!=SH_STATUS_SUCCESS) {
sh_log_error("Memory is too low or too fragmented to allocate physical bitmap.",SH_LOG_SOURCE_PAGE);
return status;
}
status=sh_page_map_contiguous_pages_range_ptp(ptp,va,pa,SH_PAGE_PRESENT | SH_PAGE_NX | SH_PAGE_RW,physical_bitmap_size_pages*SH_PAGE_SIZE);
if (status==SH_STATUS_OUT_OF_MEMORY) {
sh_log_error("Memory is too low or too fragmented to allocate physical bitmap.",SH_LOG_SOURCE_PAGE);
return status;
} else if (status!=SH_STATUS_SUCCESS) {
sh_log_error("An unknow error happened during physical bitmap pages mapping. See error below",SH_LOG_SOURCE_PAGE);
return status;
}
physical_bitmap=(sh_uint8*)va;
status=sh_mem_set_8(physical_bitmap,0xFF,physical_bitmap_size_bytes);
if (sh_status_error(status)) {
sh_log_error("An unknow error happened during physical bitmap filling with 0xFF. See error below.",SH_LOG_SOURCE_PAGE);
return status;
}
// second loop : actually set all free regions into physical bitmap
for (sh_uint64 i=0;i<memory_map_header->entry_count;i++) {
sh_uint64 start_page=memory_map_cursor[i].physical_start/4096;
sh_uint64 end_page=start_page+memory_map_cursor[i].pages_count;
if (end_page<=SH_PAGE_MAX_PAGES_COUNT) {
if (memory_map_cursor[i].type==SH_PAGE_CONVENTIONAL_MEMORY) {
SH_STATUS status=sh_page_set_pages_range_bitmap(physical_bitmap,physical_memory_pages_count,memory_map_cursor[i].physical_start/SH_PAGE_SIZE,memory_map_cursor[i].pages_count,SH_FALSE);
if (sh_status_error(status)) {
sh_log_error("Couldn't set this memory map segment to usable.",SH_LOG_SOURCE_PAGE);
return SH_STATUS_PMAP_NO_PAGES_SET;
}
}
}
}
sh_page_set_pages_range_bitmap(physical_bitmap,physical_memory_pages_count,pa/SH_PAGE_SIZE,physical_bitmap_size_pages,SH_TRUE);
return SH_STATUS_SUCCESS;
}
sh_page_VIRTUAL_ADRESS sh_page_get_physical_bitmap_ptr() {
return (sh_page_VIRTUAL_ADRESS)physical_bitmap;
}
static sh_uint64 popcount64(sh_uint64 x) {
x=x-((x>>1) & 0x5555555555555555ULL);
x=(x & 0x3333333333333333ULL)+((x>>2) & 0x3333333333333333ULL);
x=(x+(x>>4)) & 0x0F0F0F0F0F0F0F0FULL;
x=x+(x>>8);
x=x+(x>>16);
x=x+(x>>32);
return x & 0x7F;
}
SH_STATUS sh_page_get_memory_stats(sh_page_MEM_STATS *mem_stats) {
if (mem_stats==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
mem_stats->memory_total_pages=physical_memory_pages_count;
mem_stats->memory_total_bytes=physical_memory_bytes_count;
sh_uint64 free_pages=0;
sh_uint64 used_pages=0;
sh_uint64 largest_free_block=0;
sh_uint64 largest_used_block=0;
sh_uint64 free_blocks_count=0;
sh_uint64 used_blocks_count=0;
sh_uint64 current_free_block=0;
sh_uint64 current_used_block=0;
sh_uint64 full_uint64_count=physical_memory_pages_count/64;
sh_uint64 remaining_bits=physical_memory_pages_count%64;
sh_uint64 *bitmap64=(sh_uint64*)physical_bitmap;
for (sh_uint64 i=0;i<full_uint64_count;i++) {
sh_uint64 val=bitmap64[i];
unsigned ones=popcount64(val);
unsigned zeros=64-ones;
used_pages+=ones;
free_pages+=zeros;
for (int b=0;b<64;b++) {
sh_bool bit_set=(val>>b) & 1;
if (bit_set) {
current_used_block++;
if (current_free_block) {
free_blocks_count++;
if (current_free_block>largest_free_block) {
largest_free_block=current_free_block;
}
current_free_block=0;
}
} else {
current_free_block++;
if (current_used_block) {
used_blocks_count++;
if (current_used_block>largest_used_block) {
largest_used_block=current_used_block;
}
current_used_block=0;
}
}
}
}
if (remaining_bits) {
sh_uint64 val=bitmap64[full_uint64_count] & ((1ULL<<remaining_bits)-1);
unsigned ones=popcount64(val);
unsigned zeros=remaining_bits-ones;
used_pages+=ones;
free_pages+=zeros;
for (sh_uint64 b=0;b<remaining_bits;b++) {
sh_bool bit_set=(val>>b) & 1;
if (bit_set) {
current_used_block++;
if (current_free_block) {
free_blocks_count++;
if (current_free_block>largest_free_block) {
largest_free_block=current_free_block;
}
current_free_block=0;
}
} else {
current_free_block++;
if (current_used_block) {
used_blocks_count++;
if (current_used_block>largest_used_block) {
largest_used_block=current_used_block;
}
current_used_block=0;
}
}
}
}
if (current_free_block) {
free_blocks_count++;
if (current_free_block>largest_free_block) {
largest_free_block=current_free_block;
}
}
if (current_used_block) {
used_blocks_count++;
if (current_used_block>largest_used_block) {
largest_used_block=current_used_block;
}
}
mem_stats->free_pages=free_pages;
mem_stats->used_pages=used_pages;
mem_stats->free_ratio=(double)free_pages/(double)physical_memory_pages_count;
mem_stats->used_ratio=(double)used_pages/(double)physical_memory_pages_count;
mem_stats->largest_free_block=largest_free_block;
mem_stats->largest_used_block=largest_used_block;
mem_stats->free_blocks_count=free_blocks_count;
mem_stats->used_blocks_count=used_blocks_count;
mem_stats->physical_bitmap_size_bytes=(physical_memory_pages_count+7)/8;
mem_stats->physical_bitmap_size_pages=(mem_stats->physical_bitmap_size_bytes+4095)/4096;
return SH_STATUS_SUCCESS;
}

BIN
test.c.ccc
View File

Binary file not shown.