PastaBoy12345/vystem-b
2
0
Fork 0
forked from lolo859/vystem
Code Pull Requests Activity

First commit, Vystem v0.1

Browse Source
This commit is contained in:
lolo859
2026-03-31 22:15:00 +02:00
commit e15daed8c0
462 changed files with 134655 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

205
shelter/lib/src/memory/slabs/slab_generic.c Normal file
View File

@@ -0,0 +1,205 @@
// SPDX-License-Identifier: MPL-2.0
#include "memory/slabs/slab_generic.h"
#include "std/mem.h"
#include "kernel/log.h"
static sh_uint8 sig[8]=SH_SLAB_GENERIC_SLAB_SIG;
static sh_uint64 object_size_bytes[8]=SH_SLAB_GENERIC_OBJECT_SIZE_BYTES;
static sh_uint64 actual_objects_per_slab[8]=SH_SLAB_GENERIC_ACTUAL_OBJECTS_PER_SLAB;
static sh_uint64 bitmap_init[8]=SH_SLAB_GENERIC_SLAB_BITMAP_INIT;
static sh_uint64 slab_data_pages[8]=SH_SLAB_GENERIC_SLAB_DATA_PAGES;
SH_STATUS sh_slab_generic_alloc_init(sh_uint8 level,struct sh_slab_generic_SLAB_ALLOCATOR* slab_alloc,sh_pba_PAGE_BLOCK_ALLOCATOR *pba) {
if (slab_alloc==SH_NULLPTR || pba==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if (level>7) return SH_STATUS_INVALID_PARAMETER;
slab_alloc->first_slab=SH_NULLPTR;
slab_alloc->slab_count=0;
slab_alloc->partial_head=SH_NULLPTR;
slab_alloc->pba=pba;
slab_alloc->level=level;
slab_alloc->object_size_bytes=object_size_bytes[level];
slab_alloc->object_per_slab=SH_SLAB_GENERIC_OBJECTS_PER_SLAB;
slab_alloc->actual_object_per_slab=actual_objects_per_slab[level];
slab_alloc->bitmap_init=bitmap_init[level];
slab_alloc->slab_pages_count=slab_data_pages[level];
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_generic_add_slab(struct sh_slab_generic_SLAB_ALLOCATOR* alloc,sh_page_PAGE_TABLE_POOL *ptp,sh_slab_generic_SLAB** out_slab) {
if (alloc==SH_NULLPTR || out_slab==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER; // phys_plane isn't checked because it can be SH_NULLPTR
sh_page_VIRTUAL_ADDRESS slab_va;
SH_STATUS status=sh_pba_alloc(alloc->pba,ptp,&slab_va);
if (sh_status_error(status)) return status;
if (alloc->slab_count==0) {
sh_slab_generic_SLAB *new_slab=(sh_slab_generic_SLAB*)slab_va;
for (sh_iter64 i=0;i<sizeof(sig);i++) new_slab->sig[i]=sig[i];
new_slab->next_slab=SH_NULLPTR;
new_slab->prev_slab=SH_NULLPTR;
new_slab->next_partial=SH_NULLPTR;
new_slab->prev_partial=SH_NULLPTR;
new_slab->slab_index=alloc->slab_count;
new_slab->used_count=0;
for (sh_iter64 i=0;i<8;i++) new_slab->free_bitmap[i]=0;
new_slab->free_bitmap[0]|=alloc->bitmap_init;
alloc->first_slab=new_slab;
alloc->partial_head=new_slab;
alloc->slab_count++;
*out_slab=new_slab;
} else {
sh_slab_generic_SLAB *new_slab=(sh_slab_generic_SLAB*)slab_va;
for (sh_iter64 i=0;i<sizeof(sig);i++) new_slab->sig[i]=sig[i];
new_slab->next_slab=alloc->first_slab;
new_slab->prev_slab=SH_NULLPTR;
alloc->first_slab->prev_slab=new_slab;
alloc->first_slab=new_slab;
new_slab->next_partial=alloc->partial_head;
new_slab->prev_partial=SH_NULLPTR;
alloc->partial_head->prev_partial=new_slab;
alloc->partial_head=new_slab;
new_slab->slab_index=alloc->slab_count;
new_slab->used_count=0;
for (sh_iter64 i=0;i<8;i++) new_slab->free_bitmap[i]=0;
new_slab->free_bitmap[0]|=alloc->bitmap_init;
alloc->slab_count++;
*out_slab=new_slab;
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_generic_get_partial_slab(struct sh_slab_generic_SLAB_ALLOCATOR* alloc,sh_page_PAGE_TABLE_POOL* ptp,sh_slab_generic_SLAB** found_slab) {
if (alloc==SH_NULLPTR ||ptp==SH_NULLPTR || found_slab==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if (alloc->partial_head==SH_NULLPTR) {
*found_slab=SH_NULLPTR;
return SH_STATUS_NOT_FOUND;
}
sh_slab_generic_SLAB *slab=alloc->partial_head;
if (slab->used_count>=alloc->actual_object_per_slab) {
if (slab->next_partial!=SH_NULLPTR && slab->next_partial->used_count<alloc->actual_object_per_slab) {
alloc->partial_head=slab->next_partial;
alloc->partial_head->prev_partial=SH_NULLPTR;
slab->next_partial=SH_NULLPTR;
slab->prev_partial=SH_NULLPTR;
*found_slab=alloc->partial_head;
return SH_STATUS_SUCCESS;
} else {
sh_slab_generic_SLAB *new_slab;
SH_STATUS status=sh_slab_generic_add_slab(alloc,ptp,&new_slab);
if (sh_status_error(status)) return status;
*found_slab=new_slab;
return SH_STATUS_NEW_SLAB_ADDED;
}
}
*found_slab=slab;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_generic_scan_slabs(struct sh_slab_generic_SLAB_ALLOCATOR* alloc) {
if (alloc==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
sh_slab_generic_SLAB* current=alloc->first_slab;
sh_slab_generic_SLAB* last_partial=SH_NULLPTR;
alloc->partial_head=SH_NULLPTR;
sh_log_debug("Slabs rescan triggered.",SH_LOG_SOURCE_SLAB);
while (current!=SH_NULLPTR) {
if (*(sh_uint64*)current->sig!=SH_SLAB_GENERIC_MAGIC) {
return SH_STATUS_FOUND_CORRUPTED_SLAB;
}
current->next_partial=SH_NULLPTR;
current->prev_partial=SH_NULLPTR;
if (current->used_count<alloc->actual_object_per_slab) {
if (alloc->partial_head==SH_NULLPTR) {
alloc->partial_head=current;
last_partial=current;
} else {
last_partial->next_partial=current;
current->prev_partial=last_partial;
last_partial=current;
}
}
current=current->next_slab;
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_generic_find_free_object(struct sh_slab_generic_SLAB_ALLOCATOR* alloc,sh_page_PAGE_TABLE_POOL *ptp,void** out,sh_slab_generic_OBJECT_INDEX_IN_SLAB* index_in_slab) {
if (alloc==SH_NULLPTR || ptp==SH_NULLPTR || out==SH_NULLPTR || index_in_slab==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
sh_slab_generic_SLAB *partial_slab;
sh_slab_generic_SLAB *new_slab;
SH_STATUS status=sh_slab_generic_get_partial_slab(alloc,ptp,&partial_slab);
if (status==SH_STATUS_RESCAN_NEEDED) {
status=sh_slab_generic_scan_slabs(alloc);
if (sh_status_error(status)) {
return status;
}
status=sh_slab_generic_get_partial_slab(alloc,ptp,&partial_slab);
}
if (status==SH_STATUS_NOT_FOUND) {
status=sh_slab_generic_add_slab(alloc,ptp,&new_slab);
if (sh_status_error(status)) {
return status;
}
status=sh_slab_generic_get_partial_slab(alloc,ptp,&partial_slab);
if (sh_status_error(status)) {
return status;
}
} else if (sh_status_error(status) && status!=SH_STATUS_NEW_SLAB_ADDED) {
return status;
}
sh_uint16 found_index=0;
sh_bool found=SH_FALSE;
for (sh_iter64 w=0;w<8;w++) {
sh_uint64 word=partial_slab->free_bitmap[w];
sh_uint64 free_bits=(~word);
if (free_bits) {
sh_uint64 bit=(sh_uint64)__builtin_ctzll(free_bits);
found_index=(sh_uint16)((w<<6)+bit);
found=SH_TRUE;
break;
}
}
if (!found) {
return SH_STATUS_NOT_FOUND;
}
sh_uint8 *start_data=(sh_uint8*)partial_slab;
*out=start_data+found_index*alloc->object_size_bytes;
*index_in_slab=found_index;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_generic_alloc(struct sh_slab_generic_SLAB_ALLOCATOR* alloc,sh_page_PAGE_TABLE_POOL *ptp,void** out_obj) {
if (alloc==SH_NULLPTR || ptp==SH_NULLPTR || out_obj==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
sh_uint8* object_ptr;
sh_slab_generic_OBJECT_INDEX_IN_SLAB index_in_slab;
SH_STATUS status=sh_slab_generic_find_free_object(alloc,ptp,(void*)&object_ptr,&index_in_slab);
if (sh_status_error(status)) return status;
if (object_ptr==SH_NULLPTR) return SH_STATUS_ERROR_NULLPTR_RETURNED;
for (sh_iter64 i=0;i<alloc->object_size_bytes;i++) object_ptr[i]=0;
sh_slab_generic_SLAB* slab=(sh_slab_generic_SLAB*)((sh_uint64)object_ptr & ~(0xFFFULL<<alloc->level));
if (*(sh_uint64*)slab->sig!=SH_SLAB_GENERIC_MAGIC) {
return SH_STATUS_FOUND_CORRUPTED_SLAB;
}
sh_uint16 word_idx=index_in_slab>>6;
sh_uint16 bit_idx=index_in_slab&63;
slab->free_bitmap[word_idx]|=(1ULL<<bit_idx);
slab->used_count++;
*out_obj=object_ptr;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_generic_dealloc(struct sh_slab_generic_SLAB_ALLOCATOR* alloc,void *object_ptr) {
if (alloc==SH_NULLPTR || object_ptr==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if ((sh_uint64)object_ptr%alloc->object_size_bytes!=0) return SH_STATUS_INVALID_PARAMETER;
sh_slab_generic_SLAB* slab=(sh_slab_generic_SLAB*)((sh_uint64)object_ptr & ~(0xFFFULL<<alloc->level));
if (*(sh_uint64*)slab->sig!=SH_SLAB_GENERIC_MAGIC) {
return SH_STATUS_FOUND_CORRUPTED_SLAB;
}
sh_uint16 object_idx=(sh_uint16)(((sh_uint64)object_ptr & (0xFFFULL<<alloc->level))/alloc->object_size_bytes);
sh_uint16 min_idx=(sh_uint16)(alloc->object_per_slab-alloc->actual_object_per_slab);
if (object_idx<min_idx) return SH_STATUS_INVALID_PARAMETER;
sh_uint16 word=object_idx>>6;
sh_uint16 bit=object_idx&63;
if (!(slab->free_bitmap[word] & (1ULL<<bit))) return SH_STATUS_SLAB_SLOT_ALREADY_FREE;
slab->free_bitmap[word]&= ~(1ULL<<bit);
if (slab->used_count==alloc->actual_object_per_slab) {
slab->prev_partial=SH_NULLPTR;
slab->next_partial=alloc->partial_head;
if (alloc->partial_head!=SH_NULLPTR) {
alloc->partial_head->prev_partial=slab;
}
alloc->partial_head=slab;
}
slab->used_count--;
for (sh_iter64 i=0;i<alloc->object_size_bytes;i++) ((sh_uint8*)object_ptr)[i]=0;
return SH_STATUS_SUCCESS;
}

227
shelter/lib/src/memory/slabs/slab_radix_node.c Normal file
View File

@@ -0,0 +1,227 @@
// SPDX-License-Identifier: MPL-2.0
#include "memory/slabs/slab_radix_node.h"
#include "std/mem.h"
#include "kernel/log.h"
#include "memory/pez/pez.h"
static sh_uint8 sig[8]=SH_SLAB_RADIX_NODE_SLAB_SIG;
SH_STATUS sh_slab_radix_node_alloc_init(struct sh_slab_radix_node_SLAB_ALLOCATOR* slab_alloc,sh_pba_PAGE_BLOCK_ALLOCATOR *pba) {
if (slab_alloc==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
slab_alloc->first_slab=SH_NULLPTR;
slab_alloc->slab_count=0;
slab_alloc->partial_head=SH_NULLPTR;
slab_alloc->pba=pba;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_radix_node_add_slab(struct sh_slab_radix_node_SLAB_ALLOCATOR* alloc,sh_page_PAGE_TABLE_POOL *ptp,sh_slab_radix_node_SLAB** out_slab) {
if (alloc==SH_NULLPTR || out_slab==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
sh_page_VIRTUAL_ADDRESS slab_va;
SH_STATUS status=sh_pba_alloc(alloc->pba,ptp,&slab_va);
if (sh_status_error(status)) return status;
if (alloc->slab_count==0) {
sh_slab_radix_node_SLAB *new_slab=(sh_slab_radix_node_SLAB*)slab_va;
for (sh_iter64 i=0;i<sizeof(sig);i++) new_slab->sig[i]=sig[i];
new_slab->next_slab=SH_NULLPTR;
new_slab->prev_slab=SH_NULLPTR;
new_slab->next_partial=SH_NULLPTR;
new_slab->prev_partial=SH_NULLPTR;
new_slab->slab_index=alloc->slab_count;
new_slab->used_count=0;
for (sh_iter64 i=0;i<16;i++) new_slab->free_bitmap[i]=0;
new_slab->free_bitmap[0]|=0x3FFFFULL;
sh_mem_set_8(new_slab->padding,0x00,sizeof(new_slab->padding));
sh_mem_set_8((sh_uint8*)new_slab->node_bitmap,0x00,sizeof(new_slab->node_bitmap));
sh_mem_set_8((sh_uint8*)new_slab->nodes,0x00,sizeof(new_slab->nodes));
alloc->first_slab=new_slab;
alloc->partial_head=new_slab;
alloc->slab_count++;
*out_slab=new_slab;
} else {
sh_slab_radix_node_SLAB *new_slab=(sh_slab_radix_node_SLAB*)slab_va;
for (sh_iter64 i=0;i<sizeof(sig);i++) new_slab->sig[i]=sig[i];
new_slab->next_slab=alloc->first_slab;
new_slab->prev_slab=SH_NULLPTR;
alloc->first_slab->prev_slab=new_slab;
alloc->first_slab=new_slab;
new_slab->next_partial=alloc->partial_head;
new_slab->prev_partial=SH_NULLPTR;
alloc->partial_head->prev_partial=new_slab;
alloc->partial_head=new_slab;
new_slab->slab_index=alloc->slab_count;
new_slab->used_count=0;
for (sh_iter64 i=0;i<16;i++) new_slab->free_bitmap[i]=0;
new_slab->free_bitmap[0]|=0x3FFFFULL;
sh_mem_set_8(new_slab->padding,0x00,sizeof(new_slab->padding));
sh_mem_set_8((sh_uint8*)new_slab->node_bitmap,0x00,sizeof(new_slab->node_bitmap));
sh_mem_set_8((sh_uint8*)new_slab->nodes,0x00,sizeof(new_slab->nodes));
alloc->slab_count++;
*out_slab=new_slab;
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_radix_node_get_partial_slab(struct sh_slab_radix_node_SLAB_ALLOCATOR* alloc,sh_page_PAGE_TABLE_POOL* ptp,sh_slab_radix_node_SLAB** found_slab) {
if (alloc==SH_NULLPTR ||ptp==SH_NULLPTR || found_slab==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if (alloc->partial_head==SH_NULLPTR) {
*found_slab=SH_NULLPTR;
return SH_STATUS_NOT_FOUND;
}
sh_slab_radix_node_SLAB *slab=alloc->partial_head;
if (slab->used_count>=SH_SLAB_RADIX_NODE_ACTUAL_OBJECTS_PER_SLAB) {
if (slab->next_partial!=SH_NULLPTR && slab->next_partial->used_count<SH_SLAB_RADIX_NODE_ACTUAL_OBJECTS_PER_SLAB) {
alloc->partial_head=slab->next_partial;
alloc->partial_head->prev_partial=SH_NULLPTR;
slab->next_partial=SH_NULLPTR;
slab->prev_partial=SH_NULLPTR;
*found_slab=alloc->partial_head;
return SH_STATUS_SUCCESS;
} else {
sh_slab_radix_node_SLAB *new_slab;
SH_STATUS status=sh_slab_radix_node_add_slab(alloc,ptp,&new_slab);
if (sh_status_error(status)) return status;
*found_slab=new_slab;
return SH_STATUS_NEW_SLAB_ADDED;
}
}
*found_slab=slab;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_radix_node_scan_slabs(struct sh_slab_radix_node_SLAB_ALLOCATOR* alloc) {
if (alloc==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
sh_slab_radix_node_SLAB* current=alloc->first_slab;
sh_slab_radix_node_SLAB* last_partial=SH_NULLPTR;
alloc->partial_head=SH_NULLPTR;
sh_log_debug("Slabs rescan triggered.",SH_LOG_SOURCE_SLAB);
while (current!=SH_NULLPTR) {
if (*(sh_uint64*)current->sig!=SH_SLAB_RADIX_NODE_MAGIC) {
return SH_STATUS_FOUND_CORRUPTED_SLAB;
}
current->used_count=0;
for (sh_iter64 w=0;w<16;w++) current->free_bitmap[w]=0;
current->free_bitmap[0]|=0x3FFFFULL;
for (sh_uint16 obj_idx=18;obj_idx<SH_SLAB_RADIX_NODE_OBJECTS_PER_SLAB;obj_idx++) {
sh_uint8* node_ptr=(sh_uint8*)&current->nodes[obj_idx-18];
sh_bool is_free=SH_TRUE;
sh_uint64* check_ptr=(sh_uint64*)node_ptr;
for (sh_uint8 b=0;b<(SH_SLAB_RADIX_NODE_OBJECT_SIZE_BYTES/8);b++) {
if (check_ptr[b]!=0) {
is_free=SH_FALSE;
break;
}
}
sh_uint16 word=obj_idx>>6;
sh_uint16 bit=obj_idx&63;
if (!is_free) {
current->free_bitmap[word]|=(1ULL<<bit);
current->used_count++;
}
}
current->next_partial=SH_NULLPTR;
current->prev_partial=SH_NULLPTR;
if (current->used_count<SH_SLAB_RADIX_NODE_ACTUAL_OBJECTS_PER_SLAB) {
if (alloc->partial_head==SH_NULLPTR) {
alloc->partial_head=current;
last_partial=current;
} else {
last_partial->next_partial=current;
current->prev_partial=last_partial;
last_partial=current;
}
}
current=current->next_slab;
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_radix_node_find_free_object(struct sh_slab_radix_node_SLAB_ALLOCATOR* alloc,sh_page_PAGE_TABLE_POOL *ptp,sh_radix_NODE** out,sh_slab_radix_node_NODE_INDEX_IN_SLAB* index_in_slab) {
if (alloc==SH_NULLPTR || ptp==SH_NULLPTR || out==SH_NULLPTR || index_in_slab==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
sh_slab_radix_node_SLAB *partial_slab;
sh_slab_radix_node_SLAB *new_slab;
SH_STATUS status=sh_slab_radix_node_get_partial_slab(alloc,ptp,&partial_slab);
if (status==SH_STATUS_RESCAN_NEEDED) {
status=sh_slab_radix_node_scan_slabs(alloc);
if (sh_status_error(status)) return status;
status=sh_slab_radix_node_get_partial_slab(alloc,ptp,&partial_slab);
}
if (status==SH_STATUS_NOT_FOUND) {
status=sh_slab_radix_node_add_slab(alloc,ptp,&new_slab);
if (sh_status_error(status)) return status;
status=sh_slab_radix_node_get_partial_slab(alloc,ptp,&partial_slab);
if (sh_status_error(status)) return status;
} else if (sh_status_error(status) && status!=SH_STATUS_NEW_SLAB_ADDED) {
return status;
}
sh_uint16 found_idx=0;
sh_bool bit_found=SH_FALSE;
for (sh_uint8 w=0;w<16;w++) {
if (partial_slab->free_bitmap[w]!=0xFFFFFFFFFFFFFFFFULL) {
for (sh_uint8 b=0;b<64;b++) {
if (!(partial_slab->free_bitmap[w] & (1ULL<<b))) {
found_idx=(w<<6)|b;
if (found_idx<18) continue;
bit_found=SH_TRUE;
break;
}
}
}
if (bit_found) break;
}
if (!bit_found) return SH_STATUS_NOT_FOUND;
*out=&partial_slab->nodes[found_idx-18];
*index_in_slab=found_idx;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_radix_node_alloc(struct sh_slab_radix_node_SLAB_ALLOCATOR* alloc,sh_page_PAGE_TABLE_POOL *ptp,sh_radix_NODE** out_obj) {
if (alloc==SH_NULLPTR || ptp==SH_NULLPTR || out_obj==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
sh_radix_NODE *object_ptr;
sh_slab_radix_node_NODE_INDEX_IN_SLAB index_in_slab;
SH_STATUS status=sh_slab_radix_node_find_free_object(alloc,ptp,&object_ptr,&index_in_slab);
if (sh_status_error(status)) return status;
if (object_ptr==SH_NULLPTR) return SH_STATUS_ERROR_NULLPTR_RETURNED;
for (sh_iter64 i=0;i<16;i++) object_ptr->ptr[i]=0x0;
sh_slab_radix_node_SLAB* slab=(sh_slab_radix_node_SLAB*)((sh_uint64)object_ptr & ~0x1FFFFULL);
if (*(sh_uint64*)slab->sig!=SH_SLAB_RADIX_NODE_MAGIC) {
return SH_STATUS_FOUND_CORRUPTED_SLAB;
}
sh_uint16 word_idx=index_in_slab>>6;
sh_uint16 bit_idx=index_in_slab&63;
slab->free_bitmap[word_idx]|=(1ULL<<bit_idx);
slab->node_bitmap[index_in_slab]=0;
slab->used_count++;
*out_obj=object_ptr;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_radix_node_dealloc(struct sh_slab_radix_node_SLAB_ALLOCATOR* alloc,sh_radix_NODE *object_ptr) {
if (alloc==SH_NULLPTR || object_ptr==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if ((sh_uint64)object_ptr%SH_SLAB_RADIX_NODE_OBJECT_SIZE_BYTES!=0) return SH_STATUS_INVALID_PARAMETER;
sh_slab_radix_node_SLAB* slab=(sh_slab_radix_node_SLAB*)((sh_uint64)object_ptr & ~0x1FFFFULL);
if (*(sh_uint64*)slab->sig!=SH_SLAB_RADIX_NODE_MAGIC) {
return SH_STATUS_FOUND_CORRUPTED_SLAB;
}
sh_uint16 object_idx=(sh_uint16)(((sh_uint64)object_ptr & 0x1FFFFULL)/SH_SLAB_RADIX_NODE_OBJECT_SIZE_BYTES);
if (object_idx<18) return SH_STATUS_INVALID_PARAMETER;
sh_uint16 word=object_idx>>6;
sh_uint16 bit=object_idx&63;
if (!(slab->free_bitmap[word] & (1ULL<<bit))) return SH_STATUS_SLAB_SLOT_ALREADY_FREE;
slab->free_bitmap[word]&= ~(1ULL<<bit);
if (slab->used_count==SH_SLAB_RADIX_NODE_ACTUAL_OBJECTS_PER_SLAB) {
slab->prev_partial=SH_NULLPTR;
slab->next_partial=alloc->partial_head;
if (alloc->partial_head!=SH_NULLPTR) {
alloc->partial_head->prev_partial=slab;
}
alloc->partial_head=slab;
}
slab->used_count--;
for (sh_iter64 i=0;i<16;i++) object_ptr->ptr[i]=0x0;
slab->node_bitmap[object_idx]=0;
return SH_STATUS_SUCCESS;
}
sh_uint16 *sh_slab_radix_node_get_node_bitmap(struct sh_slab_radix_node_SLAB_ALLOCATOR* alloc,sh_radix_NODE *object_ptr) {
if (alloc==SH_NULLPTR || object_ptr==SH_NULLPTR) return SH_NULLPTR;
if ((sh_uint64)object_ptr%SH_SLAB_RADIX_NODE_OBJECT_SIZE_BYTES!=0) return SH_NULLPTR;
sh_slab_radix_node_SLAB* slab=(sh_slab_radix_node_SLAB*)((sh_uint64)object_ptr & ~0x1FFFFULL);
if (*(sh_uint64*)slab->sig!=SH_SLAB_RADIX_NODE_MAGIC) {
return SH_NULLPTR;
}
sh_uint16 object_idx=(sh_uint16)(((sh_uint64)object_ptr & 0x1FFFFULL)/SH_SLAB_RADIX_NODE_OBJECT_SIZE_BYTES);
if (object_idx<18) return SH_NULLPTR;
return &slab->node_bitmap[object_idx];
}

274
shelter/lib/src/memory/slabs/slab_reg_phys.c Normal file
View File

@@ -0,0 +1,274 @@
// SPDX-License-Identifier: MPL-2.0
#include "memory/slabs/slab_reg_phys.h"
#include "std/mem.h"
#include "kernel/log.h"
#include "memory/pez/pez.h"
SH_STATUS sh_slab_reg_phys_alloc_init(sh_slab_reg_phys_SLAB_ALLOCATOR* slab_alloc,sh_page_PAGE_TABLE_POOL *ptp) {
if (slab_alloc==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
slab_alloc->max_slabs=SH_SLAB_REG_PHYS_MAX_SLAB;
slab_alloc->slab_count=0;
slab_alloc->partial_head=SH_NULLPTR;
slab_alloc->slabs_data=SH_NULLPTR;
sh_page_VIRTUAL_ADDRESS header_va;
SH_STATUS status=sh_page_alloc_contiguous_extended(ptp,SH_SLAB_REG_PHYS_HEADER_LIST_SIZE_BYTES,&header_va,SH_PAGE_RW | SH_PAGE_PRESENT | SH_PAGE_NX,SH_PAGE_KERNEL_PERM_VA_BASE,(SH_PAGE_KERNEL_PERM_VA_END-SH_PAGE_KERNEL_PERM_VA_BASE+1-0x1000));
if (sh_status_error(status)) return status;
slab_alloc->slabs_header=(sh_slab_reg_phys_SLAB_STRUCT*)header_va;
status=sh_mem_set_8((sh_uint8*)(slab_alloc->slabs_header),0x00,SH_SLAB_REG_PHYS_HEADER_LIST_SIZE_BYTES);
if (sh_status_error(status)) return status;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_reg_phys_add_slab(sh_slab_reg_phys_SLAB_ALLOCATOR* alloc,sh_page_PAGE_TABLE_POOL *ptp,sh_slab_reg_phys_SLAB_STRUCT** out_slab) {
if (alloc==SH_NULLPTR || out_slab==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if (alloc->slab_count>=alloc->max_slabs) return SH_STATUS_SLAB_ALLOCATOR_FULL;
sh_page_PHYSICAL_ADDRESS slab_pa;
SH_STATUS status;
if (sh_pez_is_available()) {
sh_pez_PHYSICAL_PLANE *phys_plane=sh_pez_get_reference_phys_plane();
status=sh_pez_alloc_physical_pages(phys_plane,SH_SLAB_REG_PHYS_SLAB_DATA_PAGES,&slab_pa);
if (sh_status_error(status)) return status;
} else {
status=sh_page_search_physical_contiguous_block_na(SH_SLAB_REG_PHYS_SLAB_DATA_PAGES,&slab_pa);
if (sh_status_error(status)) return status;
}
if (alloc->slabs_data==SH_NULLPTR && alloc->slab_count==0) {
sh_page_VIRTUAL_ADDRESS alloc_slab_data_va_base=SH_SLAB_REG_PHYS_DATA_VA;
status=sh_page_is_va_range_mapped_ptp(ptp,alloc_slab_data_va_base,SH_SLAB_REG_PHYS_MAX_SLAB*SH_SLAB_REG_PHYS_SLAB_DATA_SIZE_BYTES);
if (status!=SH_STATUS_VA_NOT_MAPPED) return status;
status=sh_page_map_contiguous_pages_range_ptp(ptp,alloc_slab_data_va_base,slab_pa,SH_PAGE_PRESENT | SH_PAGE_NX | SH_PAGE_RW,SH_SLAB_REG_PHYS_SLAB_DATA_SIZE_BYTES);
if (sh_status_error(status)) return status;
alloc->slabs_data=(sh_uint8*)alloc_slab_data_va_base;
sh_slab_reg_phys_SLAB_STRUCT *new_slab_header=alloc->slabs_header;
new_slab_header->slab_index=alloc->slab_count;
new_slab_header->used_count=0;
new_slab_header->prev=SH_NULLPTR;
new_slab_header->next=SH_NULLPTR;
for (sh_iter64 i=0;i<16;i++) new_slab_header->free_bitmap[i]=0;
new_slab_header->free_bitmap[0]|=1ULL;
new_slab_header->used_count=0;
status=sh_mem_set_8(alloc->slabs_data,0x00,SH_SLAB_REG_PHYS_SLAB_DATA_SIZE_BYTES);
if (sh_status_error(status)) return status;
if (!sh_pez_is_available()) {
sh_page_MEM_STATS mem_stats;
status=sh_page_get_memory_stats(&mem_stats);
if (sh_status_error(status)) return status;
status=sh_page_set_pages_range_bitmap((sh_uint8*)sh_page_get_physical_bitmap_ptr(),mem_stats.memory_total_pages,slab_pa/SH_PAGE_SIZE,SH_SLAB_REG_PHYS_SLAB_DATA_PAGES,SH_TRUE);
if (sh_status_error(status)) return status;
}
alloc->partial_head=alloc->slabs_header;
*out_slab=new_slab_header;
alloc->slab_count++;
} else if (alloc->slabs_data!=SH_NULLPTR && alloc->slab_count!=0) {
sh_page_VIRTUAL_ADDRESS alloc_slab_data_va=SH_SLAB_REG_PHYS_DATA_VA+alloc->slab_count*SH_SLAB_REG_PHYS_SLAB_DATA_SIZE_BYTES;
status=sh_page_is_va_range_mapped_ptp(ptp,alloc_slab_data_va,SH_SLAB_REG_PHYS_SLAB_DATA_SIZE_BYTES);
if (status!=SH_STATUS_VA_NOT_MAPPED) return status;
status=sh_page_map_contiguous_pages_range_ptp(ptp,alloc_slab_data_va,slab_pa,SH_PAGE_PRESENT | SH_PAGE_NX | SH_PAGE_RW,SH_SLAB_REG_PHYS_SLAB_DATA_SIZE_BYTES);
if (sh_status_error(status)) return status;
sh_slab_reg_phys_SLAB_STRUCT *new_slab_header=alloc->slabs_header+alloc->slab_count;
new_slab_header->slab_index=alloc->slab_count;
new_slab_header->used_count=0;
new_slab_header->prev=SH_NULLPTR;
new_slab_header->next=alloc->partial_head;
sh_slab_reg_phys_SLAB_STRUCT *next_slab_header=new_slab_header->next;
next_slab_header->prev=new_slab_header;
alloc->partial_head=new_slab_header;
for (sh_iter64 i=0;i<16;i++) new_slab_header->free_bitmap[i]=0;
new_slab_header->free_bitmap[0]|=1ULL;
new_slab_header->used_count=0;
status=sh_mem_set_8(alloc->slabs_data+alloc->slab_count*SH_SLAB_REG_PHYS_SLAB_DATA_SIZE_BYTES,0x00,SH_SLAB_REG_PHYS_SLAB_DATA_SIZE_BYTES);
if (sh_status_error(status)) return status;
if (!sh_pez_is_available()) {
sh_page_MEM_STATS mem_stats;
status=sh_page_get_memory_stats(&mem_stats);
if (sh_status_error(status)) return status;
status=sh_page_set_pages_range_bitmap((sh_uint8*)sh_page_get_physical_bitmap_ptr(),mem_stats.memory_total_pages,slab_pa/SH_PAGE_SIZE,SH_SLAB_REG_PHYS_SLAB_DATA_PAGES,SH_TRUE);
if (sh_status_error(status)) return status;
}
*out_slab=new_slab_header;
alloc->slab_count++;
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_reg_phys_get_partial_slab(sh_slab_reg_phys_SLAB_ALLOCATOR* alloc,sh_page_PAGE_TABLE_POOL* ptp,sh_slab_reg_phys_SLAB_STRUCT** found_slab) {
if (alloc==SH_NULLPTR || ptp==SH_NULLPTR || found_slab==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if (alloc->partial_head==SH_NULLPTR) {
*found_slab=SH_NULLPTR;
return SH_STATUS_NOT_FOUND;
}
sh_slab_reg_phys_SLAB_STRUCT* slab=alloc->partial_head;
if (slab->used_count>=SH_SLAB_REG_PHYS_ACTUAL_OBJECTS_PER_SLAB) {
if (slab->next!=SH_NULLPTR && slab->next->used_count<SH_SLAB_REG_PHYS_ACTUAL_OBJECTS_PER_SLAB) {
alloc->partial_head=slab->next;
alloc->partial_head->prev=SH_NULLPTR;
slab->next=SH_NULLPTR;
slab->prev=SH_NULLPTR;
*found_slab=alloc->partial_head;
return SH_STATUS_SUCCESS;
} else {
sh_slab_reg_phys_SLAB_STRUCT* new_slab;
SH_STATUS status=sh_slab_reg_phys_add_slab(alloc,ptp,&new_slab);
if (sh_status_error(status)) return status;
*found_slab=new_slab;
return SH_STATUS_NEW_SLAB_ADDED;
}
}
*found_slab=slab;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_reg_phys_scan_slabs(sh_slab_reg_phys_SLAB_ALLOCATOR* alloc) {
if (alloc==SH_NULLPTR || alloc->slabs_data==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
sh_slab_reg_phys_SLAB_STRUCT* first_partial=SH_NULLPTR;
sh_slab_reg_phys_SLAB_STRUCT* last_partial=SH_NULLPTR;
sh_log_debug("Slabs rescan triggered.",SH_LOG_SOURCE_SLAB);
for (sh_uint16 i=0;i<alloc->slab_count;i++) {
sh_slab_reg_phys_SLAB_STRUCT* slab=alloc->slabs_header+i;
sh_uint8* slab_base=alloc->slabs_data+i*SH_SLAB_REG_PHYS_SLAB_DATA_SIZE_BYTES;
slab->used_count=0;
for (sh_iter64 w=0;w<16;w++) slab->free_bitmap[w]=0;
slab->free_bitmap[0]|=1ULL;
for (sh_uint16 obj_idx=1;obj_idx<SH_SLAB_REG_PHYS_OBJECTS_PER_SLAB;obj_idx++) {
sh_uint8* obj_ptr=slab_base+obj_idx*SH_SLAB_REG_PHYS_OBJECT_SIZE_BYTES;
int is_free=1;
for (sh_iter64 b=0;b<SH_SLAB_REG_PHYS_OBJECT_SIZE_BYTES;b++) {
if (obj_ptr[b]!=0) {
is_free=0;
break;
}
}
if (is_free) {
sh_uint16 word=obj_idx>>6;
sh_uint16 bit=obj_idx&63;
slab->free_bitmap[word]&= ~(1ULL<<bit);
} else {
sh_uint16 word=obj_idx>>6;
sh_uint16 bit=obj_idx&63;
slab->free_bitmap[word]|=(1ULL<<bit);
slab->used_count++;
}
}
sh_log_fdebug(SH_LOG_SOURCE_SLAB,"Slab index: %2u; used count: %2u\n",slab->slab_index,slab->used_count);
slab->prev=SH_NULLPTR;
slab->next=SH_NULLPTR;
if (slab->used_count>0 && slab->used_count<SH_SLAB_REG_PHYS_ACTUAL_OBJECTS_PER_SLAB) {
if (first_partial==SH_NULLPTR) {
first_partial=slab;
last_partial=slab;
} else {
slab->prev=last_partial;
last_partial->next=slab;
last_partial=slab;
}
}
}
alloc->partial_head=first_partial;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_reg_phys_find_free_object(sh_slab_reg_phys_SLAB_ALLOCATOR* alloc,sh_page_PAGE_TABLE_POOL *ptp,sh_slab_reg_phys_OBJECT_INDEX* out_ref) {
if (alloc==SH_NULLPTR || ptp==SH_NULLPTR || out_ref==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
sh_slab_reg_phys_SLAB_STRUCT *partial_slab;
sh_slab_reg_phys_SLAB_STRUCT *new_slab;
SH_STATUS status=sh_slab_reg_phys_get_partial_slab(alloc,ptp,&partial_slab);
if (status==SH_STATUS_RESCAN_NEEDED) {
status=sh_slab_reg_phys_scan_slabs(alloc);
if (sh_status_error(status)) return status;
status=sh_slab_reg_phys_get_partial_slab(alloc,ptp,&partial_slab);
}
if (status==SH_STATUS_NOT_FOUND) {
status=sh_slab_reg_phys_add_slab(alloc,ptp,&new_slab);
if (sh_status_error(status)) return status;
status=sh_slab_reg_phys_get_partial_slab(alloc,ptp,&partial_slab);
if (sh_status_error(status)) return status;
} else if (sh_status_error(status) && status!=SH_STATUS_NEW_SLAB_ADDED) {
return status;
}
sh_uint64 obj;
sh_bool obj_found=SH_FALSE;
for (sh_uint8 w=0;w<16;w++) {
sh_uint64 word=partial_slab->free_bitmap[w];
if (word!=SH_UINT64_MAX) {
if (w==0) {
for (sh_uint8 bit=0;bit<64;bit++) {
if (!(word & (1ULL<<bit))) {
sh_uint64 candidate=(sh_uint64)w*64+bit;
if (candidate==0) continue;
obj=candidate;
obj_found=SH_TRUE;
break;
}
}
} else {
for (sh_uint8 bit=0;bit<64;bit++) {
if (!(word & (1ULL<<bit))) {
obj=(sh_uint64)w*64+bit;
obj_found=SH_TRUE;
break;
}
}
}
}
if (obj_found==SH_TRUE) break;
}
if (obj_found==SH_FALSE) {
return SH_STATUS_RESCAN_NEEDED;
}
*out_ref=SH_SLAB_REG_PHYS_MAKE_REF(partial_slab->slab_index,obj);
return SH_STATUS_SUCCESS;
}
void* sh_slab_reg_phys_ref_to_ptr(sh_slab_reg_phys_SLAB_ALLOCATOR* alloc,sh_slab_reg_phys_OBJECT_INDEX ref) {
if (alloc==SH_NULLPTR) {
return SH_NULLPTR;
}
if (SH_SLAB_REG_PHYS_REF_SLAB(ref)>=alloc->slab_count || SH_SLAB_REG_PHYS_REF_OBJECT(ref)==0) {
return SH_NULLPTR;
}
sh_uint16 slab=SH_SLAB_REG_PHYS_REF_SLAB(ref);
sh_uint16 object=SH_SLAB_REG_PHYS_REF_OBJECT(ref);
if (object==0) {
return SH_NULLPTR;
}
return alloc->slabs_data+slab*SH_SLAB_REG_PHYS_SLAB_DATA_SIZE_BYTES+object*SH_SLAB_REG_PHYS_OBJECT_SIZE_BYTES;
}
SH_STATUS sh_slab_reg_phys_alloc(sh_slab_reg_phys_SLAB_ALLOCATOR* alloc,sh_page_PAGE_TABLE_POOL *ptp,sh_slab_reg_phys_OBJECT_INDEX* out_index) {
if (alloc==SH_NULLPTR || ptp==SH_NULLPTR || out_index==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
sh_slab_reg_phys_OBJECT_INDEX ref;
SH_STATUS status=sh_slab_reg_phys_find_free_object(alloc,ptp,&ref);
if (sh_status_error(status)) return status;
sh_uint16 slab_idx=SH_SLAB_REG_PHYS_REF_SLAB(ref);
sh_uint16 obj_idx=SH_SLAB_REG_PHYS_REF_OBJECT(ref);
sh_slab_reg_phys_SLAB_STRUCT* slab=alloc->slabs_header+slab_idx;
sh_uint16 word=obj_idx>>6;
sh_uint16 bit=obj_idx&63;
slab->free_bitmap[word]|=(1ULL<<bit);
sh_pez_REGION_PHYSICAL_OBJECT* obj=(sh_pez_REGION_PHYSICAL_OBJECT*)sh_slab_reg_phys_ref_to_ptr(alloc,ref);
if (obj==SH_NULLPTR) return SH_STATUS_ERROR_NULLPTR_RETURNED;
obj->start_page_index=0;
obj->region_size_pages=0;
obj->next_region_index[0]=obj->next_region_index[1]=obj->next_region_index[2]=0;
obj->flags=0;
slab->used_count++;
*out_index=ref;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_reg_phys_dealloc(sh_slab_reg_phys_SLAB_ALLOCATOR* alloc,sh_slab_reg_phys_OBJECT_INDEX index) {
if (alloc==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if (SH_SLAB_REG_PHYS_REF_OBJECT(index)==0) return SH_STATUS_INVALID_PARAMETER;
if (SH_SLAB_REG_PHYS_REF_SLAB(index)>=alloc->slab_count) return SH_STATUS_INVALID_PARAMETER;
sh_uint16 obj_idx=SH_SLAB_REG_PHYS_REF_OBJECT(index);
sh_uint16 word=obj_idx>>6;
sh_uint16 bit=obj_idx&63;
sh_slab_reg_phys_SLAB_STRUCT* slab_header=alloc->slabs_header+SH_SLAB_REG_PHYS_REF_SLAB(index);
if (!(slab_header->free_bitmap[word] & (1ULL<<bit))) return SH_STATUS_SLAB_SLOT_ALREADY_FREE;
slab_header->free_bitmap[word]&= ~(1ULL<<bit);
if (slab_header->used_count==SH_SLAB_REG_PHYS_ACTUAL_OBJECTS_PER_SLAB) {
slab_header->prev=SH_NULLPTR;
slab_header->next=alloc->partial_head;
if (alloc->partial_head!=SH_NULLPTR) {
alloc->partial_head->prev=slab_header;
}
alloc->partial_head=slab_header;
}
slab_header->used_count--;
sh_uint8* obj_ptr=sh_slab_reg_phys_ref_to_ptr(alloc,index);
if (obj_ptr==SH_NULLPTR) return SH_STATUS_ERROR_NULLPTR_RETURNED;
sh_mem_set_8(obj_ptr,0x00,SH_SLAB_REG_PHYS_OBJECT_SIZE_BYTES);
return SH_STATUS_SUCCESS;
}

271
shelter/lib/src/memory/slabs/slab_reg_virt.c Normal file
View File

@@ -0,0 +1,271 @@
// SPDX-License-Identifier: MPL-2.0
#include "memory/slabs/slab_reg_virt.h"
#include "std/mem.h"
#include "kernel/log.h"
#include "memory/pez/pez.h"
SH_STATUS sh_slab_reg_virt_alloc_init(sh_slab_reg_virt_SLAB_ALLOCATOR* slab_alloc,sh_page_PAGE_TABLE_POOL *ptp) {
if (slab_alloc==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
slab_alloc->max_slabs=SH_SLAB_REG_VIRT_MAX_SLAB;
slab_alloc->slab_count=0;
slab_alloc->partial_head=SH_NULLPTR;
slab_alloc->slabs_data=SH_NULLPTR;
sh_page_VIRTUAL_ADDRESS header_va;
SH_STATUS status=sh_page_alloc_contiguous_extended(ptp,SH_SLAB_REG_VIRT_HEADER_LIST_SIZE_BYTES,&header_va,SH_PAGE_RW | SH_PAGE_PRESENT | SH_PAGE_NX,SH_PAGE_KERNEL_PERM_VA_BASE,(SH_PAGE_KERNEL_PERM_VA_END-SH_PAGE_KERNEL_PERM_VA_BASE+1-0x1000));
if (sh_status_error(status)) return status;
slab_alloc->slabs_header=(sh_slab_reg_virt_SLAB_STRUCT*)header_va;
status=sh_mem_set_8((sh_uint8*)(slab_alloc->slabs_header),0x00,SH_SLAB_REG_VIRT_HEADER_LIST_SIZE_BYTES);
if (sh_status_error(status)) return status;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_reg_virt_add_slab(sh_slab_reg_virt_SLAB_ALLOCATOR* alloc,sh_page_PAGE_TABLE_POOL *ptp,sh_slab_reg_virt_SLAB_STRUCT** out_slab) {
if (alloc==SH_NULLPTR || out_slab==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if (alloc->slab_count>=alloc->max_slabs) return SH_STATUS_SLAB_ALLOCATOR_FULL;
sh_page_VIRTUAL_ADDRESS slab_pa;
SH_STATUS status;
if (sh_pez_is_available()) {
sh_pez_PHYSICAL_PLANE *phys_plane=sh_pez_get_reference_phys_plane();
status=sh_pez_alloc_physical_pages(phys_plane,SH_SLAB_REG_PHYS_SLAB_DATA_PAGES,&slab_pa);
if (sh_status_error(status)) return status;
} else {
status=sh_page_search_physical_contiguous_block_na(SH_SLAB_REG_PHYS_SLAB_DATA_PAGES,&slab_pa);
if (sh_status_error(status)) return status;
}
if (alloc->slabs_data==SH_NULLPTR && alloc->slab_count==0) {
sh_page_VIRTUAL_ADDRESS alloc_slab_data_va_base=SH_SLAB_REG_VIRT_DATA_VA;
status=sh_page_is_va_range_mapped_ptp(ptp,alloc_slab_data_va_base,SH_SLAB_REG_VIRT_MAX_SLAB*SH_SLAB_REG_VIRT_SLAB_DATA_SIZE_BYTES);
if (status!=SH_STATUS_VA_NOT_MAPPED) return status;
status=sh_page_map_contiguous_pages_range_ptp(ptp,alloc_slab_data_va_base,slab_pa,SH_PAGE_PRESENT | SH_PAGE_NX | SH_PAGE_RW,SH_SLAB_REG_VIRT_SLAB_DATA_SIZE_BYTES);
if (sh_status_error(status)) return status;
alloc->slabs_data=(sh_uint8*)alloc_slab_data_va_base;
sh_slab_reg_virt_SLAB_STRUCT *new_slab_header=alloc->slabs_header;
new_slab_header->slab_index=alloc->slab_count;
new_slab_header->used_count=0;
new_slab_header->prev=SH_NULLPTR;
new_slab_header->next=SH_NULLPTR;
for (sh_iter64 i=0;i<16;i++) new_slab_header->free_bitmap[i]=0;
new_slab_header->free_bitmap[0]|=1ULL;
new_slab_header->used_count=0;
status=sh_mem_set_8(alloc->slabs_data,0x00,SH_SLAB_REG_VIRT_SLAB_DATA_SIZE_BYTES);
if (sh_status_error(status)) return status;
if (!sh_pez_is_available()) {
sh_page_MEM_STATS mem_stats;
status=sh_page_get_memory_stats(&mem_stats);
if (sh_status_error(status)) return status;
status=sh_page_set_pages_range_bitmap((sh_uint8*)sh_page_get_physical_bitmap_ptr(),mem_stats.memory_total_pages,slab_pa/SH_PAGE_SIZE,SH_SLAB_REG_VIRT_SLAB_DATA_PAGES,SH_TRUE);
if (sh_status_error(status)) return status;
}
alloc->partial_head=alloc->slabs_header;
*out_slab=new_slab_header;
alloc->slab_count++;
} else if (alloc->slabs_data!=SH_NULLPTR && alloc->slab_count!=0) {
sh_page_VIRTUAL_ADDRESS alloc_slab_data_va=SH_SLAB_REG_VIRT_DATA_VA+alloc->slab_count*SH_SLAB_REG_VIRT_SLAB_DATA_SIZE_BYTES;
status=sh_page_is_va_range_mapped_ptp(ptp,alloc_slab_data_va,SH_SLAB_REG_VIRT_SLAB_DATA_SIZE_BYTES);
if (status!=SH_STATUS_VA_NOT_MAPPED) return status;
status=sh_page_map_contiguous_pages_range_ptp(ptp,alloc_slab_data_va,slab_pa,SH_PAGE_PRESENT | SH_PAGE_NX | SH_PAGE_RW,SH_SLAB_REG_VIRT_SLAB_DATA_SIZE_BYTES);
if (sh_status_error(status)) return status;
sh_slab_reg_virt_SLAB_STRUCT *new_slab_header=alloc->slabs_header+alloc->slab_count;
new_slab_header->slab_index=alloc->slab_count;
new_slab_header->used_count=0;
new_slab_header->prev=SH_NULLPTR;
new_slab_header->next=alloc->partial_head;
sh_slab_reg_virt_SLAB_STRUCT *next_slab_header=new_slab_header->next;
next_slab_header->prev=new_slab_header;
alloc->partial_head=new_slab_header;
for (sh_iter64 i=0;i<16;i++) new_slab_header->free_bitmap[i]=0;
new_slab_header->free_bitmap[0]|=1ULL;
new_slab_header->used_count=0;
status=sh_mem_set_8(alloc->slabs_data+alloc->slab_count*SH_SLAB_REG_VIRT_SLAB_DATA_SIZE_BYTES,0x00,SH_SLAB_REG_VIRT_SLAB_DATA_SIZE_BYTES);
if (sh_status_error(status)) return status;
if (!sh_pez_is_available()) {
sh_page_MEM_STATS mem_stats;
status=sh_page_get_memory_stats(&mem_stats);
if (sh_status_error(status)) return status;
status=sh_page_set_pages_range_bitmap((sh_uint8*)sh_page_get_physical_bitmap_ptr(),mem_stats.memory_total_pages,slab_pa/SH_PAGE_SIZE,SH_SLAB_REG_VIRT_SLAB_DATA_PAGES,SH_TRUE);
if (sh_status_error(status)) return status;
}
*out_slab=new_slab_header;
alloc->slab_count++;
}
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_reg_virt_get_partial_slab(sh_slab_reg_virt_SLAB_ALLOCATOR* alloc,sh_page_PAGE_TABLE_POOL* ptp,sh_slab_reg_virt_SLAB_STRUCT** found_slab) {
if (alloc==SH_NULLPTR || ptp==SH_NULLPTR || found_slab==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if (alloc->partial_head==SH_NULLPTR) {
*found_slab=SH_NULLPTR;
return SH_STATUS_NOT_FOUND;
}
sh_slab_reg_virt_SLAB_STRUCT* slab=alloc->partial_head;
if (slab->used_count>=SH_SLAB_REG_VIRT_ACTUAL_OBJECTS_PER_SLAB) {
if (slab->next!=SH_NULLPTR && slab->next->used_count<SH_SLAB_REG_VIRT_ACTUAL_OBJECTS_PER_SLAB) {
alloc->partial_head=slab->next;
alloc->partial_head->prev=SH_NULLPTR;
slab->next=SH_NULLPTR;
slab->prev=SH_NULLPTR;
*found_slab=alloc->partial_head;
return SH_STATUS_SUCCESS;
} else {
sh_slab_reg_virt_SLAB_STRUCT* new_slab;
SH_STATUS status=sh_slab_reg_virt_add_slab(alloc,ptp,&new_slab);
if (sh_status_error(status)) return status;
*found_slab=new_slab;
return SH_STATUS_NEW_SLAB_ADDED;
}
}
*found_slab=slab;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_reg_virt_scan_slabs(sh_slab_reg_virt_SLAB_ALLOCATOR* alloc) {
if (alloc==SH_NULLPTR || alloc->slabs_data==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
sh_slab_reg_virt_SLAB_STRUCT* first_partial=SH_NULLPTR;
sh_slab_reg_virt_SLAB_STRUCT* last_partial=SH_NULLPTR;
sh_log_debug("Slabs rescan triggered.",SH_LOG_SOURCE_SLAB);
for (sh_uint32 i=0;i<alloc->slab_count;i++) {
sh_slab_reg_virt_SLAB_STRUCT* slab=alloc->slabs_header+i;
sh_uint8* slab_base=alloc->slabs_data+i*SH_SLAB_REG_VIRT_SLAB_DATA_SIZE_BYTES;
slab->used_count=0;
for (sh_iter64 w=0;w<16;w++) slab->free_bitmap[w]=0;
slab->free_bitmap[0]|=1ULL;
for (sh_uint16 obj_idx=1;obj_idx<SH_SLAB_REG_VIRT_OBJECTS_PER_SLAB;obj_idx++) {
sh_uint8* obj_ptr=slab_base+obj_idx*SH_SLAB_REG_VIRT_OBJECT_SIZE_BYTES;
int is_free=1;
for (sh_iter64 b=0;b<SH_SLAB_REG_VIRT_OBJECT_SIZE_BYTES;b++) {
if (obj_ptr[b]!=0) {
is_free=0;
break;
}
}
if (is_free) {
sh_uint16 word=obj_idx>>6;
sh_uint16 bit=obj_idx&63;
slab->free_bitmap[word]&= ~(1ULL<<bit);
} else {
sh_uint16 word=obj_idx>>6;
sh_uint16 bit=obj_idx&63;
slab->free_bitmap[word]|=(1ULL<<bit);
slab->used_count++;
}
}
sh_log_fdebug(SH_LOG_SOURCE_SLAB,"Slab index: %4u; used count: %4u\n",slab->slab_index,slab->used_count);
slab->prev=SH_NULLPTR;
slab->next=SH_NULLPTR;
if (slab->used_count>0 && slab->used_count<SH_SLAB_REG_VIRT_ACTUAL_OBJECTS_PER_SLAB) {
if (first_partial==SH_NULLPTR) {
first_partial=slab;
last_partial=slab;
} else {
slab->prev=last_partial;
last_partial->next=slab;
last_partial=slab;
}
}
}
alloc->partial_head=first_partial;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_reg_virt_find_free_object(sh_slab_reg_virt_SLAB_ALLOCATOR* alloc,sh_page_PAGE_TABLE_POOL *ptp,sh_slab_reg_virt_OBJECT_INDEX* out_ref) {
if (alloc==SH_NULLPTR || ptp==SH_NULLPTR || out_ref==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
sh_slab_reg_virt_SLAB_STRUCT *partial_slab;
sh_slab_reg_virt_SLAB_STRUCT *new_slab;
SH_STATUS status=sh_slab_reg_virt_get_partial_slab(alloc,ptp,&partial_slab);
if (status==SH_STATUS_RESCAN_NEEDED) {
status=sh_slab_reg_virt_scan_slabs(alloc);
if (sh_status_error(status)) return status;
status=sh_slab_reg_virt_get_partial_slab(alloc,ptp,&partial_slab);
}
if (status==SH_STATUS_NOT_FOUND) {
status=sh_slab_reg_virt_add_slab(alloc,ptp,&new_slab);
if (sh_status_error(status)) return status;
status=sh_slab_reg_virt_get_partial_slab(alloc,ptp,&partial_slab);
if (sh_status_error(status)) return status;
} else if (sh_status_error(status) && status!=SH_STATUS_NEW_SLAB_ADDED) {
return status;
}
sh_uint64 obj;
sh_bool obj_found=SH_FALSE;
for (sh_uint8 w=0;w<16;w++) {
sh_uint64 word=partial_slab->free_bitmap[w];
if (word!=SH_UINT64_MAX) {
if (w==0) {
for (sh_uint8 bit=0;bit<64;bit++) {
if (!(word & (1ULL<<bit))) {
sh_uint64 candidate=(sh_uint64)w*64+bit;
if (candidate==0) continue;
obj=candidate;
obj_found=SH_TRUE;
break;
}
}
} else {
for (sh_uint8 bit=0;bit<64;bit++) {
if (!(word & (1ULL<<bit))) {
obj=(sh_uint64)w*64+bit;
obj_found=SH_TRUE;
break;
}
}
}
}
if (obj_found==SH_TRUE) break;
}
if (obj_found==SH_FALSE) {
return SH_STATUS_RESCAN_NEEDED;
}
*out_ref=SH_SLAB_REG_VIRT_MAKE_REF(partial_slab->slab_index,obj);
return SH_STATUS_SUCCESS;
}
void* sh_slab_reg_virt_ref_to_ptr(sh_slab_reg_virt_SLAB_ALLOCATOR* alloc,sh_slab_reg_virt_OBJECT_INDEX ref) {
if (alloc==SH_NULLPTR) {
return SH_NULLPTR;
}
if (SH_SLAB_REG_VIRT_REF_SLAB(ref)>=alloc->slab_count || SH_SLAB_REG_VIRT_REF_OBJECT(ref)==0) {
return SH_NULLPTR;
}
sh_uint32 slab=SH_SLAB_REG_VIRT_REF_SLAB(ref);
sh_uint16 object=SH_SLAB_REG_VIRT_REF_OBJECT(ref);
if (object==0) {
return SH_NULLPTR;
}
return alloc->slabs_data+slab*SH_SLAB_REG_VIRT_SLAB_DATA_SIZE_BYTES+object*SH_SLAB_REG_VIRT_OBJECT_SIZE_BYTES;
}
SH_STATUS sh_slab_reg_virt_alloc(sh_slab_reg_virt_SLAB_ALLOCATOR* alloc,sh_page_PAGE_TABLE_POOL *ptp,sh_slab_reg_virt_OBJECT_INDEX* out_index) {
if (alloc==SH_NULLPTR || ptp==SH_NULLPTR || out_index==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
sh_slab_reg_virt_OBJECT_INDEX ref;
SH_STATUS status=sh_slab_reg_virt_find_free_object(alloc,ptp,&ref);
if (sh_status_error(status)) return status;
sh_uint32 slab_idx=SH_SLAB_REG_VIRT_REF_SLAB(ref);
sh_uint16 obj_idx=SH_SLAB_REG_VIRT_REF_OBJECT(ref);
sh_slab_reg_virt_SLAB_STRUCT* slab=alloc->slabs_header+slab_idx;
sh_uint16 word=obj_idx>>6;
sh_uint16 bit=obj_idx&63;
slab->free_bitmap[word]|=(1ULL<<bit);
sh_pez_REGION_VIRTUAL_OBJECT* obj=(sh_pez_REGION_VIRTUAL_OBJECT*)sh_slab_reg_virt_ref_to_ptr(alloc,ref);
if (obj==SH_NULLPTR) return SH_STATUS_ERROR_NULLPTR_RETURNED;
sh_mem_set_8((sh_uint8*)obj,0x0,SH_SLAB_REG_VIRT_OBJECT_SIZE_BYTES);
slab->used_count++;
*out_index=ref;
return SH_STATUS_SUCCESS;
}
SH_STATUS sh_slab_reg_virt_dealloc(sh_slab_reg_virt_SLAB_ALLOCATOR* alloc,sh_slab_reg_virt_OBJECT_INDEX index) {
if (alloc==SH_NULLPTR) return SH_STATUS_INVALID_PARAMETER;
if (SH_SLAB_REG_VIRT_REF_OBJECT(index)==0) return SH_STATUS_INVALID_PARAMETER;
if (SH_SLAB_REG_VIRT_REF_SLAB(index)>=alloc->slab_count) return SH_STATUS_INVALID_PARAMETER;
sh_uint16 obj_idx=SH_SLAB_REG_VIRT_REF_OBJECT(index);
sh_uint16 word=obj_idx>>6;
sh_uint16 bit=obj_idx&63;
sh_slab_reg_virt_SLAB_STRUCT* slab_header=alloc->slabs_header+SH_SLAB_REG_VIRT_REF_SLAB(index);
if (!(slab_header->free_bitmap[word] & (1ULL<<bit))) return SH_STATUS_SLAB_SLOT_ALREADY_FREE;
slab_header->free_bitmap[word]&= ~(1ULL<<bit);
if (slab_header->used_count==SH_SLAB_REG_VIRT_ACTUAL_OBJECTS_PER_SLAB) {
slab_header->prev=SH_NULLPTR;
slab_header->next=alloc->partial_head;
if (alloc->partial_head!=SH_NULLPTR) {
alloc->partial_head->prev=slab_header;
}
alloc->partial_head=slab_header;
}
slab_header->used_count--;
sh_uint8* obj_ptr=sh_slab_reg_virt_ref_to_ptr(alloc,index);
if (obj_ptr==SH_NULLPTR) return SH_STATUS_ERROR_NULLPTR_RETURNED;
sh_mem_set_8(obj_ptr,0x00,SH_SLAB_REG_VIRT_OBJECT_SIZE_BYTES);
return SH_STATUS_SUCCESS;
}