private static mi_page_queue_t *mi_heap_page_queue_of(mi_heap_t *heap, [NativeTypeName("const mi_page_t*")] mi_page_t *page) { byte bin = mi_page_is_in_full(page) ? MI_BIN_FULL : _mi_bin(page->xblock_size); mi_assert_internal((MI_DEBUG > 1) && (bin <= MI_BIN_FULL)); mi_page_queue_t *pq = &heap->pages.e0 + bin; mi_assert_internal((MI_DEBUG > 1) && (mi_page_is_in_full(page) || page->xblock_size == pq->block_size)); return(pq); }
private static void mi_page_queue_enqueue_from(mi_page_queue_t *to, mi_page_queue_t *from, mi_page_t *page) { mi_assert_internal((MI_DEBUG > 1) && (page != null)); mi_assert_expensive((MI_DEBUG > 2) && mi_page_queue_contains(from, page)); mi_assert_expensive((MI_DEBUG > 2) && !mi_page_queue_contains(to, page)); mi_assert_internal((MI_DEBUG > 1) && (((page->xblock_size == to->block_size) && page->xblock_size == from->block_size) || ((page->xblock_size == to->block_size) && mi_page_queue_is_full(from)) || ((page->xblock_size == from->block_size) && mi_page_queue_is_full(to)) || ((page->xblock_size > MI_LARGE_OBJ_SIZE_MAX) && mi_page_queue_is_huge(to)) || ((page->xblock_size > MI_LARGE_OBJ_SIZE_MAX) && mi_page_queue_is_full(to)))); mi_heap_t *heap = mi_page_heap(page); if (page->prev != null) { page->prev->next = page->next; } if (page->next != null) { page->next->prev = page->prev; } if (page == from->last) { from->last = page->prev; } if (page == from->first) { from->first = page->next; mi_assert_internal((MI_DEBUG > 1) && mi_heap_contains_queue(heap, from)); // update first mi_heap_queue_first_update(heap, from); } page->prev = to->last; page->next = null; if (to->last != null) { mi_assert_internal((MI_DEBUG > 1) && (heap == mi_page_heap(to->last))); to->last->next = page; to->last = page; } else { to->first = page; to->last = page; mi_heap_queue_first_update(heap, to); } mi_page_set_in_full(page, mi_page_queue_is_full(to)); }
private static mi_page_queue_t *mi_page_queue_of([NativeTypeName("const mi_page_t*")] mi_page_t *page) { byte bin = mi_page_is_in_full(page) ? MI_BIN_FULL : _mi_bin(page->xblock_size); mi_heap_t *heap = mi_page_heap(page); mi_assert_internal((MI_DEBUG > 1) && heap != null && bin <= MI_BIN_FULL); mi_page_queue_t *pq = &heap->pages.e0 + bin; mi_assert_internal((MI_DEBUG > 1) && (bin >= MI_BIN_HUGE || page->xblock_size == pq->block_size)); mi_assert_expensive((MI_DEBUG > 2) && mi_page_queue_contains(pq, page)); return(pq); }
// Retire a page with no more used blocks // Important to not retire too quickly though as new // allocations might coming. // Note: called from `mi_free` and benchmarks often // trigger this due to freeing everything and then // allocating again so careful when changing this. private static partial void _mi_page_retire(mi_page_t *page) { mi_assert_internal((MI_DEBUG > 1) && (page != null)); mi_assert_expensive((MI_DEBUG > 2) && _mi_page_is_valid(page)); mi_assert_internal((MI_DEBUG > 1) && mi_page_all_free(page)); mi_page_set_has_aligned(page, false); // don't retire too often.. // (or we end up retiring and re-allocating most of the time) // NOTE: refine this more: we should not retire if this // is the only page left with free blocks. It is not clear // how to check this efficiently though... // for now, we don't retire if it is the only page left of this size class. mi_page_queue_t *pq = mi_page_queue_of(page); if (mi_likely((page->xblock_size <= MI_MAX_RETIRE_SIZE) && !mi_page_is_in_full(page))) { if (pq->last == page && pq->first == page) { // the only page in the queue? mi_stat_counter_increase(ref _mi_stats_main.page_no_retire, 1); page->retire_expire = (page->xblock_size <= MI_SMALL_OBJ_SIZE_MAX) ? MI_RETIRE_CYCLES : (byte)(MI_RETIRE_CYCLES / 4); mi_heap_t *heap = mi_page_heap(page); mi_assert_internal((MI_DEBUG > 1) && (pq >= &heap->pages.e0)); nuint index = (nuint)(pq - &heap->pages.e0); mi_assert_internal((MI_DEBUG > 1) && index < MI_BIN_FULL && index < MI_BIN_HUGE); if (index < heap->page_retired_min) { heap->page_retired_min = index; } if (index > heap->page_retired_max) { heap->page_retired_max = index; } mi_assert_internal((MI_DEBUG > 1) && mi_page_all_free(page)); // dont't free after all return; } } _mi_page_free(page, pq, false); }
// ------------------------------------------------------ // Check for double free in secure and debug mode // This is somewhat expensive so only enabled for secure mode 4 // ------------------------------------------------------ // linear check if the free list contains a specific element private static bool mi_list_contains([NativeTypeName("const mi_page_t*")] mi_page_t *page, [NativeTypeName("const mi_block_t*")] mi_block_t *list, [NativeTypeName("const mi_block_t*")] mi_block_t *elem) { mi_assert_internal((MI_DEBUG > 1) && (MI_ENCODE_FREELIST != 0) && ((MI_SECURE >= 4) || (MI_DEBUG != 0))); while (list != null) { if (elem == list) { return(true); } list = mi_block_next(page, list); } return(false); }
private static bool mi_check_is_double_freex([NativeTypeName("const mi_page_t*")] mi_page_t *page, [NativeTypeName("const mi_block_t*")] mi_block_t *block) { mi_assert_internal((MI_DEBUG > 1) && (MI_ENCODE_FREELIST != 0) && ((MI_SECURE >= 4) || (MI_DEBUG != 0))); // The decoded value is in the same page (or null). // Walk the free lists to verify positively if it is already freed if (mi_list_contains(page, page->free, block) || mi_list_contains(page, page->local_free, block) || mi_list_contains(page, mi_page_thread_free(page), block)) { _mi_error_message(EAGAIN, "double free detected of block {0:X} with size {1}\n", (nuint)block, mi_page_block_size(page)); return(true); } return(false); }
private static nuint mi_page_list_count(mi_page_t *page, mi_block_t *head) { mi_assert_internal((MI_DEBUG > 1) && (MI_DEBUG >= 3)); nuint count = 0; while (head != null) { mi_assert_internal((MI_DEBUG > 1) && (page == _mi_ptr_page(head))); count++; head = mi_block_next(page, head); } return(count); }
private static mi_page_t *mi_page_fresh(mi_heap_t *heap, mi_page_queue_t *pq) { mi_assert_internal((MI_DEBUG > 1) && mi_heap_contains_queue(heap, pq)); mi_page_t *page = mi_page_fresh_alloc(heap, pq, pq->block_size); if (page == null) { return(null); } mi_assert_internal((MI_DEBUG > 1) && (pq->block_size == mi_page_block_size(page))); mi_assert_internal((MI_DEBUG > 1) && (pq == mi_page_queue(heap, mi_page_block_size(page)))); return(page); }
private static void mi_page_to_full(mi_page_t *page, mi_page_queue_t *pq) { mi_assert_internal((MI_DEBUG > 1) && (pq == mi_page_queue_of(page))); mi_assert_internal((MI_DEBUG > 1) && (!mi_page_immediate_available(page))); mi_assert_internal((MI_DEBUG > 1) && (!mi_page_is_in_full(page))); if (mi_page_is_in_full(page)) { return; } mi_page_queue_enqueue_from(&mi_page_heap(page)->pages.e0 + MI_BIN_FULL, pq, page); // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set _mi_page_free_collect(page, false); }
private static partial void _mi_page_reclaim(mi_heap_t *heap, mi_page_t *page) { mi_assert_expensive((MI_DEBUG > 2) && mi_page_is_valid_init(page)); mi_assert_internal((MI_DEBUG > 1) && (mi_page_heap(page) == heap)); mi_assert_internal((MI_DEBUG > 1) && (mi_page_thread_free_flag(page) != MI_NEVER_DELAYED_FREE)); mi_assert_internal((MI_DEBUG > 1) && (_mi_page_segment(page)->page_kind != MI_PAGE_HUGE)); mi_assert_internal((MI_DEBUG > 1) && (!page->is_reset)); // TODO: push on full queue immediately if it is full? mi_page_queue_t *pq = mi_page_queue(heap, mi_page_block_size(page)); mi_page_queue_push(heap, pq, page); mi_assert_expensive((MI_DEBUG > 2) && _mi_page_is_valid(page)); }
private static nuint mi_page_usable_size_of([NativeTypeName("const mi_page_t*")] mi_page_t *page, [NativeTypeName("const mi_block_t*")] mi_block_t *block) { if ((MI_PADDING > 0) && (MI_ENCODE_FREELIST != 0)) { bool ok = mi_page_decode_padding(page, block, out nuint delta, out nuint bsize); mi_assert_internal((MI_DEBUG > 1) && ok); mi_assert_internal((MI_DEBUG > 1) && (delta <= bsize)); return(ok ? (bsize - delta) : 0); } else { return(mi_page_usable_block_size(page)); } }
// free retired pages: we don't need to look at the entire queues // since we only retire pages that are at the head position in a queue. private static partial void _mi_heap_collect_retired(mi_heap_t *heap, bool force) { nuint min = MI_BIN_FULL; nuint max = 0; for (nuint bin = heap->page_retired_min; bin <= heap->page_retired_max; bin++) { mi_page_queue_t *pq = &heap->pages.e0 + bin; mi_page_t * page = pq->first; if ((page != null) && (page->retire_expire != 0)) { if (mi_page_all_free(page)) { page->retire_expire--; if (force || (page->retire_expire == 0)) { _mi_page_free(pq->first, pq, force); } else { // keep retired, update min/max if (bin < min) { min = bin; } if (bin > max) { max = bin; } } } else { page->retire_expire = 0; } } } heap->page_retired_min = min; heap->page_retired_max = max; }
private static bool mi_check_is_double_free([NativeTypeName("const mi_page_t*")] mi_page_t *page, [NativeTypeName("const mi_block_t*")] mi_block_t *block) { if ((MI_ENCODE_FREELIST != 0) && ((MI_SECURE >= 4) || (MI_DEBUG != 0))) { // pretend it is freed, and get the decoded first field mi_block_t *n = mi_block_nextx(page, block, &page->keys.e0); // quick check: aligned pointer && in same page or null if ((((nuint)n & (MI_INTPTR_SIZE - 1)) == 0) && ((n == null) || mi_is_in_same_page(block, n))) { // Suspicous: decoded value a in block is in the same page (or null) -- maybe a double free? // (continue in separate function to improve code generation) return(mi_check_is_double_freex(page, block)); } } return(false); }
private static partial void _mi_page_free_collect(mi_page_t *page, bool force) { mi_assert_internal((MI_DEBUG > 1) && (page != null)); // collect the thread free list if (force || (mi_page_thread_free(page) != null)) { // quick test to avoid an atomic operation _mi_page_thread_free_collect(page); } // and the local free list if (page->local_free != null) { if (mi_likely(page->free == null)) { // usual case page->free = page->local_free; page->local_free = null; page->is_zero = false; } else if (force) { // append -- only on shutdown (force) as this is a linear operation mi_block_t *tail = page->local_free; mi_block_t *next; while ((next = mi_block_next(page, tail)) != null) { tail = next; } mi_block_set_next(page, tail, page->free); page->free = page->local_free; page->local_free = null; page->is_zero = false; } } mi_assert_internal((MI_DEBUG > 1) && (!force || (page->local_free == null))); }
private static bool mi_page_list_is_valid(mi_page_t *page, mi_block_t *p) { mi_assert_internal((MI_DEBUG > 1) && (MI_DEBUG >= 3)); byte *page_area = _mi_page_start(_mi_page_segment(page), page, out nuint psize); mi_block_t *start = (mi_block_t *)page_area; mi_block_t *end = (mi_block_t *)(page_area + psize); while (p != null) { if (p < start || p >= end) { return(false); } p = mi_block_next(page, p); } return(true); }
/* ----------------------------------------------------------- * Helpers * ----------------------------------------------------------- */ // Visit all pages in a heap; returns `false` if break was called. private static bool mi_heap_visit_pages(mi_heap_t *heap, [NativeTypeName("heap_page_visitor_fun*")] heap_page_visitor_fun fn, void *arg1, void *arg2) { if ((heap == null) || (heap->page_count == 0)) { return(false); } nuint total = 0; // visit all pages if (MI_DEBUG > 1) { total = heap->page_count; } nuint count = 0; for (nuint i = 0; i <= MI_BIN_FULL; i++) { mi_page_queue_t *pq = &heap->pages.e0 + i; mi_page_t * page = pq->first; while (page != null) { // save next in case the page gets removed from the queue mi_page_t *next = page->next; mi_assert_internal((MI_DEBUG > 1) && (mi_page_heap(page) == heap)); count++; if (!fn(heap, pq, page, arg1, arg2)) { return(false); } page = next; // and continue } } mi_assert_internal((MI_DEBUG > 1) && (count == total)); return(true); }
// Free a page with no more free blocks private static partial void _mi_page_free(mi_page_t *page, mi_page_queue_t *pq, bool force) { mi_assert_internal((MI_DEBUG > 1) && (page != null)); mi_assert_expensive((MI_DEBUG > 2) && _mi_page_is_valid(page)); mi_assert_internal((MI_DEBUG > 1) && (pq == mi_page_queue_of(page))); mi_assert_internal((MI_DEBUG > 1) && mi_page_all_free(page)); mi_assert_internal((MI_DEBUG > 1) && (mi_page_thread_free_flag(page) != MI_DELAYED_FREEING)); // no more aligned blocks in here mi_page_set_has_aligned(page, false); // remove from the page list // (no need to do _mi_heap_delayed_free first as all blocks are already free) mi_segments_tld_t *segments_tld = &mi_page_heap(page)->tld->segments; mi_page_queue_remove(pq, page); // and free it mi_page_set_heap(page, null); _mi_segment_page_free(page, force, segments_tld); }
private static void mi_page_queue_remove(mi_page_queue_t *queue, mi_page_t *page) { mi_assert_internal((MI_DEBUG > 1) && (page != null)); mi_assert_expensive((MI_DEBUG > 2) && mi_page_queue_contains(queue, page)); mi_assert_internal((MI_DEBUG > 1) && ((page->xblock_size == queue->block_size) || ((page->xblock_size > MI_LARGE_OBJ_SIZE_MAX) && mi_page_queue_is_huge(queue)) || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)))); mi_heap_t *heap = mi_page_heap(page); if (page->prev != null) { page->prev->next = page->next; } if (page->next != null) { page->next->prev = page->prev; } if (page == queue->last) { queue->last = page->prev; } if (page == queue->first) { queue->first = page->next; mi_assert_internal((MI_DEBUG > 1) && mi_heap_contains_queue(heap, queue)); // update first mi_heap_queue_first_update(heap, queue); } heap->page_count--; page->next = null; page->prev = null; // mi_atomic_store_ptr_release(ref page->heap, null); mi_page_set_in_full(page, false); }
private static bool mi_page_queue_contains(mi_page_queue_t *queue, [NativeTypeName("const mi_page_t*")] mi_page_t *page) { mi_assert_internal((MI_DEBUG > 1) && (MI_DEBUG > 1)); mi_assert_internal((MI_DEBUG > 1) && (page != null)); mi_page_t *list = queue->first; while (list != null) { mi_assert_internal((MI_DEBUG > 1) && ((list->next == null) || (list->next->prev == list))); mi_assert_internal((MI_DEBUG > 1) && ((list->prev == null) || (list->prev->next == list))); if (list == page) { break; } list = list->next; } return(list == page); }
// regular free private static void _mi_free_block(mi_page_t *page, bool local, mi_block_t *block) { // and push it on the free list if (mi_likely(local)) { // owning thread can free a block directly if (mi_unlikely(mi_check_is_double_free(page, block))) { return; } mi_check_padding(page, block); if (MI_DEBUG != 0) { _ = memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); } mi_block_set_next(page, block, page->local_free); page->local_free = block; page->used--; if (mi_unlikely(mi_page_all_free(page))) { _mi_page_retire(page); } else if (mi_unlikely(mi_page_is_in_full(page))) { _mi_page_unfull(page); } } else { _mi_free_block_mt(page, block); } }
/* ----------------------------------------------------------- * Unfull, abandon, free and retire * ----------------------------------------------------------- */ // Move a page from the full list back to a regular list private static partial void _mi_page_unfull(mi_page_t *page) { mi_assert_internal((MI_DEBUG > 1) && (page != null)); mi_assert_expensive((MI_DEBUG > 2) && _mi_page_is_valid(page)); mi_assert_internal((MI_DEBUG > 1) && mi_page_is_in_full(page)); if (!mi_page_is_in_full(page)) { return; } mi_heap_t * heap = mi_page_heap(page); mi_page_queue_t *pqfull = &heap->pages.e0 + MI_BIN_FULL; // to get the right queue mi_page_set_in_full(page, false); mi_page_queue_t *pq = mi_heap_page_queue_of(heap, page); mi_page_set_in_full(page, true); mi_page_queue_enqueue_from(pq, pqfull, page); }
private static partial void _mi_page_use_delayed_free(mi_page_t *page, mi_delayed_t delay, bool override_never) { #pragma warning disable CS0420 nuint tfreex; mi_delayed_t old_delay; nuint tfree; do { // note: must acquire as we can break/repeat this loop and not do a CAS; tfree = mi_atomic_load_acquire(ref page->xthread_free); tfreex = mi_tf_set_delayed(tfree, delay); old_delay = mi_tf_delayed(tfree); if (mi_unlikely(old_delay == MI_DELAYED_FREEING)) { // delay until outstanding MI_DELAYED_FREEING are done. mi_atomic_yield(); // will cause CAS to busy fail // tfree = mi_tf_set_delayed(tfree, MI_NO_DELAYED_FREE); } else if (delay == old_delay) { // avoid atomic operation if already equal break; } else if (!override_never && (old_delay == MI_NEVER_DELAYED_FREE)) { // leave never-delayed flag set break; } }while ((old_delay == MI_DELAYED_FREEING) || !mi_atomic_cas_weak_release(ref page->xthread_free, ref tfree, tfreex)); #pragma warning restore CS0420 }
private static partial void _mi_block_zero_init(mi_page_t *page, void *p, nuint size) { // note: we need to initialize the whole usable block size to zero, not just the requested size, // or the recalloc/rezalloc functions cannot safely expand in place (see issue #63) mi_assert_internal((MI_DEBUG > 1) && (p != null)); mi_assert_internal((MI_DEBUG > 1) && (mi_usable_size(p) >= size)); // size can be zero mi_assert_internal((MI_DEBUG > 1) && (_mi_ptr_page(p) == page)); if (page->is_zero && (size > SizeOf <mi_block_t>())) { // already zero initialized memory // clear the free list pointer ((mi_block_t *)p)->next = 0; mi_assert_expensive((MI_DEBUG > 2) && mi_mem_is_zero(p, mi_usable_size(p))); } else { // otherwise memset _ = memset(p, 0, mi_usable_size(p)); } }
private static void mi_page_free_list_extend_secure([NativeTypeName("mi_heap_t* const")] mi_heap_t *heap, [NativeTypeName("mi_page_t* const")] mi_page_t *page, [NativeTypeName("const size_t")] nuint bsize, [NativeTypeName("const size_t")] nuint extend, [NativeTypeName("mi_stats_t* const")] in mi_stats_t stats)
// ------------------------------------------------------ // Aligned Allocation // ------------------------------------------------------ private static void *mi_heap_malloc_zero_aligned_at([NativeTypeName("mi_heap_t* const")] mi_heap_t *heap, [NativeTypeName("const size_t")] nuint size, [NativeTypeName("const size_t")] nuint alignment, [NativeTypeName("const size_t")] nuint offset, [NativeTypeName("const bool")] bool zero) { void *p; // note: we don't require `size > offset`, we just guarantee that // the address at offset is aligned regardless of the allocated size. mi_assert((MI_DEBUG != 0) && (alignment > 0)); if (mi_unlikely(size > (nuint)PTRDIFF_MAX)) { // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>) return(null); } if (mi_unlikely((alignment == 0) || !_mi_is_power_of_two(alignment))) { // require power-of-two (see <https://en.cppreference.com/w/c/memory/aligned_alloc>) return(null); } // for any x, `(x & align_mask) == (x % alignment)` nuint align_mask = alignment - 1; // try if there is a small block available with just the right alignment nuint padsize = size + MI_PADDING_SIZE; if (mi_likely(padsize <= MI_SMALL_SIZE_MAX)) { mi_page_t *page = _mi_heap_get_free_small_page(heap, padsize); bool is_aligned = (((nuint)page->free + offset) & align_mask) == 0; if (mi_likely(page->free != null && is_aligned)) { if (MI_STAT > 1) { mi_stat_increase(ref heap->tld->stats.malloc, size); } // TODO: inline _mi_page_malloc p = _mi_page_malloc(heap, page, padsize); mi_assert_internal((MI_DEBUG > 1) && (p != null)); mi_assert_internal((MI_DEBUG > 1) && (((nuint)p + offset) % alignment == 0)); if (zero) { _mi_block_zero_init(page, p, size); } return(p); } } // use regular allocation if it is guaranteed to fit the alignment constraints if ((offset == 0) && (alignment <= padsize) && (padsize <= MI_MEDIUM_OBJ_SIZE_MAX) && ((padsize & align_mask) == 0)) { p = _mi_heap_malloc_zero(heap, size, zero); mi_assert_internal((MI_DEBUG > 1) && ((p == null) || (((nuint)p % alignment) == 0))); return(p); } // otherwise over-allocate p = _mi_heap_malloc_zero(heap, size + alignment - 1, zero); if (p == null) { return(null); } // .. and align within the allocation nuint adjust = alignment - (((nuint)p + offset) & align_mask); mi_assert_internal((MI_DEBUG > 1) && (adjust <= alignment)); void *aligned_p = adjust == alignment ? p : (void *)((nuint)p + adjust); if (aligned_p != p) { mi_page_set_has_aligned(_mi_ptr_page(p), true); } mi_assert_internal((MI_DEBUG > 1) && (((nuint)aligned_p + offset) % alignment == 0)); mi_assert_internal((MI_DEBUG > 1) && (p == _mi_page_ptr_unalign(_mi_ptr_segment(aligned_p), _mi_ptr_page(aligned_p), aligned_p))); return(aligned_p); }
// The current small page array is for efficiency and for each // small size (up to 256) it points directly to the page for that // size without having to compute the bin. This means when the // current free page queue is updated for a small bin, we need to update a // range of entries in `_mi_page_small_free`. private static void mi_heap_queue_first_update(mi_heap_t *heap, [NativeTypeName("const mi_page_queue_t*")] mi_page_queue_t *pq) { mi_assert_internal((MI_DEBUG > 1) && mi_heap_contains_queue(heap, pq)); nuint size = pq->block_size; if (size > MI_SMALL_SIZE_MAX) { return; } mi_page_t *page = pq->first; if (pq->first == null) { page = (mi_page_t *)_mi_page_empty; } // find index in the right direct page array nuint idx = _mi_wsize_from_size(size); nuint start; mi_page_t **pages_free = &heap->pages_free_direct.e0; if (pages_free[idx] == page) { // already set return; } // find start slot if (idx <= 1) { start = 0; } else { // find previous size; due to minimal alignment upto 3 previous bins may need to be skipped byte bin = _mi_bin(size); mi_page_queue_t *prev = pq - 1; while ((bin == _mi_bin(prev->block_size)) && (prev > &heap->pages.e0)) { prev--; } start = 1 + _mi_wsize_from_size(prev->block_size); if (start > idx) { start = idx; } } // set size range to the right page mi_assert((MI_DEBUG != 0) && (start <= idx)); for (nuint sz = start; sz <= idx; sz++) { pages_free[sz] = page; } }
private static void mi_page_queue_push(mi_heap_t *heap, mi_page_queue_t *queue, mi_page_t *page) { mi_assert_internal((MI_DEBUG > 1) && (mi_page_heap(page) == heap)); mi_assert_internal((MI_DEBUG > 1) && (!mi_page_queue_contains(queue, page))); mi_assert_internal((MI_DEBUG > 1) && (_mi_page_segment(page)->page_kind != MI_PAGE_HUGE)); mi_assert_internal((MI_DEBUG > 1) && ((page->xblock_size == queue->block_size) || ((page->xblock_size > MI_LARGE_OBJ_SIZE_MAX) && mi_page_queue_is_huge(queue)) || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)))); mi_page_set_in_full(page, mi_page_queue_is_full(queue)); // mi_atomic_store_ptr_release(ref page->heap, heap); page->next = queue->first; page->prev = null; if (queue->first != null) { mi_assert_internal((MI_DEBUG > 1) && (queue->first->prev == null)); queue->first->prev = page; queue->first = page; } else { queue->first = queue->last = page; } // update direct mi_heap_queue_first_update(heap, queue); heap->page_count++; }
private static byte *mi_page_area([NativeTypeName("const mi_page_t*")] mi_page_t *page) { mi_assert_internal((MI_DEBUG > 1) && (MI_DEBUG >= 3)); return(_mi_page_start(_mi_page_segment(page), page, out _)); }
private static partial void mi_page_init(mi_heap_t *heap, mi_page_t *page, [NativeTypeName("size_t")] nuint block_size, mi_tld_t *tld);
private static partial void mi_page_extend_free(mi_heap_t *heap, mi_page_t *page, mi_tld_t *tld);