From f37a994c68fff3ac5dfec30a5bcd8c987a0033e2 Mon Sep 17 00:00:00 2001 From: Ondrej Mirtes Date: Sun, 16 Aug 2026 19:19:59 +0200 Subject: [PATCH 1/3] Cache freed VM stack pages instead of freeing them immediately A call stack whose depth oscillates across a VM stack page boundary allocates and frees a 256KB page on every oscillation: zend_vm_stack_extend() on the way down, and the immediate efree() in zend_vm_stack_free_call_frame_ex() on the way back. Every such allocation goes through zend_mm_alloc_large(), whose search for contiguous free pages degrades on a large, fragmented heap - in the worst case scanning every chunk's free-page bitmap, failing, and paying an mmap/munmap round-trip per oscillation. Long-running processes with multi-GB heaps hit this hard. Profiling PHPStan analysing a large codebase in a single process showed 73-90% of all CPU time in zend_mm_alloc_pages reached from the ZEND_INIT_METHOD_CALL handlers, i.e. pure VM stack page churn. An isolated reproducer (recursion to a fixed depth in a loop, heap pre-fragmented with interleaved small allocations) runs 15-22x slower than on a fresh heap. Keep up to 32 freed standard-size pages in a per-executor free list and serve zend_vm_stack_new_page() from it; flush the list in zend_vm_stack_destroy(). Oversized pages (frames larger than the page size) are still freed eagerly. With the cache, the fragmented-heap reproducer matches the fresh-heap numbers at every depth, and deep recursion on a fresh heap improves as well (331.7 -> 229.6us per oscillation at depth 9000), since even a fast allocator round-trip is slower than popping a cached page. Benchmark and reproducer: https://gist.github.com/ondrejmirtes/1c1bc4894e63ddcb6c58d7bfe59cdb7a --- Zend/zend_execute.c | 7 +++++++ Zend/zend_execute.h | 18 ++++++++++++++++-- Zend/zend_globals.h | 2 ++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/Zend/zend_execute.c b/Zend/zend_execute.c index 14a340ffee37..2a3746d0972e 100644 --- a/Zend/zend_execute.c +++ b/Zend/zend_execute.c @@ -210,6 +210,13 @@ ZEND_API void zend_vm_stack_destroy(void) { zend_vm_stack stack = EG(vm_stack); + while (EG(vm_stack_page_cache) != NULL) { + zend_vm_stack cached = EG(vm_stack_page_cache); + EG(vm_stack_page_cache) = cached->prev; + efree(cached); + } + EG(vm_stack_page_cache_count) = 0; + while (stack != NULL) { zend_vm_stack p = stack->prev; efree(stack); diff --git a/Zend/zend_execute.h b/Zend/zend_execute.h index 017c8d208a14..8746b9ad6ede 100644 --- a/Zend/zend_execute.h +++ b/Zend/zend_execute.h @@ -325,7 +325,14 @@ ZEND_API void zend_vm_stack_destroy(void); ZEND_API void* zend_vm_stack_extend(size_t size); static zend_always_inline zend_vm_stack zend_vm_stack_new_page(size_t size, zend_vm_stack prev) { - zend_vm_stack page = (zend_vm_stack)emalloc(size); + zend_vm_stack page = EG(vm_stack_page_cache); + + if (page != NULL && EXPECTED((size_t)((char*)page->end - (char*)page) == size)) { + EG(vm_stack_page_cache) = page->prev; + EG(vm_stack_page_cache_count)--; + } else { + page = (zend_vm_stack)emalloc(size); + } page->top = ZEND_VM_STACK_ELEMENTS(page); page->end = (zval*)((char*)page + size); @@ -421,7 +428,14 @@ static zend_always_inline void zend_vm_stack_free_call_frame_ex(uint32_t call_in EG(vm_stack_top) = prev->top; EG(vm_stack_end) = prev->end; EG(vm_stack) = prev; - efree(p); + if (EG(vm_stack_page_cache_count) < 32 + && (size_t)((char*)p->end - (char*)p) == EG(vm_stack_page_size)) { + p->prev = EG(vm_stack_page_cache); + EG(vm_stack_page_cache) = p; + EG(vm_stack_page_cache_count)++; + } else { + efree(p); + } } else { EG(vm_stack_top) = (zval*)call; } diff --git a/Zend/zend_globals.h b/Zend/zend_globals.h index 4d5e300e2859..2c99f017228e 100644 --- a/Zend/zend_globals.h +++ b/Zend/zend_globals.h @@ -202,6 +202,8 @@ struct _zend_executor_globals { zval *vm_stack_end; zend_vm_stack vm_stack; size_t vm_stack_page_size; + zend_vm_stack vm_stack_page_cache; + uint32_t vm_stack_page_cache_count; struct _zend_execute_data *current_execute_data; const zend_class_entry *fake_scope; /* used to avoid checks accessing properties */ From c675c1d52d8c28b169a92a6784a6310d4659fd92 Mon Sep 17 00:00:00 2001 From: Arnaud Le Blanc Date: Wed, 26 Aug 2026 11:50:58 +0200 Subject: [PATCH 2/3] Initialize vm_stack_page_cache, vm_stack_page_cache_count EG() is not zeroed in ZTS builds --- Zend/zend_execute.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Zend/zend_execute.c b/Zend/zend_execute.c index 2a3746d0972e..4df4f4749791 100644 --- a/Zend/zend_execute.c +++ b/Zend/zend_execute.c @@ -191,6 +191,8 @@ ZEND_API const zend_internal_function zend_pass_function = { ZEND_API void zend_vm_stack_init(void) { EG(vm_stack_page_size) = ZEND_VM_STACK_PAGE_SIZE; + EG(vm_stack_page_cache) = NULL; + EG(vm_stack_page_cache_count) = 0; EG(vm_stack) = zend_vm_stack_new_page(ZEND_VM_STACK_PAGE_SIZE, NULL); EG(vm_stack_top) = EG(vm_stack)->top; EG(vm_stack_end) = EG(vm_stack)->end; @@ -201,6 +203,8 @@ ZEND_API void zend_vm_stack_init_ex(size_t page_size) /* page_size must be a power of 2 */ ZEND_ASSERT(page_size > 0 && (page_size & (page_size - 1)) == 0); EG(vm_stack_page_size) = page_size; + EG(vm_stack_page_cache) = NULL; + EG(vm_stack_page_cache_count) = 0; EG(vm_stack) = zend_vm_stack_new_page(page_size, NULL); EG(vm_stack_top) = EG(vm_stack)->top; EG(vm_stack_end) = EG(vm_stack)->end; From d15d8317d8ee3f8bcf63133f3c0cd330ed31cb0d Mon Sep 17 00:00:00 2001 From: Arnaud Le Blanc Date: Wed, 26 Aug 2026 13:55:29 +0200 Subject: [PATCH 3/3] Use a separate cache for fibers Avoid poluting the main cache with different-sized pages, and do not clear the main cache when a fiber terminates --- Zend/zend_execute.c | 22 +++++++++++++--- Zend/zend_execute.h | 56 +++++++++++++++++++++++++++++++++-------- Zend/zend_execute_API.c | 1 + Zend/zend_fibers.h | 1 - Zend/zend_globals.h | 3 +++ 5 files changed, 67 insertions(+), 16 deletions(-) diff --git a/Zend/zend_execute.c b/Zend/zend_execute.c index 4df4f4749791..8b9e05e83160 100644 --- a/Zend/zend_execute.c +++ b/Zend/zend_execute.c @@ -193,6 +193,8 @@ ZEND_API void zend_vm_stack_init(void) EG(vm_stack_page_size) = ZEND_VM_STACK_PAGE_SIZE; EG(vm_stack_page_cache) = NULL; EG(vm_stack_page_cache_count) = 0; + EG(fiber_vm_stack_page_cache) = NULL; + EG(fiber_vm_stack_page_cache_count) = 0; EG(vm_stack) = zend_vm_stack_new_page(ZEND_VM_STACK_PAGE_SIZE, NULL); EG(vm_stack_top) = EG(vm_stack)->top; EG(vm_stack_end) = EG(vm_stack)->end; @@ -205,6 +207,8 @@ ZEND_API void zend_vm_stack_init_ex(size_t page_size) EG(vm_stack_page_size) = page_size; EG(vm_stack_page_cache) = NULL; EG(vm_stack_page_cache_count) = 0; + EG(fiber_vm_stack_page_cache) = NULL; + EG(fiber_vm_stack_page_cache_count) = 0; EG(vm_stack) = zend_vm_stack_new_page(page_size, NULL); EG(vm_stack_top) = EG(vm_stack)->top; EG(vm_stack_end) = EG(vm_stack)->end; @@ -214,6 +218,15 @@ ZEND_API void zend_vm_stack_destroy(void) { zend_vm_stack stack = EG(vm_stack); + while (stack != NULL) { + zend_vm_stack p = stack->prev; + efree(stack); + stack = p; + } +} + +ZEND_API void zend_vm_stack_destroy_caches(void) +{ while (EG(vm_stack_page_cache) != NULL) { zend_vm_stack cached = EG(vm_stack_page_cache); EG(vm_stack_page_cache) = cached->prev; @@ -221,11 +234,12 @@ ZEND_API void zend_vm_stack_destroy(void) } EG(vm_stack_page_cache_count) = 0; - while (stack != NULL) { - zend_vm_stack p = stack->prev; - efree(stack); - stack = p; + while (EG(fiber_vm_stack_page_cache) != NULL) { + zend_vm_stack cached = EG(fiber_vm_stack_page_cache); + EG(fiber_vm_stack_page_cache) = cached->prev; + efree(cached); } + EG(fiber_vm_stack_page_cache_count) = 0; } ZEND_API void* zend_vm_stack_extend(size_t size) diff --git a/Zend/zend_execute.h b/Zend/zend_execute.h index 8746b9ad6ede..2250a873af29 100644 --- a/Zend/zend_execute.h +++ b/Zend/zend_execute.h @@ -322,15 +322,39 @@ ZEND_STATIC_ASSERT(ZEND_MM_ALIGNED_SIZE(sizeof(zval)) == sizeof(zval), ZEND_API void zend_vm_stack_init(void); ZEND_API void zend_vm_stack_init_ex(size_t page_size); ZEND_API void zend_vm_stack_destroy(void); +ZEND_API void zend_vm_stack_destroy_caches(void); ZEND_API void* zend_vm_stack_extend(size_t size); -static zend_always_inline zend_vm_stack zend_vm_stack_new_page(size_t size, zend_vm_stack prev) { - zend_vm_stack page = EG(vm_stack_page_cache); +#define ZEND_FIBER_VM_STACK_SIZE (1024 * sizeof(zval)) + +static zend_always_inline zend_vm_stack zend_vm_stack_cached_page(size_t size) { + zend_vm_stack page; - if (page != NULL && EXPECTED((size_t)((char*)page->end - (char*)page) == size)) { - EG(vm_stack_page_cache) = page->prev; - EG(vm_stack_page_cache_count)--; + if (size == ZEND_FIBER_VM_STACK_SIZE) { + page = EG(fiber_vm_stack_page_cache); + if (page) { + ZEND_ASSERT((size_t)((char*)page->end - (char*)page) == size); + EG(fiber_vm_stack_page_cache) = page->prev; + EG(fiber_vm_stack_page_cache_count)--; + return page; + } } else { + page = EG(vm_stack_page_cache); + ZEND_ASSERT(!page || ((size_t)((char*)page->end - (char*)page) == size) || size != EG(vm_stack_page_size)); + if (page && EXPECTED((size_t)((char*)page->end - (char*)page) == size)) { + EG(vm_stack_page_cache) = page->prev; + EG(vm_stack_page_cache_count)--; + return page; + } + } + + return NULL; +} + +static zend_always_inline zend_vm_stack zend_vm_stack_new_page(size_t size, zend_vm_stack prev) { + zend_vm_stack page = zend_vm_stack_cached_page(size); + + if (!page) { page = (zend_vm_stack)emalloc(size); } @@ -428,13 +452,23 @@ static zend_always_inline void zend_vm_stack_free_call_frame_ex(uint32_t call_in EG(vm_stack_top) = prev->top; EG(vm_stack_end) = prev->end; EG(vm_stack) = prev; - if (EG(vm_stack_page_cache_count) < 32 - && (size_t)((char*)p->end - (char*)p) == EG(vm_stack_page_size)) { - p->prev = EG(vm_stack_page_cache); - EG(vm_stack_page_cache) = p; - EG(vm_stack_page_cache_count)++; + if ((size_t)((char*)p->end - (char*)p) == ZEND_FIBER_VM_STACK_SIZE) { + if (EG(fiber_vm_stack_page_cache_count) < 32) { + p->prev = EG(fiber_vm_stack_page_cache); + EG(fiber_vm_stack_page_cache) = p; + EG(fiber_vm_stack_page_cache_count)++; + } else { + efree(p); + } } else { - efree(p); + if (EG(vm_stack_page_cache_count) < 32 + && (size_t)((char*)p->end - (char*)p) == EG(vm_stack_page_size)) { + p->prev = EG(vm_stack_page_cache); + EG(vm_stack_page_cache) = p; + EG(vm_stack_page_cache_count)++; + } else { + efree(p); + } } } else { EG(vm_stack_top) = (zval*)call; diff --git a/Zend/zend_execute_API.c b/Zend/zend_execute_API.c index b48352a5aaf3..c10e88c12118 100644 --- a/Zend/zend_execute_API.c +++ b/Zend/zend_execute_API.c @@ -482,6 +482,7 @@ void shutdown_executor(void) /* {{{ */ zend_hash_discard(EG(class_table), EG(persistent_classes_count)); } else { zend_vm_stack_destroy(); + zend_vm_stack_destroy_caches(); if (EG(full_tables_cleanup)) { zend_hash_reverse_apply(EG(function_table), clean_non_persistent_function_full); diff --git a/Zend/zend_fibers.h b/Zend/zend_fibers.h index c72ffdc8f18e..ec495c9ab9e3 100644 --- a/Zend/zend_fibers.h +++ b/Zend/zend_fibers.h @@ -25,7 +25,6 @@ #define ZEND_FIBER_GUARD_PAGES 1 #define ZEND_FIBER_DEFAULT_C_STACK_SIZE (4096 * (((sizeof(void *)) < 8) ? 256 : 512)) -#define ZEND_FIBER_VM_STACK_SIZE (1024 * sizeof(zval)) BEGIN_EXTERN_C() diff --git a/Zend/zend_globals.h b/Zend/zend_globals.h index 2c99f017228e..7c68c780387e 100644 --- a/Zend/zend_globals.h +++ b/Zend/zend_globals.h @@ -204,6 +204,9 @@ struct _zend_executor_globals { size_t vm_stack_page_size; zend_vm_stack vm_stack_page_cache; uint32_t vm_stack_page_cache_count; + /* Fibers use a different page size, so they need a separate cache */ + zend_vm_stack fiber_vm_stack_page_cache; + uint32_t fiber_vm_stack_page_cache_count; struct _zend_execute_data *current_execute_data; const zend_class_entry *fake_scope; /* used to avoid checks accessing properties */