Integrate mimalloc v3

Author: daanx

Doc/license.rst

@@ -1034,7 +1034,7 @@ mimalloc
 
 MIT License::
 
-   Copyright (c) 2018-2021 Microsoft Corporation, Daan Leijen
+   Copyright (c) 2018-2025 Microsoft Corporation, Daan Leijen
 
    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to deal

Include/internal/pycore_mimalloc.h

@@ -5,7 +5,7 @@
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#if defined(MIMALLOC_H) || defined(MIMALLOC_TYPES_H)
+#if defined(MIMALLOC_H) || defined(MI_TYPES_H)
 #  error "pycore_mimalloc.h must be included before mimalloc.h"
 #endif
 
@@ -20,8 +20,21 @@ typedef enum {
 #include "pycore_pymem.h"
 
 #ifdef WITH_MIMALLOC
+#  define MI_NO_PTHREAD_ATFORK 1
 #  ifdef Py_GIL_DISABLED
 #    define MI_PRIM_THREAD_ID   _Py_ThreadId
+#    define MI_CPYTHON 1
+
+struct mi_page_qsbr_s {
+    uint64_t goal;
+};
+
+struct mi_heap_qsbr_s {
+    struct _qsbr_shared *shared;
+};
+#  else
+// Keep full pages in their owning thread's allocation statistics.
+#    define MI_DEFAULT_PAGE_FULL_RETAIN -1
 #  endif
 #  define MI_DEBUG_UNINIT     PYMEM_CLEANBYTE
 #  define MI_DEBUG_FREED      PYMEM_DEADBYTE
@@ -52,18 +65,17 @@ extern "C++" {
 
 #ifdef Py_GIL_DISABLED
 struct _mimalloc_interp_state {
-    // When exiting, threads place any segments with live blocks in this
-    // shared pool for other threads to claim and reuse.
-    mi_abandoned_pool_t abandoned_pool;
+    mi_heap_t *heaps[_Py_MIMALLOC_HEAP_COUNT];
 };
 
 struct _mimalloc_thread_state {
-    mi_heap_t *current_object_heap;
-    mi_heap_t heaps[_Py_MIMALLOC_HEAP_COUNT];
-    mi_tld_t tld;
+    mi_theap_t *current_object_heap;
+    mi_theap_t *heaps[_Py_MIMALLOC_HEAP_COUNT];
     int initialized;
-    struct llist_node page_list;
 };
+
+extern int _PyMem_InitMimalloc(PyInterpreterState *interp);
+extern void _PyMem_FiniMimalloc(PyInterpreterState *interp);
 #endif
 
 #endif // Py_INTERNAL_MIMALLOC_H

Include/internal/pycore_object_alloc.h

@@ -14,18 +14,18 @@ extern "C" {
 #endif
 
 #ifdef Py_GIL_DISABLED
-static inline mi_heap_t *
+static inline mi_theap_t *
 _PyObject_GetAllocationHeap(_PyThreadStateImpl *tstate, PyTypeObject *tp)
 {
     struct _mimalloc_thread_state *m = &tstate->mimalloc;
     if (_PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER)) {
-        return &m->heaps[_Py_MIMALLOC_HEAP_GC_PRE];
+        return m->heaps[_Py_MIMALLOC_HEAP_GC_PRE];
     }
     else if (_PyType_IS_GC(tp)) {
-        return &m->heaps[_Py_MIMALLOC_HEAP_GC];
+        return m->heaps[_Py_MIMALLOC_HEAP_GC];
     }
     else {
-        return &m->heaps[_Py_MIMALLOC_HEAP_OBJECT];
+        return m->heaps[_Py_MIMALLOC_HEAP_OBJECT];
     }
 }
 #endif
@@ -45,7 +45,7 @@ _PyObject_MallocWithType(PyTypeObject *tp, size_t size)
 #endif
     void *mem = PyObject_Malloc(size);
 #ifdef Py_GIL_DISABLED
-    m->current_object_heap = &m->heaps[_Py_MIMALLOC_HEAP_OBJECT];
+    m->current_object_heap = m->heaps[_Py_MIMALLOC_HEAP_OBJECT];
 #endif
     return mem;
 }
@@ -60,7 +60,7 @@ _PyObject_ReallocWithType(PyTypeObject *tp, void *ptr, size_t size)
 #endif
     void *mem = PyObject_Realloc(ptr, size);
 #ifdef Py_GIL_DISABLED
-    m->current_object_heap = &m->heaps[_Py_MIMALLOC_HEAP_OBJECT];
+    m->current_object_heap = m->heaps[_Py_MIMALLOC_HEAP_OBJECT];
 #endif
     return mem;
 }

InternalDocs/qsbr.md

@@ -136,6 +136,47 @@ by slightly delaying when memory can be reclaimed; the size-based thresholds
 above bound that extra memory.
 
 
+### Mimalloc Integration
+
+Each interpreter owns four mimalloc heaps: one for `PyMem` allocations and
+three for objects with different header layouts. Only the object heaps enable
+QSBR. Each thread state retains a `mi_theap_t` handle for each heap. Clearing a
+thread state abandons its pages without invalidating live objects; heap
+visitation includes both owned and abandoned pages, so they remain visible to
+the garbage collector.
+
+CPython supplies four typed hooks: `mi_heap_qsbr_init` associates a heap with its
+interpreter's QSBR state, `mi_page_qsbr_clear` and `mi_page_qsbr_is_clear` manage
+a page's goal, and `mi_page_qsbr_should_delay` establishes a goal if needed and
+checks cached read-sequence progress. These hooks may run without an attached
+Python thread state and while allocator locks are held. They must not poll the
+thread registry. `_PyMem_ProcessDelayed` polls outside those locks, then drains
+each object heap's thread-local delayed-page queue.
+
+Mimalloc owns the delayed-page queues and page transitions. A delayed empty
+page can still supply allocations of its existing size class. If it becomes
+empty again, its previous goal is cleared and a new grace period is required.
+Queues are FIFO; an incomplete goal at the head may delay later pages. During
+stop-the-world collection, CPython also sweeps abandoned pages using the same
+cached QSBR progress.
+
+The object's reference-count fields and any pre-header remain untouched by
+debug fills, using the heap's `debug_fill_preserve` setting. At interpreter
+shutdown, empty heaps can be destroyed, but live allocations may still be
+referenced elsewhere, including by a shared interned-string dictionary.
+Retained object heaps are disconnected from the interpreter's QSBR state and
+conservatively keep their pages rather than consulting a deleted domain.
+
+CPython disables mimalloc's automatic `pthread_atfork` handler and invokes its
+child cleanup from `PyOS_AfterFork_Child`, after a coordinated fork. Exec-only
+subprocess forks do not stop other allocator users and must not perform that
+cleanup or use the allocator before exec.
+
+The regular GIL build does not enable QSBR and retains full pages in their
+owning thread, as in mimalloc v2. Its allocation statistics visit only the
+current thread's pages, so early abandonment would make those counts incomplete.
+
+
 ## Limitations
 
 Determining the `rd_seq` requires scanning over all thread states. This operation

Lib/test/test_free_threading/test_gc.py

@@ -17,6 +17,29 @@ class MyObj:
 
 @threading_helper.requires_working_threading()
 class TestGC(TestCase):
+    def test_collect_cycles_after_thread_exit(self):
+        class Slotted:
+            __slots__ = ("cycle", "__weakref__")
+
+        refs = []
+
+        def worker():
+            for cls in (MyObj, Slotted):
+                for _ in range(200):
+                    obj = cls()
+                    obj.cycle = obj
+                    refs.append(weakref.ref(obj))
+            obj = Slotted()
+            obj.cycle = (obj,) * 100_000
+            refs.append(weakref.ref(obj))
+
+        with support.disable_gc():
+            with threading_helper.start_threads([Thread(target=worker)]):
+                pass
+            self.assertTrue(all(ref() is not None for ref in refs))
+            gc.collect()
+            self.assertTrue(all(ref() is None for ref in refs))
+
     def test_get_objects(self):
         event = threading.Event()
 

Lib/test/test_free_threading/test_interpreters.py

@@ -9,6 +9,22 @@
 
 
 class InterpreterTeardownTests(unittest.TestCase):
+    def test_shared_interned_strings_survive_teardown(self):
+        import_helper.import_module('_testcapi')
+        script = textwrap.dedent("""
+            import _testcapi
+            import gc
+            import sys
+
+            for i in range(10):
+                code = f'import sys; sys.intern("retained_{i}_" + "x" * 5000)'
+                assert _testcapi.run_in_subinterp(code) == 0
+                gc.collect()
+                expected = f"retained_{i}_" + "x" * 5000
+                assert sys.intern(expected) == expected
+        """)
+        script_helper.assert_python_ok('-c', script)
+
     def test_destroy_subinterpreter_does_not_abort(self):
         # gh-153176: destroy_interpreter(basic=True) used to call
         # PyThreadState_Clear() on a non-current thread state, which on a

Makefile.pre.in

@@ -374,12 +374,9 @@ IO_OBJS=	\
 
 MIMALLOC_HEADERS= \
 	$(srcdir)/Include/internal/pycore_mimalloc.h \
-	$(srcdir)/Include/internal/mimalloc/mimalloc.h \
-	$(srcdir)/Include/internal/mimalloc/mimalloc/atomic.h \
-	$(srcdir)/Include/internal/mimalloc/mimalloc/internal.h \
-	$(srcdir)/Include/internal/mimalloc/mimalloc/prim.h \
-	$(srcdir)/Include/internal/mimalloc/mimalloc/track.h \
-	$(srcdir)/Include/internal/mimalloc/mimalloc/types.h
+	$(wildcard $(srcdir)/Include/internal/mimalloc/*.h \
+		$(srcdir)/Include/internal/mimalloc/mimalloc/*.h \
+		$(srcdir)/Include/internal/mimalloc/mimalloc/*/*.h)
 
 
 ##########################################################################
@@ -2076,26 +2073,11 @@ Objects/unicodeobject.o: $(srcdir)/Objects/unicodeobject.c $(UNICODE_DEPS)
 Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h
 Objects/setobject.o: $(srcdir)/Objects/stringlib/eq.h
 
-Objects/obmalloc.o: $(srcdir)/Objects/mimalloc/alloc.c \
-		$(srcdir)/Objects/mimalloc/alloc-aligned.c \
-		$(srcdir)/Objects/mimalloc/alloc-posix.c \
-		$(srcdir)/Objects/mimalloc/arena.c \
-		$(srcdir)/Objects/mimalloc/bitmap.c \
-		$(srcdir)/Objects/mimalloc/heap.c \
-		$(srcdir)/Objects/mimalloc/init.c \
-		$(srcdir)/Objects/mimalloc/options.c \
-		$(srcdir)/Objects/mimalloc/os.c \
-		$(srcdir)/Objects/mimalloc/page.c \
-		$(srcdir)/Objects/mimalloc/random.c \
-		$(srcdir)/Objects/mimalloc/segment.c \
-		$(srcdir)/Objects/mimalloc/segment-map.c \
-		$(srcdir)/Objects/mimalloc/stats.c \
-		$(srcdir)/Objects/mimalloc/prim/prim.c \
-		$(srcdir)/Objects/mimalloc/prim/osx/prim.c \
-		$(srcdir)/Objects/mimalloc/prim/unix/prim.c \
-		$(srcdir)/Objects/mimalloc/prim/wasi/prim.c
-
-Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c
+Objects/obmalloc.o: $(wildcard $(srcdir)/Objects/mimalloc/*.c \
+		$(srcdir)/Objects/mimalloc/*/*.c \
+		$(srcdir)/Objects/mimalloc/*/*/*.c \
+		$(srcdir)/Objects/mimalloc/*/*.h \
+		$(srcdir)/Objects/mimalloc/*/*/*.h)
 
 
 # Regenerate various files from Python/bytecodes.c
@@ -2924,12 +2906,12 @@ inclinstall:
 		$(INSTALL_DATA) $$i $(DESTDIR)$(INCLUDEPY)/internal; \
 	done
 	@if test "$(INSTALL_MIMALLOC)" = "yes"; then \
-		echo $(INSTALL_DATA) $(srcdir)/Include/internal/mimalloc/mimalloc.h $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc.h; \
-		$(INSTALL_DATA) $(srcdir)/Include/internal/mimalloc/mimalloc.h $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc.h; \
-		for i in $(srcdir)/Include/internal/mimalloc/mimalloc/*.h; \
+		for i in $(filter-out $(srcdir)/Include/internal/pycore_mimalloc.h,$(MIMALLOC_HEADERS)); \
 		do \
-			echo $(INSTALL_DATA) $$i $(INCLUDEPY)/internal/mimalloc/mimalloc; \
-			$(INSTALL_DATA) $$i $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc; \
+			dest=$(DESTDIR)$(INCLUDEPY)/$${i#$(srcdir)/Include/}; \
+			$(INSTALL) -d -m $(DIRMODE) "$$(dirname "$$dest")"; \
+			echo $(INSTALL_DATA) $$i $$dest; \
+			$(INSTALL_DATA) $$i $$dest; \
 		done; \
 	fi
 	echo $(INSTALL_DATA) pyconfig.h $(DESTDIR)$(CONFINCLUDEPY)/pyconfig.h

Modules/posixmodule.c

@@ -760,6 +760,10 @@ PyOS_AfterFork_Child(void)
     PyStatus status;
     _PyRuntimeState *runtime = &_PyRuntime;
 
+#ifdef WITH_MIMALLOC
+    _mi_fork_child();
+#endif
+
     // re-creates runtime->interpreters.mutex (HEAD_UNLOCK)
     status = _PyRuntimeState_ReInitThreads(runtime);
     if (_PyStatus_EXCEPTION(status)) {