Line data Source code
1 : //
2 : // Copyright (c) 2025 Vinnie Falco (vinnie dot falco at gmail dot com)
3 : //
4 : // Distributed under the Boost Software License, Version 1.0. (See accompanying
5 : // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 : //
7 : // Official repository: https://github.com/cppalliance/capy
8 : //
9 :
10 : #ifndef BOOST_CAPY_FRAME_ALLOCATOR_HPP
11 : #define BOOST_CAPY_FRAME_ALLOCATOR_HPP
12 :
13 : #include <boost/capy/detail/config.hpp>
14 : #include <boost/capy/concept/frame_allocator.hpp>
15 :
16 : #include <cstddef>
17 : #include <cstdint>
18 : #include <new>
19 : #include <utility>
20 :
21 : namespace boost {
22 : namespace capy {
23 :
24 : //----------------------------------------------------------
25 : // Public API
26 : //----------------------------------------------------------
27 :
28 : /** A frame allocator that passes through to global new/delete.
29 :
30 : This allocator provides no pooling or recycling—each allocation
31 : goes directly to `::operator new` and each deallocation goes to
32 : `::operator delete`. It serves as a baseline for comparison and
33 : as a fallback when pooling is not desired.
34 : */
35 : struct default_frame_allocator
36 : {
37 : void* allocate(std::size_t n)
38 : {
39 : return ::operator new(n);
40 : }
41 :
42 : void deallocate(void* p, std::size_t)
43 : {
44 : ::operator delete(p);
45 : }
46 : };
47 :
48 : static_assert(frame_allocator<default_frame_allocator>);
49 :
50 : //----------------------------------------------------------
51 : // Implementation details
52 : //----------------------------------------------------------
53 :
54 : namespace detail {
55 :
56 : /** Abstract base class for internal frame allocator wrappers.
57 :
58 : This class provides a polymorphic interface used internally
59 : by the frame allocation machinery. User-defined allocators
60 : do not inherit from this class.
61 : */
62 : class frame_allocator_base
63 : {
64 : public:
65 122 : virtual ~frame_allocator_base() {}
66 :
67 : /** Allocate memory for a coroutine frame.
68 :
69 : @param n The number of bytes to allocate.
70 :
71 : @return A pointer to the allocated memory.
72 : */
73 : virtual void* allocate(std::size_t n) = 0;
74 :
75 : /** Deallocate memory for a child coroutine frame.
76 :
77 : @param p Pointer to the memory to deallocate.
78 : @param n The user-requested size (not total allocation).
79 : */
80 : virtual void deallocate(void* p, std::size_t n) = 0;
81 :
82 : /** Deallocate the first coroutine frame (where this wrapper is embedded).
83 :
84 : This method handles the special case where the wrapper itself
85 : is embedded at the end of the block being deallocated.
86 :
87 : @param block Pointer to the block to deallocate.
88 : @param user_size The user-requested size (not total allocation).
89 : */
90 : virtual void deallocate_embedded(void* block, std::size_t user_size) = 0;
91 : };
92 :
93 : // Forward declaration
94 : template<frame_allocator Allocator>
95 : class frame_allocator_wrapper;
96 :
97 : /** Wrapper that embeds a frame_allocator_wrapper in the first allocation.
98 :
99 : This wrapper lives on the stack (in async_run_awaitable) and is used only
100 : for the FIRST coroutine frame allocation. It embeds a copy of
101 : frame_allocator_wrapper at the end of the allocated block, then
102 : updates TLS to point to that embedded wrapper for subsequent
103 : allocations.
104 :
105 : @tparam Allocator The underlying allocator type satisfying frame_allocator.
106 : */
107 : template<frame_allocator Allocator>
108 : class embedding_frame_allocator : public frame_allocator_base
109 : {
110 : Allocator alloc_;
111 :
112 : static constexpr std::size_t alignment = alignof(void*);
113 :
114 : static_assert(
115 : alignof(frame_allocator_wrapper<Allocator>) <= alignment,
116 : "alignment must be at least as strict as wrapper alignment");
117 :
118 : static std::size_t
119 61 : aligned_offset(std::size_t n) noexcept
120 : {
121 61 : return (n + alignment - 1) & ~(alignment - 1);
122 : }
123 :
124 : public:
125 61 : explicit embedding_frame_allocator(Allocator a)
126 61 : : alloc_(std::move(a))
127 : {
128 61 : }
129 :
130 : void*
131 : allocate(std::size_t n) override;
132 :
133 : void
134 0 : deallocate(void*, std::size_t) override
135 : {
136 : // Never called - stack wrapper not used for deallocation
137 0 : }
138 :
139 : void
140 0 : deallocate_embedded(void*, std::size_t) override
141 : {
142 : // Never called
143 0 : }
144 : };
145 :
146 : /** Wrapper embedded in the first coroutine frame.
147 :
148 : This wrapper is constructed at the end of the first coroutine
149 : frame by embedding_frame_allocator. It handles all subsequent
150 : allocations (storing a pointer to itself) and all deallocations.
151 :
152 : IMPORTANT: This wrapper stores a POINTER to the allocator in the
153 : launcher's embedder, not a copy. This is safe because Frame #1
154 : (where this wrapper lives) is always destroyed before Frame #2
155 : (the launcher, where embedder lives).
156 :
157 : @tparam Allocator The underlying allocator type satisfying frame_allocator.
158 : */
159 : template<frame_allocator Allocator>
160 : class frame_allocator_wrapper : public frame_allocator_base
161 : {
162 : Allocator* alloc_; // Pointer, not copy
163 :
164 : static constexpr std::size_t alignment = alignof(void*);
165 :
166 : static std::size_t
167 411 : aligned_offset(std::size_t n) noexcept
168 : {
169 411 : return (n + alignment - 1) & ~(alignment - 1);
170 : }
171 :
172 : public:
173 61 : explicit frame_allocator_wrapper(Allocator& a)
174 61 : : alloc_(&a)
175 : {
176 61 : }
177 :
178 : void*
179 175 : allocate(std::size_t n) override
180 : {
181 : // Layout: [frame | ptr]
182 175 : std::size_t ptr_offset = aligned_offset(n);
183 175 : std::size_t total = ptr_offset + sizeof(frame_allocator_base*);
184 :
185 175 : void* raw = alloc_->allocate(total);
186 :
187 : // Store untagged pointer to self at fixed offset
188 175 : auto* ptr_loc = reinterpret_cast<frame_allocator_base**>(
189 : static_cast<char*>(raw) + ptr_offset);
190 175 : *ptr_loc = this;
191 :
192 175 : return raw;
193 : }
194 :
195 : void
196 175 : deallocate(void* block, std::size_t user_size) override
197 : {
198 : // Child frame deallocation: layout is [frame | ptr]
199 175 : std::size_t ptr_offset = aligned_offset(user_size);
200 175 : std::size_t total = ptr_offset + sizeof(frame_allocator_base*);
201 175 : alloc_->deallocate(block, total);
202 175 : }
203 :
204 : void
205 61 : deallocate_embedded(void* block, std::size_t user_size) override
206 : {
207 : // First frame deallocation: layout is [frame | ptr | wrapper]
208 61 : std::size_t ptr_offset = aligned_offset(user_size);
209 61 : std::size_t wrapper_offset = ptr_offset + sizeof(frame_allocator_base*);
210 61 : std::size_t total = wrapper_offset + sizeof(frame_allocator_wrapper);
211 :
212 : // Safe to use alloc_ pointer because embedder (in Frame #2)
213 : // is guaranteed to outlive this wrapper (in Frame #1)
214 61 : Allocator* alloc_ptr = alloc_; // Save pointer before destroying self
215 61 : this->~frame_allocator_wrapper();
216 61 : alloc_ptr->deallocate(block, total);
217 61 : }
218 : };
219 :
220 : } // namespace detail
221 :
222 : /** Mixin base for promise types to support custom frame allocation.
223 :
224 : Derive your promise_type from this class to enable custom coroutine
225 : frame allocation via a thread-local allocator pointer.
226 :
227 : The allocation strategy:
228 : @li If a thread-local allocator is set, use it for allocation
229 : @li Otherwise, fall back to global `::operator new`/`::operator delete`
230 :
231 : A pointer is stored at the end of each allocation to enable correct
232 : deallocation regardless of which allocator was active at allocation time.
233 :
234 : @par Memory Layout
235 :
236 : For the first coroutine frame (allocated via embedding_frame_allocator):
237 : @code
238 : [coroutine frame | tagged_ptr | frame_allocator_wrapper]
239 : @endcode
240 :
241 : For subsequent frames (allocated via frame_allocator_wrapper):
242 : @code
243 : [coroutine frame | ptr]
244 : @endcode
245 :
246 : The tag bit (low bit) distinguishes the two cases during deallocation.
247 :
248 : @see frame_allocator
249 : */
250 : struct frame_allocating_base
251 : {
252 : private:
253 : static constexpr std::size_t alignment = alignof(void*);
254 :
255 : static std::size_t
256 296 : aligned_offset(std::size_t n) noexcept
257 : {
258 296 : return (n + alignment - 1) & ~(alignment - 1);
259 : }
260 :
261 : static detail::frame_allocator_base*&
262 940 : current_allocator() noexcept
263 : {
264 : static thread_local detail::frame_allocator_base* alloc = nullptr;
265 940 : return alloc;
266 : }
267 :
268 : public:
269 : /** Set the thread-local frame allocator.
270 :
271 : The allocator will be used for subsequent coroutine frame
272 : allocations on this thread until changed or cleared.
273 :
274 : @param alloc The allocator to use. Must outlive all coroutines
275 : allocated with it.
276 : */
277 : static void
278 408 : set_frame_allocator(detail::frame_allocator_base& alloc) noexcept
279 : {
280 408 : current_allocator() = &alloc;
281 408 : }
282 :
283 : /** Clear the thread-local frame allocator.
284 :
285 : Subsequent allocations will use global `::operator new`.
286 : */
287 : static void
288 61 : clear_frame_allocator() noexcept
289 : {
290 61 : current_allocator() = nullptr;
291 61 : }
292 :
293 : /** Get the current thread-local frame allocator.
294 :
295 : @return Pointer to current allocator, or nullptr if none set.
296 : */
297 : static detail::frame_allocator_base*
298 205 : get_frame_allocator() noexcept
299 : {
300 205 : return current_allocator();
301 : }
302 :
303 : static void*
304 266 : operator new(std::size_t size)
305 : {
306 266 : auto* alloc = current_allocator();
307 266 : if(!alloc)
308 : {
309 : // No allocator: allocate extra space for null pointer marker
310 30 : std::size_t ptr_offset = aligned_offset(size);
311 30 : std::size_t total = ptr_offset + sizeof(detail::frame_allocator_base*);
312 30 : void* raw = ::operator new(total);
313 :
314 : // Store nullptr to indicate global new/delete
315 30 : auto* ptr_loc = reinterpret_cast<detail::frame_allocator_base**>(
316 : static_cast<char*>(raw) + ptr_offset);
317 30 : *ptr_loc = nullptr;
318 :
319 30 : return raw;
320 : }
321 236 : return alloc->allocate(size);
322 : }
323 :
324 : /** Deallocate a coroutine frame.
325 :
326 : Reads the pointer stored at the end of the frame to find
327 : the allocator. The tag bit (low bit) indicates whether
328 : this is the first frame (with embedded wrapper) or a
329 : child frame (with pointer to external wrapper).
330 :
331 : A null pointer indicates the frame was allocated with
332 : global new/delete (no custom allocator was active).
333 : */
334 : static void
335 266 : operator delete(void* ptr, std::size_t size)
336 : {
337 : // Pointer is always at aligned_offset(size)
338 266 : std::size_t ptr_offset = aligned_offset(size);
339 266 : auto* ptr_loc = reinterpret_cast<detail::frame_allocator_base**>(
340 : static_cast<char*>(ptr) + ptr_offset);
341 266 : auto raw_ptr = reinterpret_cast<std::uintptr_t>(*ptr_loc);
342 :
343 : // Null pointer means global new/delete
344 266 : if(raw_ptr == 0)
345 : {
346 : #if __cpp_sized_deallocation >= 201309
347 30 : std::size_t total = ptr_offset + sizeof(detail::frame_allocator_base*);
348 30 : ::operator delete(ptr, total);
349 : #else
350 : ::operator delete(ptr);
351 : #endif
352 30 : return;
353 : }
354 :
355 : // Tag bit distinguishes first frame (embedded) from child frames
356 236 : bool is_embedded = raw_ptr & 1;
357 236 : auto* wrapper = reinterpret_cast<detail::frame_allocator_base*>(
358 236 : raw_ptr & ~std::uintptr_t(1));
359 :
360 236 : if(is_embedded)
361 61 : wrapper->deallocate_embedded(ptr, size);
362 : else
363 175 : wrapper->deallocate(ptr, size);
364 : }
365 : };
366 :
367 : //----------------------------------------------------------
368 : // embedding_frame_allocator implementation
369 : // (must come after frame_allocating_base is defined)
370 : //----------------------------------------------------------
371 :
372 : namespace detail {
373 :
374 : template<frame_allocator Allocator>
375 : void*
376 61 : embedding_frame_allocator<Allocator>::allocate(std::size_t n)
377 : {
378 : // Layout: [frame | ptr | wrapper]
379 61 : std::size_t ptr_offset = aligned_offset(n);
380 61 : std::size_t wrapper_offset = ptr_offset + sizeof(frame_allocator_base*);
381 61 : std::size_t total = wrapper_offset + sizeof(frame_allocator_wrapper<Allocator>);
382 :
383 61 : void* raw = alloc_.allocate(total);
384 :
385 : // Construct embedded wrapper after the pointer
386 : // Pass REFERENCE to alloc_ so wrapper stores pointer, not copy
387 61 : auto* wrapper_loc = static_cast<char*>(raw) + wrapper_offset;
388 61 : auto* embedded = new (wrapper_loc) frame_allocator_wrapper<Allocator>(alloc_);
389 :
390 : // Store tagged pointer at fixed offset (bit 0 set = embedded)
391 61 : auto* ptr_loc = reinterpret_cast<frame_allocator_base**>(
392 : static_cast<char*>(raw) + ptr_offset);
393 61 : *ptr_loc = reinterpret_cast<frame_allocator_base*>(
394 61 : reinterpret_cast<std::uintptr_t>(embedded) | 1);
395 :
396 : // Update TLS to embedded wrapper for subsequent allocations
397 61 : frame_allocating_base::set_frame_allocator(*embedded);
398 :
399 61 : return raw;
400 : }
401 :
402 : } // namespace detail
403 :
404 : } // namespace capy
405 : } // namespace boost
406 :
407 : #endif
|
